drivers/crypto/omap-aes-gcm.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for OMAP AES GCM HW acceleration.
   5  *
   6  * Copyright (c) 2016 Texas Instruments Incorporated
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as published
  10  * by the Free Software Foundation.
  11  *
  12  */
  13
  14 #include <linux/errno.h>
  15 #include <linux/scatterlist.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/dmaengine.h>
  18 #include <linux/omap-dma.h>
  19 #include <linux/interrupt.h>
  20 #include <crypto/aes.h>
  21 #include <crypto/gcm.h>
  22 #include <crypto/scatterwalk.h>
  23 #include <crypto/skcipher.h>
  24 #include <crypto/internal/aead.h>
  25
  26 #include "omap-crypto.h"
  27 #include "omap-aes.h"
  28
  29 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
  30                                      struct aead_request *req);
  31
  32 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
  33 {
  34         struct aead_request *req = dd->aead_req;
  35
  36         dd->flags &= ~FLAGS_BUSY;
  37         dd->in_sg = NULL;
  38         dd->out_sg = NULL;
  39
  40         req->base.complete(&req->base, ret);
  41 }
  42
  43 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
  44 {
  45         u8 *tag;
  46         int alen, clen, i, ret = 0, nsg;
  47         struct omap_aes_reqctx *rctx;
  48
  49         alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
  50         clen = ALIGN(dd->total, AES_BLOCK_SIZE);
  51         rctx = aead_request_ctx(dd->aead_req);
  52
  53         nsg = !!(dd->assoc_len && dd->total);
  54
  55         dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
  56                                DMA_FROM_DEVICE);
  57         dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  58         dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
  59         omap_aes_crypt_dma_stop(dd);
  60
  61         omap_crypto_cleanup(dd->out_sg, dd->orig_out,
  62                             dd->aead_req->assoclen, dd->total,
  63                             FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
  64
  65         if (dd->flags & FLAGS_ENCRYPT)
  66                 scatterwalk_map_and_copy(rctx->auth_tag,
  67                                          dd->aead_req->dst,
  68                                          dd->total + dd->aead_req->assoclen,
  69                                          dd->authsize, 1);
  70
  71         omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
  72                             FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
  73
  74         omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
  75                             FLAGS_IN_DATA_ST_SHIFT, dd->flags);
  76
  77         if (!(dd->flags & FLAGS_ENCRYPT)) {
  78                 tag = (u8 *)rctx->auth_tag;
  79                 for (i = 0; i < dd->authsize; i++) {
  80                         if (tag[i]) {
  81                                 dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
  82                                 ret = -EBADMSG;
  83                         }
  84                 }
  85         }
  86
  87         omap_aes_gcm_finish_req(dd, ret);
  88         omap_aes_gcm_handle_queue(dd, NULL);
  89 }
  90
  91 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
  92                                      struct aead_request *req)
  93 {
  94         int alen, clen, cryptlen, assoclen, ret;
  95         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  96         unsigned int authlen = crypto_aead_authsize(aead);
  97         struct scatterlist *tmp, sg_arr[2];
  98         int nsg;
  99         u16 flags;
 100
 101         assoclen = req->assoclen;
 102         cryptlen = req->cryptlen;
 103
 104         if (dd->flags & FLAGS_RFC4106_GCM)
 105                 assoclen -= 8;
 106
 107         if (!(dd->flags & FLAGS_ENCRYPT))
 108                 cryptlen -= authlen;
 109
 110         alen = ALIGN(assoclen, AES_BLOCK_SIZE);
 111         clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
 112
 113         nsg = !!(assoclen && cryptlen);
 114
 115         omap_aes_clear_copy_flags(dd);
 116
 117         sg_init_table(dd->in_sgl, nsg + 1);
 118         if (assoclen) {
 119                 tmp = req->src;
 120                 ret = omap_crypto_align_sg(&tmp, assoclen,
 121                                            AES_BLOCK_SIZE, dd->in_sgl,
 122                                            OMAP_CRYPTO_COPY_DATA |
 123                                            OMAP_CRYPTO_ZERO_BUF |
 124                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 125                                            FLAGS_ASSOC_DATA_ST_SHIFT,
 126                                            &dd->flags);
 127         }
 128
 129         if (cryptlen) {
 130                 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
 131
 132                 ret = omap_crypto_align_sg(&tmp, cryptlen,
 133                                            AES_BLOCK_SIZE, &dd->in_sgl[nsg],
 134                                            OMAP_CRYPTO_COPY_DATA |
 135                                            OMAP_CRYPTO_ZERO_BUF |
 136                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 137                                            FLAGS_IN_DATA_ST_SHIFT,
 138                                            &dd->flags);
 139         }
 140
 141         dd->in_sg = dd->in_sgl;
 142         dd->total = cryptlen;
 143         dd->assoc_len = assoclen;
 144         dd->authsize = authlen;
 145
 146         dd->out_sg = req->dst;
 147         dd->orig_out = req->dst;
 148
 149         dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
 150
 151         flags = 0;
 152         if (req->src == req->dst || dd->out_sg == sg_arr)
 153                 flags |= OMAP_CRYPTO_FORCE_COPY;
 154
 155         ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
 156                                    AES_BLOCK_SIZE, &dd->out_sgl,
 157                                    flags,
 158                                    FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
 159         if (ret)
 160                 return ret;
 161
 162         dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
 163         dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
 164
 165         return 0;
 166 }
 167
 168 static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
 169 {
 170         struct omap_aes_gcm_result *res = req->data;
 171
 172         if (err == -EINPROGRESS)
 173                 return;
 174
 175         res->err = err;
 176         complete(&res->completion);
 177 }
 178
 179 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
 180 {
 181         struct scatterlist iv_sg, tag_sg;
 182         struct skcipher_request *sk_req;
 183         struct omap_aes_gcm_result result;
 184         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 185         int ret = 0;
 186
 187         sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
 188         if (!sk_req) {
 189                 pr_err("skcipher: Failed to allocate request\n");
 190                 return -ENOMEM;
 191         }
 192
 193         init_completion(&result.completion);
 194
 195         sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
 196         sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
 197         skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 198                                       omap_aes_gcm_complete, &result);
 199         ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
 200         skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
 201                                    NULL);
 202         ret = crypto_skcipher_encrypt(sk_req);
 203         switch (ret) {
 204         case 0:
 205                 break;
 206         case -EINPROGRESS:
 207         case -EBUSY:
 208                 ret = wait_for_completion_interruptible(&result.completion);
 209                 if (!ret) {
 210                         ret = result.err;
 211                         if (!ret) {
 212                                 reinit_completion(&result.completion);
 213                                 break;
 214                         }
 215                 }
 216                 /* fall through */
 217         default:
 218                 pr_err("Encryption of IV failed for GCM mode\n");
 219                 break;
 220         }
 221
 222         skcipher_request_free(sk_req);
 223         return ret;
 224 }
 225
 226 void omap_aes_gcm_dma_out_callback(void *data)
 227 {
 228         struct omap_aes_dev *dd = data;
 229         struct omap_aes_reqctx *rctx;
 230         int i, val;
 231         u32 *auth_tag, tag[4];
 232
 233         if (!(dd->flags & FLAGS_ENCRYPT))
 234                 scatterwalk_map_and_copy(tag, dd->aead_req->src,
 235                                          dd->total + dd->aead_req->assoclen,
 236                                          dd->authsize, 0);
 237
 238         rctx = aead_request_ctx(dd->aead_req);
 239         auth_tag = (u32 *)rctx->auth_tag;
 240         for (i = 0; i < 4; i++) {
 241                 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
 242                 auth_tag[i] = val ^ auth_tag[i];
 243                 if (!(dd->flags & FLAGS_ENCRYPT))
 244                         auth_tag[i] = auth_tag[i] ^ tag[i];
 245         }
 246
 247         omap_aes_gcm_done_task(dd);
 248 }
 249
 250 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
 251                                      struct aead_request *req)
 252 {
 253         struct omap_aes_ctx *ctx;
 254         struct aead_request *backlog;
 255         struct omap_aes_reqctx *rctx;
 256         unsigned long flags;
 257         int err, ret = 0;
 258
 259         spin_lock_irqsave(&dd->lock, flags);
 260         if (req)
 261                 ret = aead_enqueue_request(&dd->aead_queue, req);
 262         if (dd->flags & FLAGS_BUSY) {
 263                 spin_unlock_irqrestore(&dd->lock, flags);
 264                 return ret;
 265         }
 266
 267         backlog = aead_get_backlog(&dd->aead_queue);
 268         req = aead_dequeue_request(&dd->aead_queue);
 269         if (req)
 270                 dd->flags |= FLAGS_BUSY;
 271         spin_unlock_irqrestore(&dd->lock, flags);
 272
 273         if (!req)
 274                 return ret;
 275
 276         if (backlog)
 277                 backlog->base.complete(&backlog->base, -EINPROGRESS);
 278
 279         ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 280         rctx = aead_request_ctx(req);
 281
 282         dd->ctx = ctx;
 283         rctx->dd = dd;
 284         dd->aead_req = req;
 285
 286         rctx->mode &= FLAGS_MODE_MASK;
 287         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 288
 289         err = omap_aes_gcm_copy_buffers(dd, req);
 290         if (err)
 291                 return err;
 292
 293         err = omap_aes_write_ctrl(dd);
 294         if (!err)
 295                 err = omap_aes_crypt_dma_start(dd);
 296
 297         if (err) {
 298                 omap_aes_gcm_finish_req(dd, err);
 299                 omap_aes_gcm_handle_queue(dd, NULL);
 300         }
 301
 302         return ret;
 303 }
 304
 305 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
 306 {
 307         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 308         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 309         unsigned int authlen = crypto_aead_authsize(aead);
 310         struct omap_aes_dev *dd;
 311         __be32 counter = cpu_to_be32(1);
 312         int err, assoclen;
 313
 314         memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
 315         memcpy(rctx->iv + GCM_AES_IV_SIZE, &counter, 4);
 316
 317         err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
 318         if (err)
 319                 return err;
 320
 321         if (mode & FLAGS_RFC4106_GCM)
 322                 assoclen = req->assoclen - 8;
 323         else
 324                 assoclen = req->assoclen;
 325         if (assoclen + req->cryptlen == 0) {
 326                 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
 327                                          1);
 328                 return 0;
 329         }
 330
 331         dd = omap_aes_find_dev(rctx);
 332         if (!dd)
 333                 return -ENODEV;
 334         rctx->mode = mode;
 335
 336         return omap_aes_gcm_handle_queue(dd, req);
 337 }
 338
 339 int omap_aes_gcm_encrypt(struct aead_request *req)
 340 {
 341         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 342
 343         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 344         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
 345 }
 346
 347 int omap_aes_gcm_decrypt(struct aead_request *req)
 348 {
 349         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 350
 351         memcpy(rctx->iv, req->iv, GCM_AES_IV_SIZE);
 352         return omap_aes_gcm_crypt(req, FLAGS_GCM);
 353 }
 354
 355 int omap_aes_4106gcm_encrypt(struct aead_request *req)
 356 {
 357         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 358         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 359
 360         memcpy(rctx->iv, ctx->nonce, 4);
 361         memcpy(rctx->iv + 4, req->iv, 8);
 362         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
 363                                   FLAGS_RFC4106_GCM);
 364 }
 365
 366 int omap_aes_4106gcm_decrypt(struct aead_request *req)
 367 {
 368         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 369         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 370
 371         memcpy(rctx->iv, ctx->nonce, 4);
 372         memcpy(rctx->iv + 4, req->iv, 8);
 373         return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
 374 }
 375
 376 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 377                         unsigned int keylen)
 378 {
 379         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 380
 381         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 382             keylen != AES_KEYSIZE_256)
 383                 return -EINVAL;
 384
 385         memcpy(ctx->key, key, keylen);
 386         ctx->keylen = keylen;
 387
 388         return 0;
 389 }
 390
 391 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 392                             unsigned int keylen)
 393 {
 394         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 395
 396         if (keylen < 4)
 397                 return -EINVAL;
 398
 399         keylen -= 4;
 400         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 401             keylen != AES_KEYSIZE_256)
 402                 return -EINVAL;
 403
 404         memcpy(ctx->key, key, keylen);
 405         memcpy(ctx->nonce, key + keylen, 4);
 406         ctx->keylen = keylen;
 407
 408         return 0;
 409 }