drivers/crypto/hisilicon/sec/sec_algs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2016-2017 Hisilicon Limited. */
   3 #include <linux/crypto.h>
   4 #include <linux/dma-mapping.h>
   5 #include <linux/dmapool.h>
   6 #include <linux/module.h>
   7 #include <linux/mutex.h>
   8 #include <linux/slab.h>
   9
  10 #include <crypto/aes.h>
  11 #include <crypto/algapi.h>
  12 #include <crypto/internal/des.h>
  13 #include <crypto/skcipher.h>
  14 #include <crypto/xts.h>
  15 #include <crypto/internal/skcipher.h>
  16
  17 #include "sec_drv.h"
  18
  19 #define SEC_MAX_CIPHER_KEY              64
  20 #define SEC_REQ_LIMIT SZ_32M
  21
  22 struct sec_c_alg_cfg {
  23         unsigned c_alg          : 3;
  24         unsigned c_mode         : 3;
  25         unsigned key_len        : 2;
  26         unsigned c_width        : 2;
  27 };
  28
  29 static const struct sec_c_alg_cfg sec_c_alg_cfgs[] =  {
  30         [SEC_C_DES_ECB_64] = {
  31                 .c_alg = SEC_C_ALG_DES,
  32                 .c_mode = SEC_C_MODE_ECB,
  33                 .key_len = SEC_KEY_LEN_DES,
  34         },
  35         [SEC_C_DES_CBC_64] = {
  36                 .c_alg = SEC_C_ALG_DES,
  37                 .c_mode = SEC_C_MODE_CBC,
  38                 .key_len = SEC_KEY_LEN_DES,
  39         },
  40         [SEC_C_3DES_ECB_192_3KEY] = {
  41                 .c_alg = SEC_C_ALG_3DES,
  42                 .c_mode = SEC_C_MODE_ECB,
  43                 .key_len = SEC_KEY_LEN_3DES_3_KEY,
  44         },
  45         [SEC_C_3DES_ECB_192_2KEY] = {
  46                 .c_alg = SEC_C_ALG_3DES,
  47                 .c_mode = SEC_C_MODE_ECB,
  48                 .key_len = SEC_KEY_LEN_3DES_2_KEY,
  49         },
  50         [SEC_C_3DES_CBC_192_3KEY] = {
  51                 .c_alg = SEC_C_ALG_3DES,
  52                 .c_mode = SEC_C_MODE_CBC,
  53                 .key_len = SEC_KEY_LEN_3DES_3_KEY,
  54         },
  55         [SEC_C_3DES_CBC_192_2KEY] = {
  56                 .c_alg = SEC_C_ALG_3DES,
  57                 .c_mode = SEC_C_MODE_CBC,
  58                 .key_len = SEC_KEY_LEN_3DES_2_KEY,
  59         },
  60         [SEC_C_AES_ECB_128] = {
  61                 .c_alg = SEC_C_ALG_AES,
  62                 .c_mode = SEC_C_MODE_ECB,
  63                 .key_len = SEC_KEY_LEN_AES_128,
  64         },
  65         [SEC_C_AES_ECB_192] = {
  66                 .c_alg = SEC_C_ALG_AES,
  67                 .c_mode = SEC_C_MODE_ECB,
  68                 .key_len = SEC_KEY_LEN_AES_192,
  69         },
  70         [SEC_C_AES_ECB_256] = {
  71                 .c_alg = SEC_C_ALG_AES,
  72                 .c_mode = SEC_C_MODE_ECB,
  73                 .key_len = SEC_KEY_LEN_AES_256,
  74         },
  75         [SEC_C_AES_CBC_128] = {
  76                 .c_alg = SEC_C_ALG_AES,
  77                 .c_mode = SEC_C_MODE_CBC,
  78                 .key_len = SEC_KEY_LEN_AES_128,
  79         },
  80         [SEC_C_AES_CBC_192] = {
  81                 .c_alg = SEC_C_ALG_AES,
  82                 .c_mode = SEC_C_MODE_CBC,
  83                 .key_len = SEC_KEY_LEN_AES_192,
  84         },
  85         [SEC_C_AES_CBC_256] = {
  86                 .c_alg = SEC_C_ALG_AES,
  87                 .c_mode = SEC_C_MODE_CBC,
  88                 .key_len = SEC_KEY_LEN_AES_256,
  89         },
  90         [SEC_C_AES_CTR_128] = {
  91                 .c_alg = SEC_C_ALG_AES,
  92                 .c_mode = SEC_C_MODE_CTR,
  93                 .key_len = SEC_KEY_LEN_AES_128,
  94         },
  95         [SEC_C_AES_CTR_192] = {
  96                 .c_alg = SEC_C_ALG_AES,
  97                 .c_mode = SEC_C_MODE_CTR,
  98                 .key_len = SEC_KEY_LEN_AES_192,
  99         },
 100         [SEC_C_AES_CTR_256] = {
 101                 .c_alg = SEC_C_ALG_AES,
 102                 .c_mode = SEC_C_MODE_CTR,
 103                 .key_len = SEC_KEY_LEN_AES_256,
 104         },
 105         [SEC_C_AES_XTS_128] = {
 106                 .c_alg = SEC_C_ALG_AES,
 107                 .c_mode = SEC_C_MODE_XTS,
 108                 .key_len = SEC_KEY_LEN_AES_128,
 109         },
 110         [SEC_C_AES_XTS_256] = {
 111                 .c_alg = SEC_C_ALG_AES,
 112                 .c_mode = SEC_C_MODE_XTS,
 113                 .key_len = SEC_KEY_LEN_AES_256,
 114         },
 115         [SEC_C_NULL] = {
 116         },
 117 };
 118
 119 /*
 120  * Mutex used to ensure safe operation of reference count of
 121  * alg providers
 122  */
 123 static DEFINE_MUTEX(algs_lock);
 124 static unsigned int active_devs;
 125
 126 static void sec_alg_skcipher_init_template(struct sec_alg_tfm_ctx *ctx,
 127                                            struct sec_bd_info *req,
 128                                            enum sec_cipher_alg alg)
 129 {
 130         const struct sec_c_alg_cfg *cfg = &sec_c_alg_cfgs[alg];
 131
 132         memset(req, 0, sizeof(*req));
 133         req->w0 |= cfg->c_mode << SEC_BD_W0_C_MODE_S;
 134         req->w1 |= cfg->c_alg << SEC_BD_W1_C_ALG_S;
 135         req->w3 |= cfg->key_len << SEC_BD_W3_C_KEY_LEN_S;
 136         req->w0 |= cfg->c_width << SEC_BD_W0_C_WIDTH_S;
 137
 138         req->cipher_key_addr_lo = lower_32_bits(ctx->pkey);
 139         req->cipher_key_addr_hi = upper_32_bits(ctx->pkey);
 140 }
 141
 142 static void sec_alg_skcipher_init_context(struct crypto_skcipher *atfm,
 143                                           const u8 *key,
 144                                           unsigned int keylen,
 145                                           enum sec_cipher_alg alg)
 146 {
 147         struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
 148         struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 149
 150         ctx->cipher_alg = alg;
 151         memcpy(ctx->key, key, keylen);
 152         sec_alg_skcipher_init_template(ctx, &ctx->req_template,
 153                                        ctx->cipher_alg);
 154 }
 155
 156 static void sec_free_hw_sgl(struct sec_hw_sgl *hw_sgl,
 157                             dma_addr_t psec_sgl, struct sec_dev_info *info)
 158 {
 159         struct sec_hw_sgl *sgl_current, *sgl_next;
 160         dma_addr_t sgl_next_dma;
 161
 162         sgl_current = hw_sgl;
 163         while (sgl_current) {
 164                 sgl_next = sgl_current->next;
 165                 sgl_next_dma = sgl_current->next_sgl;
 166
 167                 dma_pool_free(info->hw_sgl_pool, sgl_current, psec_sgl);
 168
 169                 sgl_current = sgl_next;
 170                 psec_sgl = sgl_next_dma;
 171         }
 172 }
 173
 174 static int sec_alloc_and_fill_hw_sgl(struct sec_hw_sgl **sec_sgl,
 175                                      dma_addr_t *psec_sgl,
 176                                      struct scatterlist *sgl,
 177                                      int count,
 178                                      struct sec_dev_info *info)
 179 {
 180         struct sec_hw_sgl *sgl_current = NULL;
 181         struct sec_hw_sgl *sgl_next;
 182         dma_addr_t sgl_next_dma;
 183         struct scatterlist *sg;
 184         int ret, sge_index, i;
 185
 186         if (!count)
 187                 return -EINVAL;
 188
 189         for_each_sg(sgl, sg, count, i) {
 190                 sge_index = i % SEC_MAX_SGE_NUM;
 191                 if (sge_index == 0) {
 192                         sgl_next = dma_pool_zalloc(info->hw_sgl_pool,
 193                                                    GFP_KERNEL, &sgl_next_dma);
 194                         if (!sgl_next) {
 195                                 ret = -ENOMEM;
 196                                 goto err_free_hw_sgls;
 197                         }
 198
 199                         if (!sgl_current) { /* First one */
 200                                 *psec_sgl = sgl_next_dma;
 201                                 *sec_sgl = sgl_next;
 202                         } else { /* Chained */
 203                                 sgl_current->entry_sum_in_sgl = SEC_MAX_SGE_NUM;
 204                                 sgl_current->next_sgl = sgl_next_dma;
 205                                 sgl_current->next = sgl_next;
 206                         }
 207                         sgl_current = sgl_next;
 208                 }
 209                 sgl_current->sge_entries[sge_index].buf = sg_dma_address(sg);
 210                 sgl_current->sge_entries[sge_index].len = sg_dma_len(sg);
 211                 sgl_current->data_bytes_in_sgl += sg_dma_len(sg);
 212         }
 213         sgl_current->entry_sum_in_sgl = count % SEC_MAX_SGE_NUM;
 214         sgl_current->next_sgl = 0;
 215         (*sec_sgl)->entry_sum_in_chain = count;
 216
 217         return 0;
 218
 219 err_free_hw_sgls:
 220         sec_free_hw_sgl(*sec_sgl, *psec_sgl, info);
 221         *psec_sgl = 0;
 222
 223         return ret;
 224 }
 225
 226 static int sec_alg_skcipher_setkey(struct crypto_skcipher *tfm,
 227                                    const u8 *key, unsigned int keylen,
 228                                    enum sec_cipher_alg alg)
 229 {
 230         struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 231         struct device *dev = ctx->queue->dev_info->dev;
 232
 233         mutex_lock(&ctx->lock);
 234         if (ctx->key) {
 235                 /* rekeying */
 236                 memset(ctx->key, 0, SEC_MAX_CIPHER_KEY);
 237         } else {
 238                 /* new key */
 239                 ctx->key = dma_alloc_coherent(dev, SEC_MAX_CIPHER_KEY,
 240                                               &ctx->pkey, GFP_KERNEL);
 241                 if (!ctx->key) {
 242                         mutex_unlock(&ctx->lock);
 243                         return -ENOMEM;
 244                 }
 245         }
 246         mutex_unlock(&ctx->lock);
 247         sec_alg_skcipher_init_context(tfm, key, keylen, alg);
 248
 249         return 0;
 250 }
 251
 252 static int sec_alg_skcipher_setkey_aes_ecb(struct crypto_skcipher *tfm,
 253                                            const u8 *key, unsigned int keylen)
 254 {
 255         enum sec_cipher_alg alg;
 256
 257         switch (keylen) {
 258         case AES_KEYSIZE_128:
 259                 alg = SEC_C_AES_ECB_128;
 260                 break;
 261         case AES_KEYSIZE_192:
 262                 alg = SEC_C_AES_ECB_192;
 263                 break;
 264         case AES_KEYSIZE_256:
 265                 alg = SEC_C_AES_ECB_256;
 266                 break;
 267         default:
 268                 return -EINVAL;
 269         }
 270
 271         return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 272 }
 273
 274 static int sec_alg_skcipher_setkey_aes_cbc(struct crypto_skcipher *tfm,
 275                                            const u8 *key, unsigned int keylen)
 276 {
 277         enum sec_cipher_alg alg;
 278
 279         switch (keylen) {
 280         case AES_KEYSIZE_128:
 281                 alg = SEC_C_AES_CBC_128;
 282                 break;
 283         case AES_KEYSIZE_192:
 284                 alg = SEC_C_AES_CBC_192;
 285                 break;
 286         case AES_KEYSIZE_256:
 287                 alg = SEC_C_AES_CBC_256;
 288                 break;
 289         default:
 290                 return -EINVAL;
 291         }
 292
 293         return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 294 }
 295
 296 static int sec_alg_skcipher_setkey_aes_ctr(struct crypto_skcipher *tfm,
 297                                            const u8 *key, unsigned int keylen)
 298 {
 299         enum sec_cipher_alg alg;
 300
 301         switch (keylen) {
 302         case AES_KEYSIZE_128:
 303                 alg = SEC_C_AES_CTR_128;
 304                 break;
 305         case AES_KEYSIZE_192:
 306                 alg = SEC_C_AES_CTR_192;
 307                 break;
 308         case AES_KEYSIZE_256:
 309                 alg = SEC_C_AES_CTR_256;
 310                 break;
 311         default:
 312                 return -EINVAL;
 313         }
 314
 315         return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 316 }
 317
 318 static int sec_alg_skcipher_setkey_aes_xts(struct crypto_skcipher *tfm,
 319                                            const u8 *key, unsigned int keylen)
 320 {
 321         enum sec_cipher_alg alg;
 322         int ret;
 323
 324         ret = xts_verify_key(tfm, key, keylen);
 325         if (ret)
 326                 return ret;
 327
 328         switch (keylen) {
 329         case AES_KEYSIZE_128 * 2:
 330                 alg = SEC_C_AES_XTS_128;
 331                 break;
 332         case AES_KEYSIZE_256 * 2:
 333                 alg = SEC_C_AES_XTS_256;
 334                 break;
 335         default:
 336                 return -EINVAL;
 337         }
 338
 339         return sec_alg_skcipher_setkey(tfm, key, keylen, alg);
 340 }
 341
 342 static int sec_alg_skcipher_setkey_des_ecb(struct crypto_skcipher *tfm,
 343                                            const u8 *key, unsigned int keylen)
 344 {
 345         return verify_skcipher_des_key(tfm, key) ?:
 346                sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_ECB_64);
 347 }
 348
 349 static int sec_alg_skcipher_setkey_des_cbc(struct crypto_skcipher *tfm,
 350                                            const u8 *key, unsigned int keylen)
 351 {
 352         return verify_skcipher_des_key(tfm, key) ?:
 353                sec_alg_skcipher_setkey(tfm, key, keylen, SEC_C_DES_CBC_64);
 354 }
 355
 356 static int sec_alg_skcipher_setkey_3des_ecb(struct crypto_skcipher *tfm,
 357                                             const u8 *key, unsigned int keylen)
 358 {
 359         return verify_skcipher_des3_key(tfm, key) ?:
 360                sec_alg_skcipher_setkey(tfm, key, keylen,
 361                                        SEC_C_3DES_ECB_192_3KEY);
 362 }
 363
 364 static int sec_alg_skcipher_setkey_3des_cbc(struct crypto_skcipher *tfm,
 365                                             const u8 *key, unsigned int keylen)
 366 {
 367         return verify_skcipher_des3_key(tfm, key) ?:
 368                sec_alg_skcipher_setkey(tfm, key, keylen,
 369                                        SEC_C_3DES_CBC_192_3KEY);
 370 }
 371
 372 static void sec_alg_free_el(struct sec_request_el *el,
 373                             struct sec_dev_info *info)
 374 {
 375         sec_free_hw_sgl(el->out, el->dma_out, info);
 376         sec_free_hw_sgl(el->in, el->dma_in, info);
 377         kfree(el->sgl_in);
 378         kfree(el->sgl_out);
 379         kfree(el);
 380 }
 381
 382 /* queuelock must be held */
 383 static int sec_send_request(struct sec_request *sec_req, struct sec_queue *queue)
 384 {
 385         struct sec_request_el *el, *temp;
 386         int ret = 0;
 387
 388         mutex_lock(&sec_req->lock);
 389         list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
 390                 /*
 391                  * Add to hardware queue only under following circumstances
 392                  * 1) Software and hardware queue empty so no chain dependencies
 393                  * 2) No dependencies as new IV - (check software queue empty
 394                  *    to maintain order)
 395                  * 3) No dependencies because the mode does no chaining.
 396                  *
 397                  * In other cases first insert onto the software queue which
 398                  * is then emptied as requests complete
 399                  */
 400                 if (!queue->havesoftqueue ||
 401                     (kfifo_is_empty(&queue->softqueue) &&
 402                      sec_queue_empty(queue))) {
 403                         ret = sec_queue_send(queue, &el->req, sec_req);
 404                         if (ret == -EAGAIN) {
 405                                 /* Wait unti we can send then try again */
 406                                 /* DEAD if here - should not happen */
 407                                 ret = -EBUSY;
 408                                 goto err_unlock;
 409                         }
 410                 } else {
 411                         kfifo_put(&queue->softqueue, el);
 412                 }
 413         }
 414 err_unlock:
 415         mutex_unlock(&sec_req->lock);
 416
 417         return ret;
 418 }
 419
 420 static void sec_skcipher_alg_callback(struct sec_bd_info *sec_resp,
 421                                       struct crypto_async_request *req_base)
 422 {
 423         struct skcipher_request *skreq = container_of(req_base,
 424                                                       struct skcipher_request,
 425                                                       base);
 426         struct sec_request *sec_req = skcipher_request_ctx(skreq);
 427         struct sec_request *backlog_req;
 428         struct sec_request_el *sec_req_el, *nextrequest;
 429         struct sec_alg_tfm_ctx *ctx = sec_req->tfm_ctx;
 430         struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
 431         struct device *dev = ctx->queue->dev_info->dev;
 432         int icv_or_skey_en, ret;
 433         bool done;
 434
 435         sec_req_el = list_first_entry(&sec_req->elements, struct sec_request_el,
 436                                       head);
 437         icv_or_skey_en = (sec_resp->w0 & SEC_BD_W0_ICV_OR_SKEY_EN_M) >>
 438                 SEC_BD_W0_ICV_OR_SKEY_EN_S;
 439         if (sec_resp->w1 & SEC_BD_W1_BD_INVALID || icv_or_skey_en == 3) {
 440                 dev_err(dev, "Got an invalid answer %lu %d\n",
 441                         sec_resp->w1 & SEC_BD_W1_BD_INVALID,
 442                         icv_or_skey_en);
 443                 sec_req->err = -EINVAL;
 444                 /*
 445                  * We need to muddle on to avoid getting stuck with elements
 446                  * on the queue. Error will be reported so requester so
 447                  * it should be able to handle appropriately.
 448                  */
 449         }
 450
 451         mutex_lock(&ctx->queue->queuelock);
 452         /* Put the IV in place for chained cases */
 453         switch (ctx->cipher_alg) {
 454         case SEC_C_AES_CBC_128:
 455         case SEC_C_AES_CBC_192:
 456         case SEC_C_AES_CBC_256:
 457                 if (sec_req_el->req.w0 & SEC_BD_W0_DE)
 458                         sg_pcopy_to_buffer(sec_req_el->sgl_out,
 459                                            sg_nents(sec_req_el->sgl_out),
 460                                            skreq->iv,
 461                                            crypto_skcipher_ivsize(atfm),
 462                                            sec_req_el->el_length -
 463                                            crypto_skcipher_ivsize(atfm));
 464                 else
 465                         sg_pcopy_to_buffer(sec_req_el->sgl_in,
 466                                            sg_nents(sec_req_el->sgl_in),
 467                                            skreq->iv,
 468                                            crypto_skcipher_ivsize(atfm),
 469                                            sec_req_el->el_length -
 470                                            crypto_skcipher_ivsize(atfm));
 471                 /* No need to sync to the device as coherent DMA */
 472                 break;
 473         case SEC_C_AES_CTR_128:
 474         case SEC_C_AES_CTR_192:
 475         case SEC_C_AES_CTR_256:
 476                 crypto_inc(skreq->iv, 16);
 477                 break;
 478         default:
 479                 /* Do not update */
 480                 break;
 481         }
 482
 483         if (ctx->queue->havesoftqueue &&
 484             !kfifo_is_empty(&ctx->queue->softqueue) &&
 485             sec_queue_empty(ctx->queue)) {
 486                 ret = kfifo_get(&ctx->queue->softqueue, &nextrequest);
 487                 if (ret <= 0)
 488                         dev_err(dev,
 489                                 "Error getting next element from kfifo %d\n",
 490                                 ret);
 491                 else
 492                         /* We know there is space so this cannot fail */
 493                         sec_queue_send(ctx->queue, &nextrequest->req,
 494                                        nextrequest->sec_req);
 495         } else if (!list_empty(&ctx->backlog)) {
 496                 /* Need to verify there is room first */
 497                 backlog_req = list_first_entry(&ctx->backlog,
 498                                                typeof(*backlog_req),
 499                                                backlog_head);
 500                 if (sec_queue_can_enqueue(ctx->queue,
 501                     backlog_req->num_elements) ||
 502                     (ctx->queue->havesoftqueue &&
 503                      kfifo_avail(&ctx->queue->softqueue) >
 504                      backlog_req->num_elements)) {
 505                         sec_send_request(backlog_req, ctx->queue);
 506                         backlog_req->req_base->complete(backlog_req->req_base,
 507                                                         -EINPROGRESS);
 508                         list_del(&backlog_req->backlog_head);
 509                 }
 510         }
 511         mutex_unlock(&ctx->queue->queuelock);
 512
 513         mutex_lock(&sec_req->lock);
 514         list_del(&sec_req_el->head);
 515         mutex_unlock(&sec_req->lock);
 516         sec_alg_free_el(sec_req_el, ctx->queue->dev_info);
 517
 518         /*
 519          * Request is done.
 520          * The dance is needed as the lock is freed in the completion
 521          */
 522         mutex_lock(&sec_req->lock);
 523         done = list_empty(&sec_req->elements);
 524         mutex_unlock(&sec_req->lock);
 525         if (done) {
 526                 if (crypto_skcipher_ivsize(atfm)) {
 527                         dma_unmap_single(dev, sec_req->dma_iv,
 528                                          crypto_skcipher_ivsize(atfm),
 529                                          DMA_TO_DEVICE);
 530                 }
 531                 dma_unmap_sg(dev, skreq->src, sec_req->len_in,
 532                              DMA_BIDIRECTIONAL);
 533                 if (skreq->src != skreq->dst)
 534                         dma_unmap_sg(dev, skreq->dst, sec_req->len_out,
 535                                      DMA_BIDIRECTIONAL);
 536                 skreq->base.complete(&skreq->base, sec_req->err);
 537         }
 538 }
 539
 540 void sec_alg_callback(struct sec_bd_info *resp, void *shadow)
 541 {
 542         struct sec_request *sec_req = shadow;
 543
 544         sec_req->cb(resp, sec_req->req_base);
 545 }
 546
 547 static int sec_alg_alloc_and_calc_split_sizes(int length, size_t **split_sizes,
 548                                               int *steps)
 549 {
 550         size_t *sizes;
 551         int i;
 552
 553         /* Split into suitable sized blocks */
 554         *steps = roundup(length, SEC_REQ_LIMIT) / SEC_REQ_LIMIT;
 555         sizes = kcalloc(*steps, sizeof(*sizes), GFP_KERNEL);
 556         if (!sizes)
 557                 return -ENOMEM;
 558
 559         for (i = 0; i < *steps - 1; i++)
 560                 sizes[i] = SEC_REQ_LIMIT;
 561         sizes[*steps - 1] = length - SEC_REQ_LIMIT * (*steps - 1);
 562         *split_sizes = sizes;
 563
 564         return 0;
 565 }
 566
 567 static int sec_map_and_split_sg(struct scatterlist *sgl, size_t *split_sizes,
 568                                 int steps, struct scatterlist ***splits,
 569                                 int **splits_nents,
 570                                 int sgl_len_in,
 571                                 struct device *dev)
 572 {
 573         int ret, count;
 574
 575         count = dma_map_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
 576         if (!count)
 577                 return -EINVAL;
 578
 579         *splits = kcalloc(steps, sizeof(struct scatterlist *), GFP_KERNEL);
 580         if (!*splits) {
 581                 ret = -ENOMEM;
 582                 goto err_unmap_sg;
 583         }
 584         *splits_nents = kcalloc(steps, sizeof(int), GFP_KERNEL);
 585         if (!*splits_nents) {
 586                 ret = -ENOMEM;
 587                 goto err_free_splits;
 588         }
 589
 590         /* output the scatter list before and after this */
 591         ret = sg_split(sgl, count, 0, steps, split_sizes,
 592                        *splits, *splits_nents, GFP_KERNEL);
 593         if (ret) {
 594                 ret = -ENOMEM;
 595                 goto err_free_splits_nents;
 596         }
 597
 598         return 0;
 599
 600 err_free_splits_nents:
 601         kfree(*splits_nents);
 602 err_free_splits:
 603         kfree(*splits);
 604 err_unmap_sg:
 605         dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
 606
 607         return ret;
 608 }
 609
 610 /*
 611  * Reverses the sec_map_and_split_sg call for messages not yet added to
 612  * the queues.
 613  */
 614 static void sec_unmap_sg_on_err(struct scatterlist *sgl, int steps,
 615                                 struct scatterlist **splits, int *splits_nents,
 616                                 int sgl_len_in, struct device *dev)
 617 {
 618         int i;
 619
 620         for (i = 0; i < steps; i++)
 621                 kfree(splits[i]);
 622         kfree(splits_nents);
 623         kfree(splits);
 624
 625         dma_unmap_sg(dev, sgl, sgl_len_in, DMA_BIDIRECTIONAL);
 626 }
 627
 628 static struct sec_request_el
 629 *sec_alg_alloc_and_fill_el(struct sec_bd_info *template, int encrypt,
 630                            int el_size, bool different_dest,
 631                            struct scatterlist *sgl_in, int n_ents_in,
 632                            struct scatterlist *sgl_out, int n_ents_out,
 633                            struct sec_dev_info *info)
 634 {
 635         struct sec_request_el *el;
 636         struct sec_bd_info *req;
 637         int ret;
 638
 639         el = kzalloc(sizeof(*el), GFP_KERNEL);
 640         if (!el)
 641                 return ERR_PTR(-ENOMEM);
 642         el->el_length = el_size;
 643         req = &el->req;
 644         memcpy(req, template, sizeof(*req));
 645
 646         req->w0 &= ~SEC_BD_W0_CIPHER_M;
 647         if (encrypt)
 648                 req->w0 |= SEC_CIPHER_ENCRYPT << SEC_BD_W0_CIPHER_S;
 649         else
 650                 req->w0 |= SEC_CIPHER_DECRYPT << SEC_BD_W0_CIPHER_S;
 651
 652         req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_19_16_M;
 653         req->w0 |= ((el_size >> 16) << SEC_BD_W0_C_GRAN_SIZE_19_16_S) &
 654                 SEC_BD_W0_C_GRAN_SIZE_19_16_M;
 655
 656         req->w0 &= ~SEC_BD_W0_C_GRAN_SIZE_21_20_M;
 657         req->w0 |= ((el_size >> 20) << SEC_BD_W0_C_GRAN_SIZE_21_20_S) &
 658                 SEC_BD_W0_C_GRAN_SIZE_21_20_M;
 659
 660         /* Writing whole u32 so no need to take care of masking */
 661         req->w2 = ((1 << SEC_BD_W2_GRAN_NUM_S) & SEC_BD_W2_GRAN_NUM_M) |
 662                 ((el_size << SEC_BD_W2_C_GRAN_SIZE_15_0_S) &
 663                  SEC_BD_W2_C_GRAN_SIZE_15_0_M);
 664
 665         req->w3 &= ~SEC_BD_W3_CIPHER_LEN_OFFSET_M;
 666         req->w1 |= SEC_BD_W1_ADDR_TYPE;
 667
 668         el->sgl_in = sgl_in;
 669
 670         ret = sec_alloc_and_fill_hw_sgl(&el->in, &el->dma_in, el->sgl_in,
 671                                         n_ents_in, info);
 672         if (ret)
 673                 goto err_free_el;
 674
 675         req->data_addr_lo = lower_32_bits(el->dma_in);
 676         req->data_addr_hi = upper_32_bits(el->dma_in);
 677
 678         if (different_dest) {
 679                 el->sgl_out = sgl_out;
 680                 ret = sec_alloc_and_fill_hw_sgl(&el->out, &el->dma_out,
 681                                                 el->sgl_out,
 682                                                 n_ents_out, info);
 683                 if (ret)
 684                         goto err_free_hw_sgl_in;
 685
 686                 req->w0 |= SEC_BD_W0_DE;
 687                 req->cipher_destin_addr_lo = lower_32_bits(el->dma_out);
 688                 req->cipher_destin_addr_hi = upper_32_bits(el->dma_out);
 689
 690         } else {
 691                 req->w0 &= ~SEC_BD_W0_DE;
 692                 req->cipher_destin_addr_lo = lower_32_bits(el->dma_in);
 693                 req->cipher_destin_addr_hi = upper_32_bits(el->dma_in);
 694         }
 695
 696         return el;
 697
 698 err_free_hw_sgl_in:
 699         sec_free_hw_sgl(el->in, el->dma_in, info);
 700 err_free_el:
 701         kfree(el);
 702
 703         return ERR_PTR(ret);
 704 }
 705
 706 static int sec_alg_skcipher_crypto(struct skcipher_request *skreq,
 707                                    bool encrypt)
 708 {
 709         struct crypto_skcipher *atfm = crypto_skcipher_reqtfm(skreq);
 710         struct crypto_tfm *tfm = crypto_skcipher_tfm(atfm);
 711         struct sec_alg_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
 712         struct sec_queue *queue = ctx->queue;
 713         struct sec_request *sec_req = skcipher_request_ctx(skreq);
 714         struct sec_dev_info *info = queue->dev_info;
 715         int i, ret, steps;
 716         size_t *split_sizes;
 717         struct scatterlist **splits_in;
 718         struct scatterlist **splits_out = NULL;
 719         int *splits_in_nents;
 720         int *splits_out_nents = NULL;
 721         struct sec_request_el *el, *temp;
 722         bool split = skreq->src != skreq->dst;
 723
 724         mutex_init(&sec_req->lock);
 725         sec_req->req_base = &skreq->base;
 726         sec_req->err = 0;
 727         /* SGL mapping out here to allow us to break it up as necessary */
 728         sec_req->len_in = sg_nents(skreq->src);
 729
 730         ret = sec_alg_alloc_and_calc_split_sizes(skreq->cryptlen, &split_sizes,
 731                                                  &steps);
 732         if (ret)
 733                 return ret;
 734         sec_req->num_elements = steps;
 735         ret = sec_map_and_split_sg(skreq->src, split_sizes, steps, &splits_in,
 736                                    &splits_in_nents, sec_req->len_in,
 737                                    info->dev);
 738         if (ret)
 739                 goto err_free_split_sizes;
 740
 741         if (split) {
 742                 sec_req->len_out = sg_nents(skreq->dst);
 743                 ret = sec_map_and_split_sg(skreq->dst, split_sizes, steps,
 744                                            &splits_out, &splits_out_nents,
 745                                            sec_req->len_out, info->dev);
 746                 if (ret)
 747                         goto err_unmap_in_sg;
 748         }
 749         /* Shared info stored in seq_req - applies to all BDs */
 750         sec_req->tfm_ctx = ctx;
 751         sec_req->cb = sec_skcipher_alg_callback;
 752         INIT_LIST_HEAD(&sec_req->elements);
 753
 754         /*
 755          * Future optimization.
 756          * In the chaining case we can't use a dma pool bounce buffer
 757          * but in the case where we know there is no chaining we can
 758          */
 759         if (crypto_skcipher_ivsize(atfm)) {
 760                 sec_req->dma_iv = dma_map_single(info->dev, skreq->iv,
 761                                                  crypto_skcipher_ivsize(atfm),
 762                                                  DMA_TO_DEVICE);
 763                 if (dma_mapping_error(info->dev, sec_req->dma_iv)) {
 764                         ret = -ENOMEM;
 765                         goto err_unmap_out_sg;
 766                 }
 767         }
 768
 769         /* Set them all up then queue - cleaner error handling. */
 770         for (i = 0; i < steps; i++) {
 771                 el = sec_alg_alloc_and_fill_el(&ctx->req_template,
 772                                                encrypt ? 1 : 0,
 773                                                split_sizes[i],
 774                                                skreq->src != skreq->dst,
 775                                                splits_in[i], splits_in_nents[i],
 776                                                split ? splits_out[i] : NULL,
 777                                                split ? splits_out_nents[i] : 0,
 778                                                info);
 779                 if (IS_ERR(el)) {
 780                         ret = PTR_ERR(el);
 781                         goto err_free_elements;
 782                 }
 783                 el->req.cipher_iv_addr_lo = lower_32_bits(sec_req->dma_iv);
 784                 el->req.cipher_iv_addr_hi = upper_32_bits(sec_req->dma_iv);
 785                 el->sec_req = sec_req;
 786                 list_add_tail(&el->head, &sec_req->elements);
 787         }
 788
 789         /*
 790          * Only attempt to queue if the whole lot can fit in the queue -
 791          * we can't successfully cleanup after a partial queing so this
 792          * must succeed or fail atomically.
 793          *
 794          * Big hammer test of both software and hardware queues - could be
 795          * more refined but this is unlikely to happen so no need.
 796          */
 797
 798         /* Grab a big lock for a long time to avoid concurrency issues */
 799         mutex_lock(&queue->queuelock);
 800
 801         /*
 802          * Can go on to queue if we have space in either:
 803          * 1) The hardware queue and no software queue
 804          * 2) The software queue
 805          * AND there is nothing in the backlog.  If there is backlog we
 806          * have to only queue to the backlog queue and return busy.
 807          */
 808         if ((!sec_queue_can_enqueue(queue, steps) &&
 809              (!queue->havesoftqueue ||
 810               kfifo_avail(&queue->softqueue) > steps)) ||
 811             !list_empty(&ctx->backlog)) {
 812                 ret = -EBUSY;
 813                 if ((skreq->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
 814                         list_add_tail(&sec_req->backlog_head, &ctx->backlog);
 815                         mutex_unlock(&queue->queuelock);
 816                         goto out;
 817                 }
 818
 819                 mutex_unlock(&queue->queuelock);
 820                 goto err_free_elements;
 821         }
 822         ret = sec_send_request(sec_req, queue);
 823         mutex_unlock(&queue->queuelock);
 824         if (ret)
 825                 goto err_free_elements;
 826
 827         ret = -EINPROGRESS;
 828 out:
 829         /* Cleanup - all elements in pointer arrays have been copied */
 830         kfree(splits_in_nents);
 831         kfree(splits_in);
 832         kfree(splits_out_nents);
 833         kfree(splits_out);
 834         kfree(split_sizes);
 835         return ret;
 836
 837 err_free_elements:
 838         list_for_each_entry_safe(el, temp, &sec_req->elements, head) {
 839                 list_del(&el->head);
 840                 sec_alg_free_el(el, info);
 841         }
 842         if (crypto_skcipher_ivsize(atfm))
 843                 dma_unmap_single(info->dev, sec_req->dma_iv,
 844                                  crypto_skcipher_ivsize(atfm),
 845                                  DMA_BIDIRECTIONAL);
 846 err_unmap_out_sg:
 847         if (split)
 848                 sec_unmap_sg_on_err(skreq->dst, steps, splits_out,
 849                                     splits_out_nents, sec_req->len_out,
 850                                     info->dev);
 851 err_unmap_in_sg:
 852         sec_unmap_sg_on_err(skreq->src, steps, splits_in, splits_in_nents,
 853                             sec_req->len_in, info->dev);
 854 err_free_split_sizes:
 855         kfree(split_sizes);
 856
 857         return ret;
 858 }
 859
 860 static int sec_alg_skcipher_encrypt(struct skcipher_request *req)
 861 {
 862         return sec_alg_skcipher_crypto(req, true);
 863 }
 864
 865 static int sec_alg_skcipher_decrypt(struct skcipher_request *req)
 866 {
 867         return sec_alg_skcipher_crypto(req, false);
 868 }
 869
 870 static int sec_alg_skcipher_init(struct crypto_skcipher *tfm)
 871 {
 872         struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 873
 874         mutex_init(&ctx->lock);
 875         INIT_LIST_HEAD(&ctx->backlog);
 876         crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_request));
 877
 878         ctx->queue = sec_queue_alloc_start_safe();
 879         if (IS_ERR(ctx->queue))
 880                 return PTR_ERR(ctx->queue);
 881
 882         mutex_init(&ctx->queue->queuelock);
 883         ctx->queue->havesoftqueue = false;
 884
 885         return 0;
 886 }
 887
 888 static void sec_alg_skcipher_exit(struct crypto_skcipher *tfm)
 889 {
 890         struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 891         struct device *dev = ctx->queue->dev_info->dev;
 892
 893         if (ctx->key) {
 894                 memzero_explicit(ctx->key, SEC_MAX_CIPHER_KEY);
 895                 dma_free_coherent(dev, SEC_MAX_CIPHER_KEY, ctx->key,
 896                                   ctx->pkey);
 897         }
 898         sec_queue_stop_release(ctx->queue);
 899 }
 900
 901 static int sec_alg_skcipher_init_with_queue(struct crypto_skcipher *tfm)
 902 {
 903         struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 904         int ret;
 905
 906         ret = sec_alg_skcipher_init(tfm);
 907         if (ret)
 908                 return ret;
 909
 910         INIT_KFIFO(ctx->queue->softqueue);
 911         ret = kfifo_alloc(&ctx->queue->softqueue, 512, GFP_KERNEL);
 912         if (ret) {
 913                 sec_alg_skcipher_exit(tfm);
 914                 return ret;
 915         }
 916         ctx->queue->havesoftqueue = true;
 917
 918         return 0;
 919 }
 920
 921 static void sec_alg_skcipher_exit_with_queue(struct crypto_skcipher *tfm)
 922 {
 923         struct sec_alg_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
 924
 925         kfifo_free(&ctx->queue->softqueue);
 926         sec_alg_skcipher_exit(tfm);
 927 }
 928
 929 static struct skcipher_alg sec_algs[] = {
 930         {
 931                 .base = {
 932                         .cra_name = "ecb(aes)",
 933                         .cra_driver_name = "hisi_sec_aes_ecb",
 934                         .cra_priority = 4001,
 935                         .cra_flags = CRYPTO_ALG_ASYNC,
 936                         .cra_blocksize = AES_BLOCK_SIZE,
 937                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 938                         .cra_alignmask = 0,
 939                         .cra_module = THIS_MODULE,
 940                 },
 941                 .init = sec_alg_skcipher_init,
 942                 .exit = sec_alg_skcipher_exit,
 943                 .setkey = sec_alg_skcipher_setkey_aes_ecb,
 944                 .decrypt = sec_alg_skcipher_decrypt,
 945                 .encrypt = sec_alg_skcipher_encrypt,
 946                 .min_keysize = AES_MIN_KEY_SIZE,
 947                 .max_keysize = AES_MAX_KEY_SIZE,
 948                 .ivsize = 0,
 949         }, {
 950                 .base = {
 951                         .cra_name = "cbc(aes)",
 952                         .cra_driver_name = "hisi_sec_aes_cbc",
 953                         .cra_priority = 4001,
 954                         .cra_flags = CRYPTO_ALG_ASYNC,
 955                         .cra_blocksize = AES_BLOCK_SIZE,
 956                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 957                         .cra_alignmask = 0,
 958                         .cra_module = THIS_MODULE,
 959                 },
 960                 .init = sec_alg_skcipher_init_with_queue,
 961                 .exit = sec_alg_skcipher_exit_with_queue,
 962                 .setkey = sec_alg_skcipher_setkey_aes_cbc,
 963                 .decrypt = sec_alg_skcipher_decrypt,
 964                 .encrypt = sec_alg_skcipher_encrypt,
 965                 .min_keysize = AES_MIN_KEY_SIZE,
 966                 .max_keysize = AES_MAX_KEY_SIZE,
 967                 .ivsize = AES_BLOCK_SIZE,
 968         }, {
 969                 .base = {
 970                         .cra_name = "ctr(aes)",
 971                         .cra_driver_name = "hisi_sec_aes_ctr",
 972                         .cra_priority = 4001,
 973                         .cra_flags = CRYPTO_ALG_ASYNC,
 974                         .cra_blocksize = AES_BLOCK_SIZE,
 975                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 976                         .cra_alignmask = 0,
 977                         .cra_module = THIS_MODULE,
 978                 },
 979                 .init = sec_alg_skcipher_init_with_queue,
 980                 .exit = sec_alg_skcipher_exit_with_queue,
 981                 .setkey = sec_alg_skcipher_setkey_aes_ctr,
 982                 .decrypt = sec_alg_skcipher_decrypt,
 983                 .encrypt = sec_alg_skcipher_encrypt,
 984                 .min_keysize = AES_MIN_KEY_SIZE,
 985                 .max_keysize = AES_MAX_KEY_SIZE,
 986                 .ivsize = AES_BLOCK_SIZE,
 987         }, {
 988                 .base = {
 989                         .cra_name = "xts(aes)",
 990                         .cra_driver_name = "hisi_sec_aes_xts",
 991                         .cra_priority = 4001,
 992                         .cra_flags = CRYPTO_ALG_ASYNC,
 993                         .cra_blocksize = AES_BLOCK_SIZE,
 994                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
 995                         .cra_alignmask = 0,
 996                         .cra_module = THIS_MODULE,
 997                 },
 998                 .init = sec_alg_skcipher_init,
 999                 .exit = sec_alg_skcipher_exit,
1000                 .setkey = sec_alg_skcipher_setkey_aes_xts,
1001                 .decrypt = sec_alg_skcipher_decrypt,
1002                 .encrypt = sec_alg_skcipher_encrypt,
1003                 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1004                 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1005                 .ivsize = AES_BLOCK_SIZE,
1006         }, {
1007         /* Unable to find any test vectors so untested */
1008                 .base = {
1009                         .cra_name = "ecb(des)",
1010                         .cra_driver_name = "hisi_sec_des_ecb",
1011                         .cra_priority = 4001,
1012                         .cra_flags = CRYPTO_ALG_ASYNC,
1013                         .cra_blocksize = DES_BLOCK_SIZE,
1014                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1015                         .cra_alignmask = 0,
1016                         .cra_module = THIS_MODULE,
1017                 },
1018                 .init = sec_alg_skcipher_init,
1019                 .exit = sec_alg_skcipher_exit,
1020                 .setkey = sec_alg_skcipher_setkey_des_ecb,
1021                 .decrypt = sec_alg_skcipher_decrypt,
1022                 .encrypt = sec_alg_skcipher_encrypt,
1023                 .min_keysize = DES_KEY_SIZE,
1024                 .max_keysize = DES_KEY_SIZE,
1025                 .ivsize = 0,
1026         }, {
1027                 .base = {
1028                         .cra_name = "cbc(des)",
1029                         .cra_driver_name = "hisi_sec_des_cbc",
1030                         .cra_priority = 4001,
1031                         .cra_flags = CRYPTO_ALG_ASYNC,
1032                         .cra_blocksize = DES_BLOCK_SIZE,
1033                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1034                         .cra_alignmask = 0,
1035                         .cra_module = THIS_MODULE,
1036                 },
1037                 .init = sec_alg_skcipher_init_with_queue,
1038                 .exit = sec_alg_skcipher_exit_with_queue,
1039                 .setkey = sec_alg_skcipher_setkey_des_cbc,
1040                 .decrypt = sec_alg_skcipher_decrypt,
1041                 .encrypt = sec_alg_skcipher_encrypt,
1042                 .min_keysize = DES_KEY_SIZE,
1043                 .max_keysize = DES_KEY_SIZE,
1044                 .ivsize = DES_BLOCK_SIZE,
1045         }, {
1046                 .base = {
1047                         .cra_name = "cbc(des3_ede)",
1048                         .cra_driver_name = "hisi_sec_3des_cbc",
1049                         .cra_priority = 4001,
1050                         .cra_flags = CRYPTO_ALG_ASYNC,
1051                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1052                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1053                         .cra_alignmask = 0,
1054                         .cra_module = THIS_MODULE,
1055                 },
1056                 .init = sec_alg_skcipher_init_with_queue,
1057                 .exit = sec_alg_skcipher_exit_with_queue,
1058                 .setkey = sec_alg_skcipher_setkey_3des_cbc,
1059                 .decrypt = sec_alg_skcipher_decrypt,
1060                 .encrypt = sec_alg_skcipher_encrypt,
1061                 .min_keysize = DES3_EDE_KEY_SIZE,
1062                 .max_keysize = DES3_EDE_KEY_SIZE,
1063                 .ivsize = DES3_EDE_BLOCK_SIZE,
1064         }, {
1065                 .base = {
1066                         .cra_name = "ecb(des3_ede)",
1067                         .cra_driver_name = "hisi_sec_3des_ecb",
1068                         .cra_priority = 4001,
1069                         .cra_flags = CRYPTO_ALG_ASYNC,
1070                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1071                         .cra_ctxsize = sizeof(struct sec_alg_tfm_ctx),
1072                         .cra_alignmask = 0,
1073                         .cra_module = THIS_MODULE,
1074                 },
1075                 .init = sec_alg_skcipher_init,
1076                 .exit = sec_alg_skcipher_exit,
1077                 .setkey = sec_alg_skcipher_setkey_3des_ecb,
1078                 .decrypt = sec_alg_skcipher_decrypt,
1079                 .encrypt = sec_alg_skcipher_encrypt,
1080                 .min_keysize = DES3_EDE_KEY_SIZE,
1081                 .max_keysize = DES3_EDE_KEY_SIZE,
1082                 .ivsize = 0,
1083         }
1084 };
1085
1086 int sec_algs_register(void)
1087 {
1088         int ret = 0;
1089
1090         mutex_lock(&algs_lock);
1091         if (++active_devs != 1)
1092                 goto unlock;
1093
1094         ret = crypto_register_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1095         if (ret)
1096                 --active_devs;
1097 unlock:
1098         mutex_unlock(&algs_lock);
1099
1100         return ret;
1101 }
1102
1103 void sec_algs_unregister(void)
1104 {
1105         mutex_lock(&algs_lock);
1106         if (--active_devs != 0)
1107                 goto unlock;
1108         crypto_unregister_skciphers(sec_algs, ARRAY_SIZE(sec_algs));
1109
1110 unlock:
1111         mutex_unlock(&algs_lock);
1112 }