drivers/crypto/allwinner/sun8i-ss/sun8i-ss-cipher.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * sun8i-ss-cipher.c - hardware cryptographic offloader for
   4  * Allwinner A80/A83T SoC
   5  *
   6  * Copyright (C) 2016-2019 Corentin LABBE <clabbe.montjoie@gmail.com>
   7  *
   8  * This file add support for AES cipher with 128,192,256 bits keysize in
   9  * CBC and ECB mode.
  10  *
  11  * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst
  12  */
  13
  14 #include <linux/bottom_half.h>
  15 #include <linux/crypto.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/io.h>
  18 #include <linux/pm_runtime.h>
  19 #include <crypto/scatterwalk.h>
  20 #include <crypto/internal/skcipher.h>
  21 #include "sun8i-ss.h"
  22
  23 static bool sun8i_ss_need_fallback(struct skcipher_request *areq)
  24 {
  25         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
  26         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
  27         struct sun8i_ss_alg_template *algt = container_of(alg, struct sun8i_ss_alg_template, alg.skcipher.base);
  28         struct scatterlist *in_sg = areq->src;
  29         struct scatterlist *out_sg = areq->dst;
  30         struct scatterlist *sg;
  31         unsigned int todo, len;
  32
  33         if (areq->cryptlen == 0 || areq->cryptlen % 16) {
  34                 algt->stat_fb_len++;
  35                 return true;
  36         }
  37
  38         if (sg_nents_for_len(areq->src, areq->cryptlen) > 8 ||
  39                 sg_nents_for_len(areq->dst, areq->cryptlen) > 8) {
  40                 algt->stat_fb_sgnum++;
  41                 return true;
  42         }
  43
  44         len = areq->cryptlen;
  45         sg = areq->src;
  46         while (sg) {
  47                 todo = min(len, sg->length);
  48                 if ((todo % 16) != 0) {
  49                         algt->stat_fb_sglen++;
  50                         return true;
  51                 }
  52                 if (!IS_ALIGNED(sg->offset, 16)) {
  53                         algt->stat_fb_align++;
  54                         return true;
  55                 }
  56                 len -= todo;
  57                 sg = sg_next(sg);
  58         }
  59         len = areq->cryptlen;
  60         sg = areq->dst;
  61         while (sg) {
  62                 todo = min(len, sg->length);
  63                 if ((todo % 16) != 0) {
  64                         algt->stat_fb_sglen++;
  65                         return true;
  66                 }
  67                 if (!IS_ALIGNED(sg->offset, 16)) {
  68                         algt->stat_fb_align++;
  69                         return true;
  70                 }
  71                 len -= todo;
  72                 sg = sg_next(sg);
  73         }
  74
  75         /* SS need same numbers of SG (with same length) for source and destination */
  76         in_sg = areq->src;
  77         out_sg = areq->dst;
  78         while (in_sg && out_sg) {
  79                 if (in_sg->length != out_sg->length)
  80                         return true;
  81                 in_sg = sg_next(in_sg);
  82                 out_sg = sg_next(out_sg);
  83         }
  84         if (in_sg || out_sg)
  85                 return true;
  86         return false;
  87 }
  88
  89 static int sun8i_ss_cipher_fallback(struct skcipher_request *areq)
  90 {
  91         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
  92         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
  93         struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
  94         int err;
  95
  96         if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG)) {
  97                 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
  98                 struct sun8i_ss_alg_template *algt __maybe_unused;
  99
 100                 algt = container_of(alg, struct sun8i_ss_alg_template,
 101                                     alg.skcipher.base);
 102
 103 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
 104                 algt->stat_fb++;
 105 #endif
 106         }
 107
 108         skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
 109         skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
 110                                       areq->base.complete, areq->base.data);
 111         skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
 112                                    areq->cryptlen, areq->iv);
 113         if (rctx->op_dir & SS_DECRYPTION)
 114                 err = crypto_skcipher_decrypt(&rctx->fallback_req);
 115         else
 116                 err = crypto_skcipher_encrypt(&rctx->fallback_req);
 117         return err;
 118 }
 119
 120 static int sun8i_ss_setup_ivs(struct skcipher_request *areq)
 121 {
 122         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
 123         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
 124         struct sun8i_ss_dev *ss = op->ss;
 125         struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 126         struct scatterlist *sg = areq->src;
 127         unsigned int todo, offset;
 128         unsigned int len = areq->cryptlen;
 129         unsigned int ivsize = crypto_skcipher_ivsize(tfm);
 130         struct sun8i_ss_flow *sf = &ss->flows[rctx->flow];
 131         int i = 0;
 132         dma_addr_t a;
 133         int err;
 134
 135         rctx->ivlen = ivsize;
 136         if (rctx->op_dir & SS_DECRYPTION) {
 137                 offset = areq->cryptlen - ivsize;
 138                 scatterwalk_map_and_copy(sf->biv, areq->src, offset,
 139                                          ivsize, 0);
 140         }
 141
 142         /* we need to copy all IVs from source in case DMA is bi-directionnal */
 143         while (sg && len) {
 144                 if (sg_dma_len(sg) == 0) {
 145                         sg = sg_next(sg);
 146                         continue;
 147                 }
 148                 if (i == 0)
 149                         memcpy(sf->iv[0], areq->iv, ivsize);
 150                 a = dma_map_single(ss->dev, sf->iv[i], ivsize, DMA_TO_DEVICE);
 151                 if (dma_mapping_error(ss->dev, a)) {
 152                         memzero_explicit(sf->iv[i], ivsize);
 153                         dev_err(ss->dev, "Cannot DMA MAP IV\n");
 154                         err = -EFAULT;
 155                         goto dma_iv_error;
 156                 }
 157                 rctx->p_iv[i] = a;
 158                 /* we need to setup all others IVs only in the decrypt way */
 159                 if (rctx->op_dir == SS_ENCRYPTION)
 160                         return 0;
 161                 todo = min(len, sg_dma_len(sg));
 162                 len -= todo;
 163                 i++;
 164                 if (i < MAX_SG) {
 165                         offset = sg->length - ivsize;
 166                         scatterwalk_map_and_copy(sf->iv[i], sg, offset, ivsize, 0);
 167                 }
 168                 rctx->niv = i;
 169                 sg = sg_next(sg);
 170         }
 171
 172         return 0;
 173 dma_iv_error:
 174         i--;
 175         while (i >= 0) {
 176                 dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE);
 177                 memzero_explicit(sf->iv[i], ivsize);
 178                 i--;
 179         }
 180         return err;
 181 }
 182
 183 static int sun8i_ss_cipher(struct skcipher_request *areq)
 184 {
 185         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
 186         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
 187         struct sun8i_ss_dev *ss = op->ss;
 188         struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 189         struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
 190         struct sun8i_ss_alg_template *algt;
 191         struct sun8i_ss_flow *sf = &ss->flows[rctx->flow];
 192         struct scatterlist *sg;
 193         unsigned int todo, len, offset, ivsize;
 194         int nr_sgs = 0;
 195         int nr_sgd = 0;
 196         int err = 0;
 197         int nsgs = sg_nents_for_len(areq->src, areq->cryptlen);
 198         int nsgd = sg_nents_for_len(areq->dst, areq->cryptlen);
 199         int i;
 200
 201         algt = container_of(alg, struct sun8i_ss_alg_template, alg.skcipher.base);
 202
 203         dev_dbg(ss->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,
 204                 crypto_tfm_alg_name(areq->base.tfm),
 205                 areq->cryptlen,
 206                 rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),
 207                 op->keylen);
 208
 209 #ifdef CONFIG_CRYPTO_DEV_SUN8I_SS_DEBUG
 210         algt->stat_req++;
 211 #endif
 212
 213         rctx->op_mode = ss->variant->op_mode[algt->ss_blockmode];
 214         rctx->method = ss->variant->alg_cipher[algt->ss_algo_id];
 215         rctx->keylen = op->keylen;
 216
 217         rctx->p_key = dma_map_single(ss->dev, op->key, op->keylen, DMA_TO_DEVICE);
 218         if (dma_mapping_error(ss->dev, rctx->p_key)) {
 219                 dev_err(ss->dev, "Cannot DMA MAP KEY\n");
 220                 err = -EFAULT;
 221                 goto theend;
 222         }
 223
 224         ivsize = crypto_skcipher_ivsize(tfm);
 225         if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
 226                 err = sun8i_ss_setup_ivs(areq);
 227                 if (err)
 228                         goto theend_key;
 229         }
 230         if (areq->src == areq->dst) {
 231                 nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_BIDIRECTIONAL);
 232                 if (nr_sgs <= 0 || nr_sgs > 8) {
 233                         dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs);
 234                         err = -EINVAL;
 235                         goto theend_iv;
 236                 }
 237                 nr_sgd = nr_sgs;
 238         } else {
 239                 nr_sgs = dma_map_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE);
 240                 if (nr_sgs <= 0 || nr_sgs > 8) {
 241                         dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs);
 242                         err = -EINVAL;
 243                         goto theend_iv;
 244                 }
 245                 nr_sgd = dma_map_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE);
 246                 if (nr_sgd <= 0 || nr_sgd > 8) {
 247                         dev_err(ss->dev, "Invalid sg number %d\n", nr_sgd);
 248                         err = -EINVAL;
 249                         goto theend_sgs;
 250                 }
 251         }
 252
 253         len = areq->cryptlen;
 254         i = 0;
 255         sg = areq->src;
 256         while (i < nr_sgs && sg && len) {
 257                 if (sg_dma_len(sg) == 0)
 258                         goto sgs_next;
 259                 rctx->t_src[i].addr = sg_dma_address(sg);
 260                 todo = min(len, sg_dma_len(sg));
 261                 rctx->t_src[i].len = todo / 4;
 262                 dev_dbg(ss->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__,
 263                         areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo);
 264                 len -= todo;
 265                 i++;
 266 sgs_next:
 267                 sg = sg_next(sg);
 268         }
 269         if (len > 0) {
 270                 dev_err(ss->dev, "remaining len %d\n", len);
 271                 err = -EINVAL;
 272                 goto theend_sgs;
 273         }
 274
 275         len = areq->cryptlen;
 276         i = 0;
 277         sg = areq->dst;
 278         while (i < nr_sgd && sg && len) {
 279                 if (sg_dma_len(sg) == 0)
 280                         goto sgd_next;
 281                 rctx->t_dst[i].addr = sg_dma_address(sg);
 282                 todo = min(len, sg_dma_len(sg));
 283                 rctx->t_dst[i].len = todo / 4;
 284                 dev_dbg(ss->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__,
 285                         areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo);
 286                 len -= todo;
 287                 i++;
 288 sgd_next:
 289                 sg = sg_next(sg);
 290         }
 291         if (len > 0) {
 292                 dev_err(ss->dev, "remaining len %d\n", len);
 293                 err = -EINVAL;
 294                 goto theend_sgs;
 295         }
 296
 297         err = sun8i_ss_run_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm));
 298
 299 theend_sgs:
 300         if (areq->src == areq->dst) {
 301                 dma_unmap_sg(ss->dev, areq->src, nsgs, DMA_BIDIRECTIONAL);
 302         } else {
 303                 dma_unmap_sg(ss->dev, areq->src, nsgs, DMA_TO_DEVICE);
 304                 dma_unmap_sg(ss->dev, areq->dst, nsgd, DMA_FROM_DEVICE);
 305         }
 306
 307 theend_iv:
 308         if (areq->iv && ivsize > 0) {
 309                 for (i = 0; i < rctx->niv; i++) {
 310                         dma_unmap_single(ss->dev, rctx->p_iv[i], ivsize, DMA_TO_DEVICE);
 311                         memzero_explicit(sf->iv[i], ivsize);
 312                 }
 313
 314                 offset = areq->cryptlen - ivsize;
 315                 if (rctx->op_dir & SS_DECRYPTION) {
 316                         memcpy(areq->iv, sf->biv, ivsize);
 317                         memzero_explicit(sf->biv, ivsize);
 318                 } else {
 319                         scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
 320                                         ivsize, 0);
 321                 }
 322         }
 323
 324 theend_key:
 325         dma_unmap_single(ss->dev, rctx->p_key, op->keylen, DMA_TO_DEVICE);
 326
 327 theend:
 328
 329         return err;
 330 }
 331
 332 int sun8i_ss_handle_cipher_request(struct crypto_engine *engine, void *areq)
 333 {
 334         int err;
 335         struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
 336
 337         err = sun8i_ss_cipher(breq);
 338         local_bh_disable();
 339         crypto_finalize_skcipher_request(engine, breq, err);
 340         local_bh_enable();
 341
 342         return 0;
 343 }
 344
 345 int sun8i_ss_skdecrypt(struct skcipher_request *areq)
 346 {
 347         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
 348         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
 349         struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 350         struct crypto_engine *engine;
 351         int e;
 352
 353         memset(rctx, 0, sizeof(struct sun8i_cipher_req_ctx));
 354         rctx->op_dir = SS_DECRYPTION;
 355
 356         if (sun8i_ss_need_fallback(areq))
 357                 return sun8i_ss_cipher_fallback(areq);
 358
 359         e = sun8i_ss_get_engine_number(op->ss);
 360         engine = op->ss->flows[e].engine;
 361         rctx->flow = e;
 362
 363         return crypto_transfer_skcipher_request_to_engine(engine, areq);
 364 }
 365
 366 int sun8i_ss_skencrypt(struct skcipher_request *areq)
 367 {
 368         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
 369         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
 370         struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
 371         struct crypto_engine *engine;
 372         int e;
 373
 374         memset(rctx, 0, sizeof(struct sun8i_cipher_req_ctx));
 375         rctx->op_dir = SS_ENCRYPTION;
 376
 377         if (sun8i_ss_need_fallback(areq))
 378                 return sun8i_ss_cipher_fallback(areq);
 379
 380         e = sun8i_ss_get_engine_number(op->ss);
 381         engine = op->ss->flows[e].engine;
 382         rctx->flow = e;
 383
 384         return crypto_transfer_skcipher_request_to_engine(engine, areq);
 385 }
 386
 387 int sun8i_ss_cipher_init(struct crypto_tfm *tfm)
 388 {
 389         struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
 390         struct sun8i_ss_alg_template *algt;
 391         const char *name = crypto_tfm_alg_name(tfm);
 392         struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
 393         struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
 394         int err;
 395
 396         memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx));
 397
 398         algt = container_of(alg, struct sun8i_ss_alg_template, alg.skcipher.base);
 399         op->ss = algt->ss;
 400
 401         op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
 402         if (IS_ERR(op->fallback_tfm)) {
 403                 dev_err(op->ss->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
 404                         name, PTR_ERR(op->fallback_tfm));
 405                 return PTR_ERR(op->fallback_tfm);
 406         }
 407
 408         crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) +
 409                                     crypto_skcipher_reqsize(op->fallback_tfm));
 410
 411         memcpy(algt->fbname,
 412                crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)),
 413                CRYPTO_MAX_ALG_NAME);
 414
 415         err = pm_runtime_resume_and_get(op->ss->dev);
 416         if (err < 0) {
 417                 dev_err(op->ss->dev, "pm error %d\n", err);
 418                 goto error_pm;
 419         }
 420
 421         return 0;
 422 error_pm:
 423         crypto_free_skcipher(op->fallback_tfm);
 424         return err;
 425 }
 426
 427 void sun8i_ss_cipher_exit(struct crypto_tfm *tfm)
 428 {
 429         struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
 430
 431         kfree_sensitive(op->key);
 432         crypto_free_skcipher(op->fallback_tfm);
 433         pm_runtime_put_sync(op->ss->dev);
 434 }
 435
 436 int sun8i_ss_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
 437                         unsigned int keylen)
 438 {
 439         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
 440         struct sun8i_ss_dev *ss = op->ss;
 441
 442         switch (keylen) {
 443         case 128 / 8:
 444                 break;
 445         case 192 / 8:
 446                 break;
 447         case 256 / 8:
 448                 break;
 449         default:
 450                 dev_dbg(ss->dev, "ERROR: Invalid keylen %u\n", keylen);
 451                 return -EINVAL;
 452         }
 453         kfree_sensitive(op->key);
 454         op->keylen = keylen;
 455         op->key = kmemdup(key, keylen, GFP_KERNEL);
 456         if (!op->key)
 457                 return -ENOMEM;
 458
 459         crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
 460         crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
 461
 462         return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
 463 }
 464
 465 int sun8i_ss_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
 466                          unsigned int keylen)
 467 {
 468         struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
 469         struct sun8i_ss_dev *ss = op->ss;
 470
 471         if (unlikely(keylen != 3 * DES_KEY_SIZE)) {
 472                 dev_dbg(ss->dev, "Invalid keylen %u\n", keylen);
 473                 return -EINVAL;
 474         }
 475
 476         kfree_sensitive(op->key);
 477         op->keylen = keylen;
 478         op->key = kmemdup(key, keylen, GFP_KERNEL);
 479         if (!op->key)
 480                 return -ENOMEM;
 481
 482         crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
 483         crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
 484
 485         return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
 486 }