drivers/crypto/rockchip/rk3288_crypto_ahash.c

   1 /*
   2  * Crypto acceleration support for Rockchip RK3288
   3  *
   4  * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd
   5  *
   6  * Author: Zain Wang <zain.wang@rock-chips.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms and conditions of the GNU General Public License,
  10  * version 2, as published by the Free Software Foundation.
  11  *
  12  * Some ideas are from marvell/cesa.c and s5p-sss.c driver.
  13  */
  14 #include "rk3288_crypto.h"
  15
  16 /*
  17  * IC can not process zero message hash,
  18  * so we put the fixed hash out when met zero message.
  19  */
  20
  21 static int zero_message_process(struct ahash_request *req)
  22 {
  23         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  24         int rk_digest_size = crypto_ahash_digestsize(tfm);
  25
  26         switch (rk_digest_size) {
  27         case SHA1_DIGEST_SIZE:
  28                 memcpy(req->result, sha1_zero_message_hash, rk_digest_size);
  29                 break;
  30         case SHA256_DIGEST_SIZE:
  31                 memcpy(req->result, sha256_zero_message_hash, rk_digest_size);
  32                 break;
  33         case MD5_DIGEST_SIZE:
  34                 memcpy(req->result, md5_zero_message_hash, rk_digest_size);
  35                 break;
  36         default:
  37                 return -EINVAL;
  38         }
  39
  40         return 0;
  41 }
  42
  43 static void rk_ahash_crypto_complete(struct crypto_async_request *base, int err)
  44 {
  45         if (base->complete)
  46                 base->complete(base, err);
  47 }
  48
  49 static void rk_ahash_reg_init(struct rk_crypto_info *dev)
  50 {
  51         struct ahash_request *req = ahash_request_cast(dev->async_req);
  52         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
  53         int reg_status = 0;
  54
  55         reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL) |
  56                      RK_CRYPTO_HASH_FLUSH | _SBF(0xffff, 16);
  57         CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status);
  58
  59         reg_status = CRYPTO_READ(dev, RK_CRYPTO_CTRL);
  60         reg_status &= (~RK_CRYPTO_HASH_FLUSH);
  61         reg_status |= _SBF(0xffff, 16);
  62         CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, reg_status);
  63
  64         memset_io(dev->reg + RK_CRYPTO_HASH_DOUT_0, 0, 32);
  65
  66         CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, RK_CRYPTO_HRDMA_ERR_ENA |
  67                                             RK_CRYPTO_HRDMA_DONE_ENA);
  68
  69         CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, RK_CRYPTO_HRDMA_ERR_INT |
  70                                             RK_CRYPTO_HRDMA_DONE_INT);
  71
  72         CRYPTO_WRITE(dev, RK_CRYPTO_HASH_CTRL, rctx->mode |
  73                                                RK_CRYPTO_HASH_SWAP_DO);
  74
  75         CRYPTO_WRITE(dev, RK_CRYPTO_CONF, RK_CRYPTO_BYTESWAP_HRFIFO |
  76                                           RK_CRYPTO_BYTESWAP_BRFIFO |
  77                                           RK_CRYPTO_BYTESWAP_BTFIFO);
  78
  79         CRYPTO_WRITE(dev, RK_CRYPTO_HASH_MSG_LEN, dev->total);
  80 }
  81
  82 static int rk_ahash_init(struct ahash_request *req)
  83 {
  84         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
  85         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  86         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
  87
  88         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  89         rctx->fallback_req.base.flags = req->base.flags &
  90                                         CRYPTO_TFM_REQ_MAY_SLEEP;
  91
  92         return crypto_ahash_init(&rctx->fallback_req);
  93 }
  94
  95 static int rk_ahash_update(struct ahash_request *req)
  96 {
  97         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
  98         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  99         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 100
 101         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 102         rctx->fallback_req.base.flags = req->base.flags &
 103                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 104         rctx->fallback_req.nbytes = req->nbytes;
 105         rctx->fallback_req.src = req->src;
 106
 107         return crypto_ahash_update(&rctx->fallback_req);
 108 }
 109
 110 static int rk_ahash_final(struct ahash_request *req)
 111 {
 112         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 113         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 114         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 115
 116         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 117         rctx->fallback_req.base.flags = req->base.flags &
 118                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 119         rctx->fallback_req.result = req->result;
 120
 121         return crypto_ahash_final(&rctx->fallback_req);
 122 }
 123
 124 static int rk_ahash_finup(struct ahash_request *req)
 125 {
 126         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 127         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 128         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 129
 130         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 131         rctx->fallback_req.base.flags = req->base.flags &
 132                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 133
 134         rctx->fallback_req.nbytes = req->nbytes;
 135         rctx->fallback_req.src = req->src;
 136         rctx->fallback_req.result = req->result;
 137
 138         return crypto_ahash_finup(&rctx->fallback_req);
 139 }
 140
 141 static int rk_ahash_import(struct ahash_request *req, const void *in)
 142 {
 143         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 144         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 145         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 146
 147         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 148         rctx->fallback_req.base.flags = req->base.flags &
 149                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 150
 151         return crypto_ahash_import(&rctx->fallback_req, in);
 152 }
 153
 154 static int rk_ahash_export(struct ahash_request *req, void *out)
 155 {
 156         struct rk_ahash_rctx *rctx = ahash_request_ctx(req);
 157         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 158         struct rk_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
 159
 160         ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
 161         rctx->fallback_req.base.flags = req->base.flags &
 162                                         CRYPTO_TFM_REQ_MAY_SLEEP;
 163
 164         return crypto_ahash_export(&rctx->fallback_req, out);
 165 }
 166
 167 static int rk_ahash_digest(struct ahash_request *req)
 168 {
 169         struct rk_ahash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
 170         struct rk_crypto_info *dev = tctx->dev;
 171
 172         if (!req->nbytes)
 173                 return zero_message_process(req);
 174         else
 175                 return dev->enqueue(dev, &req->base);
 176 }
 177
 178 static void crypto_ahash_dma_start(struct rk_crypto_info *dev)
 179 {
 180         CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAS, dev->addr_in);
 181         CRYPTO_WRITE(dev, RK_CRYPTO_HRDMAL, (dev->count + 3) / 4);
 182         CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_HASH_START |
 183                                           (RK_CRYPTO_HASH_START << 16));
 184 }
 185
 186 static int rk_ahash_set_data_start(struct rk_crypto_info *dev)
 187 {
 188         int err;
 189
 190         err = dev->load_data(dev, dev->sg_src, NULL);
 191         if (!err)
 192                 crypto_ahash_dma_start(dev);
 193         return err;
 194 }
 195
 196 static int rk_ahash_start(struct rk_crypto_info *dev)
 197 {
 198         struct ahash_request *req = ahash_request_cast(dev->async_req);
 199         struct crypto_ahash *tfm;
 200         struct rk_ahash_rctx *rctx;
 201
 202         dev->total = req->nbytes;
 203         dev->left_bytes = req->nbytes;
 204         dev->aligned = 0;
 205         dev->align_size = 4;
 206         dev->sg_dst = NULL;
 207         dev->sg_src = req->src;
 208         dev->first = req->src;
 209         dev->nents = sg_nents(req->src);
 210         rctx = ahash_request_ctx(req);
 211         rctx->mode = 0;
 212
 213         tfm = crypto_ahash_reqtfm(req);
 214         switch (crypto_ahash_digestsize(tfm)) {
 215         case SHA1_DIGEST_SIZE:
 216                 rctx->mode = RK_CRYPTO_HASH_SHA1;
 217                 break;
 218         case SHA256_DIGEST_SIZE:
 219                 rctx->mode = RK_CRYPTO_HASH_SHA256;
 220                 break;
 221         case MD5_DIGEST_SIZE:
 222                 rctx->mode = RK_CRYPTO_HASH_MD5;
 223                 break;
 224         default:
 225                 return -EINVAL;
 226         }
 227
 228         rk_ahash_reg_init(dev);
 229         return rk_ahash_set_data_start(dev);
 230 }
 231
 232 static int rk_ahash_crypto_rx(struct rk_crypto_info *dev)
 233 {
 234         int err = 0;
 235         struct ahash_request *req = ahash_request_cast(dev->async_req);
 236         struct crypto_ahash *tfm;
 237
 238         dev->unload_data(dev);
 239         if (dev->left_bytes) {
 240                 if (dev->aligned) {
 241                         if (sg_is_last(dev->sg_src)) {
 242                                 dev_warn(dev->dev, "[%s:%d], Lack of data\n",
 243                                          __func__, __LINE__);
 244                                 err = -ENOMEM;
 245                                 goto out_rx;
 246                         }
 247                         dev->sg_src = sg_next(dev->sg_src);
 248                 }
 249                 err = rk_ahash_set_data_start(dev);
 250         } else {
 251                 /*
 252                  * it will take some time to process date after last dma
 253                  * transmission.
 254                  *
 255                  * waiting time is relative with the last date len,
 256                  * so cannot set a fixed time here.
 257                  * 10us makes system not call here frequently wasting
 258                  * efficiency, and make it response quickly when dma
 259                  * complete.
 260                  */
 261                 while (!CRYPTO_READ(dev, RK_CRYPTO_HASH_STS))
 262                         udelay(10);
 263
 264                 tfm = crypto_ahash_reqtfm(req);
 265                 memcpy_fromio(req->result, dev->reg + RK_CRYPTO_HASH_DOUT_0,
 266                               crypto_ahash_digestsize(tfm));
 267                 dev->complete(dev->async_req, 0);
 268                 tasklet_schedule(&dev->queue_task);
 269         }
 270
 271 out_rx:
 272         return err;
 273 }
 274
 275 static int rk_cra_hash_init(struct crypto_tfm *tfm)
 276 {
 277         struct rk_ahash_ctx *tctx = crypto_tfm_ctx(tfm);
 278         struct rk_crypto_tmp *algt;
 279         struct ahash_alg *alg = __crypto_ahash_alg(tfm->__crt_alg);
 280
 281         const char *alg_name = crypto_tfm_alg_name(tfm);
 282
 283         algt = container_of(alg, struct rk_crypto_tmp, alg.hash);
 284
 285         tctx->dev = algt->dev;
 286         tctx->dev->addr_vir = (void *)__get_free_page(GFP_KERNEL);
 287         if (!tctx->dev->addr_vir) {
 288                 dev_err(tctx->dev->dev, "failed to kmalloc for addr_vir\n");
 289                 return -ENOMEM;
 290         }
 291         tctx->dev->start = rk_ahash_start;
 292         tctx->dev->update = rk_ahash_crypto_rx;
 293         tctx->dev->complete = rk_ahash_crypto_complete;
 294
 295         /* for fallback */
 296         tctx->fallback_tfm = crypto_alloc_ahash(alg_name, 0,
 297                                                CRYPTO_ALG_NEED_FALLBACK);
 298         if (IS_ERR(tctx->fallback_tfm)) {
 299                 dev_err(tctx->dev->dev, "Could not load fallback driver.\n");
 300                 return PTR_ERR(tctx->fallback_tfm);
 301         }
 302         crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
 303                                  sizeof(struct rk_ahash_rctx) +
 304                                  crypto_ahash_reqsize(tctx->fallback_tfm));
 305
 306         return tctx->dev->enable_clk(tctx->dev);
 307 }
 308
 309 static void rk_cra_hash_exit(struct crypto_tfm *tfm)
 310 {
 311         struct rk_ahash_ctx *tctx = crypto_tfm_ctx(tfm);
 312
 313         free_page((unsigned long)tctx->dev->addr_vir);
 314         return tctx->dev->disable_clk(tctx->dev);
 315 }
 316
 317 struct rk_crypto_tmp rk_ahash_sha1 = {
 318         .type = ALG_TYPE_HASH,
 319         .alg.hash = {
 320                 .init = rk_ahash_init,
 321                 .update = rk_ahash_update,
 322                 .final = rk_ahash_final,
 323                 .finup = rk_ahash_finup,
 324                 .export = rk_ahash_export,
 325                 .import = rk_ahash_import,
 326                 .digest = rk_ahash_digest,
 327                 .halg = {
 328                          .digestsize = SHA1_DIGEST_SIZE,
 329                          .statesize = sizeof(struct sha1_state),
 330                          .base = {
 331                                   .cra_name = "sha1",
 332                                   .cra_driver_name = "rk-sha1",
 333                                   .cra_priority = 300,
 334                                   .cra_flags = CRYPTO_ALG_ASYNC |
 335                                                CRYPTO_ALG_NEED_FALLBACK,
 336                                   .cra_blocksize = SHA1_BLOCK_SIZE,
 337                                   .cra_ctxsize = sizeof(struct rk_ahash_ctx),
 338                                   .cra_alignmask = 3,
 339                                   .cra_init = rk_cra_hash_init,
 340                                   .cra_exit = rk_cra_hash_exit,
 341                                   .cra_module = THIS_MODULE,
 342                                   }
 343                          }
 344         }
 345 };
 346
 347 struct rk_crypto_tmp rk_ahash_sha256 = {
 348         .type = ALG_TYPE_HASH,
 349         .alg.hash = {
 350                 .init = rk_ahash_init,
 351                 .update = rk_ahash_update,
 352                 .final = rk_ahash_final,
 353                 .finup = rk_ahash_finup,
 354                 .export = rk_ahash_export,
 355                 .import = rk_ahash_import,
 356                 .digest = rk_ahash_digest,
 357                 .halg = {
 358                          .digestsize = SHA256_DIGEST_SIZE,
 359                          .statesize = sizeof(struct sha256_state),
 360                          .base = {
 361                                   .cra_name = "sha256",
 362                                   .cra_driver_name = "rk-sha256",
 363                                   .cra_priority = 300,
 364                                   .cra_flags = CRYPTO_ALG_ASYNC |
 365                                                CRYPTO_ALG_NEED_FALLBACK,
 366                                   .cra_blocksize = SHA256_BLOCK_SIZE,
 367                                   .cra_ctxsize = sizeof(struct rk_ahash_ctx),
 368                                   .cra_alignmask = 3,
 369                                   .cra_init = rk_cra_hash_init,
 370                                   .cra_exit = rk_cra_hash_exit,
 371                                   .cra_module = THIS_MODULE,
 372                                   }
 373                          }
 374         }
 375 };
 376
 377 struct rk_crypto_tmp rk_ahash_md5 = {
 378         .type = ALG_TYPE_HASH,
 379         .alg.hash = {
 380                 .init = rk_ahash_init,
 381                 .update = rk_ahash_update,
 382                 .final = rk_ahash_final,
 383                 .finup = rk_ahash_finup,
 384                 .export = rk_ahash_export,
 385                 .import = rk_ahash_import,
 386                 .digest = rk_ahash_digest,
 387                 .halg = {
 388                          .digestsize = MD5_DIGEST_SIZE,
 389                          .statesize = sizeof(struct md5_state),
 390                          .base = {
 391                                   .cra_name = "md5",
 392                                   .cra_driver_name = "rk-md5",
 393                                   .cra_priority = 300,
 394                                   .cra_flags = CRYPTO_ALG_ASYNC |
 395                                                CRYPTO_ALG_NEED_FALLBACK,
 396                                   .cra_blocksize = SHA1_BLOCK_SIZE,
 397                                   .cra_ctxsize = sizeof(struct rk_ahash_ctx),
 398                                   .cra_alignmask = 3,
 399                                   .cra_init = rk_cra_hash_init,
 400                                   .cra_exit = rk_cra_hash_exit,
 401                                   .cra_module = THIS_MODULE,
 402                                   }
 403                         }
 404         }
 405 };