crypto/cryptd.c

   1 /*
   2  * Software async crypto daemon.
   3  *
   4  * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License as published by the Free
   8  * Software Foundation; either version 2 of the License, or (at your option)
   9  * any later version.
  10  *
  11  */
  12
  13 #include <crypto/algapi.h>
  14 #include <crypto/internal/hash.h>
  15 #include <linux/err.h>
  16 #include <linux/init.h>
  17 #include <linux/kernel.h>
  18 #include <linux/kthread.h>
  19 #include <linux/list.h>
  20 #include <linux/module.h>
  21 #include <linux/mutex.h>
  22 #include <linux/scatterlist.h>
  23 #include <linux/sched.h>
  24 #include <linux/slab.h>
  25 #include <linux/spinlock.h>
  26
  27 #define CRYPTD_MAX_QLEN 100
  28
  29 struct cryptd_state {
  30         spinlock_t lock;
  31         struct mutex mutex;
  32         struct crypto_queue queue;
  33         struct task_struct *task;
  34 };
  35
  36 struct cryptd_instance_ctx {
  37         struct crypto_spawn spawn;
  38         struct cryptd_state *state;
  39 };
  40
  41 struct cryptd_blkcipher_ctx {
  42         struct crypto_blkcipher *child;
  43 };
  44
  45 struct cryptd_blkcipher_request_ctx {
  46         crypto_completion_t complete;
  47 };
  48
  49 struct cryptd_hash_ctx {
  50         struct crypto_hash *child;
  51 };
  52
  53 struct cryptd_hash_request_ctx {
  54         crypto_completion_t complete;
  55 };
  56
  57 static inline struct cryptd_state *cryptd_get_state(struct crypto_tfm *tfm)
  58 {
  59         struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
  60         struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
  61         return ictx->state;
  62 }
  63
  64 static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
  65                                    const u8 *key, unsigned int keylen)
  66 {
  67         struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
  68         struct crypto_blkcipher *child = ctx->child;
  69         int err;
  70
  71         crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
  72         crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
  73                                           CRYPTO_TFM_REQ_MASK);
  74         err = crypto_blkcipher_setkey(child, key, keylen);
  75         crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
  76                                             CRYPTO_TFM_RES_MASK);
  77         return err;
  78 }
  79
  80 static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
  81                                    struct crypto_blkcipher *child,
  82                                    int err,
  83                                    int (*crypt)(struct blkcipher_desc *desc,
  84                                                 struct scatterlist *dst,
  85                                                 struct scatterlist *src,
  86                                                 unsigned int len))
  87 {
  88         struct cryptd_blkcipher_request_ctx *rctx;
  89         struct blkcipher_desc desc;
  90
  91         rctx = ablkcipher_request_ctx(req);
  92
  93         if (unlikely(err == -EINPROGRESS))
  94                 goto out;
  95
  96         desc.tfm = child;
  97         desc.info = req->info;
  98         desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
  99
 100         err = crypt(&desc, req->dst, req->src, req->nbytes);
 101
 102         req->base.complete = rctx->complete;
 103
 104 out:
 105         local_bh_disable();
 106         rctx->complete(&req->base, err);
 107         local_bh_enable();
 108 }
 109
 110 static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
 111 {
 112         struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
 113         struct crypto_blkcipher *child = ctx->child;
 114
 115         cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
 116                                crypto_blkcipher_crt(child)->encrypt);
 117 }
 118
 119 static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
 120 {
 121         struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
 122         struct crypto_blkcipher *child = ctx->child;
 123
 124         cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
 125                                crypto_blkcipher_crt(child)->decrypt);
 126 }
 127
 128 static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
 129                                     crypto_completion_t complete)
 130 {
 131         struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
 132         struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
 133         struct cryptd_state *state =
 134                 cryptd_get_state(crypto_ablkcipher_tfm(tfm));
 135         int err;
 136
 137         rctx->complete = req->base.complete;
 138         req->base.complete = complete;
 139
 140         spin_lock_bh(&state->lock);
 141         err = ablkcipher_enqueue_request(&state->queue, req);
 142         spin_unlock_bh(&state->lock);
 143
 144         wake_up_process(state->task);
 145         return err;
 146 }
 147
 148 static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
 149 {
 150         return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
 151 }
 152
 153 static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
 154 {
 155         return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
 156 }
 157
 158 static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
 159 {
 160         struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 161         struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
 162         struct crypto_spawn *spawn = &ictx->spawn;
 163         struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 164         struct crypto_blkcipher *cipher;
 165
 166         cipher = crypto_spawn_blkcipher(spawn);
 167         if (IS_ERR(cipher))
 168                 return PTR_ERR(cipher);
 169
 170         ctx->child = cipher;
 171         tfm->crt_ablkcipher.reqsize =
 172                 sizeof(struct cryptd_blkcipher_request_ctx);
 173         return 0;
 174 }
 175
 176 static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
 177 {
 178         struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
 179         struct cryptd_state *state = cryptd_get_state(tfm);
 180         int active;
 181
 182         mutex_lock(&state->mutex);
 183         active = ablkcipher_tfm_in_queue(&state->queue,
 184                                          __crypto_ablkcipher_cast(tfm));
 185         mutex_unlock(&state->mutex);
 186
 187         BUG_ON(active);
 188
 189         crypto_free_blkcipher(ctx->child);
 190 }
 191
 192 static struct crypto_instance *cryptd_alloc_instance(struct crypto_alg *alg,
 193                                                      struct cryptd_state *state)
 194 {
 195         struct crypto_instance *inst;
 196         struct cryptd_instance_ctx *ctx;
 197         int err;
 198
 199         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 200         if (!inst) {
 201                 inst = ERR_PTR(-ENOMEM);
 202                 goto out;
 203         }
 204
 205         err = -ENAMETOOLONG;
 206         if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 207                      "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 208                 goto out_free_inst;
 209
 210         ctx = crypto_instance_ctx(inst);
 211         err = crypto_init_spawn(&ctx->spawn, alg, inst,
 212                                 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
 213         if (err)
 214                 goto out_free_inst;
 215
 216         ctx->state = state;
 217
 218         memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
 219
 220         inst->alg.cra_priority = alg->cra_priority + 50;
 221         inst->alg.cra_blocksize = alg->cra_blocksize;
 222         inst->alg.cra_alignmask = alg->cra_alignmask;
 223
 224 out:
 225         return inst;
 226
 227 out_free_inst:
 228         kfree(inst);
 229         inst = ERR_PTR(err);
 230         goto out;
 231 }
 232
 233 static struct crypto_instance *cryptd_alloc_blkcipher(
 234         struct rtattr **tb, struct cryptd_state *state)
 235 {
 236         struct crypto_instance *inst;
 237         struct crypto_alg *alg;
 238
 239         alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER,
 240                                   CRYPTO_ALG_TYPE_MASK);
 241         if (IS_ERR(alg))
 242                 return ERR_CAST(alg);
 243
 244         inst = cryptd_alloc_instance(alg, state);
 245         if (IS_ERR(inst))
 246                 goto out_put_alg;
 247
 248         inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
 249         inst->alg.cra_type = &crypto_ablkcipher_type;
 250
 251         inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
 252         inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
 253         inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
 254
 255         inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
 256
 257         inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
 258
 259         inst->alg.cra_init = cryptd_blkcipher_init_tfm;
 260         inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
 261
 262         inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
 263         inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
 264         inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
 265
 266 out_put_alg:
 267         crypto_mod_put(alg);
 268         return inst;
 269 }
 270
 271 static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
 272 {
 273         struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
 274         struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
 275         struct crypto_spawn *spawn = &ictx->spawn;
 276         struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 277         struct crypto_hash *cipher;
 278
 279         cipher = crypto_spawn_hash(spawn);
 280         if (IS_ERR(cipher))
 281                 return PTR_ERR(cipher);
 282
 283         ctx->child = cipher;
 284         tfm->crt_ahash.reqsize =
 285                 sizeof(struct cryptd_hash_request_ctx);
 286         return 0;
 287 }
 288
 289 static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
 290 {
 291         struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
 292         struct cryptd_state *state = cryptd_get_state(tfm);
 293         int active;
 294
 295         mutex_lock(&state->mutex);
 296         active = ahash_tfm_in_queue(&state->queue,
 297                                 __crypto_ahash_cast(tfm));
 298         mutex_unlock(&state->mutex);
 299
 300         BUG_ON(active);
 301
 302         crypto_free_hash(ctx->child);
 303 }
 304
 305 static int cryptd_hash_setkey(struct crypto_ahash *parent,
 306                                    const u8 *key, unsigned int keylen)
 307 {
 308         struct cryptd_hash_ctx *ctx   = crypto_ahash_ctx(parent);
 309         struct crypto_hash     *child = ctx->child;
 310         int err;
 311
 312         crypto_hash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 313         crypto_hash_set_flags(child, crypto_ahash_get_flags(parent) &
 314                                           CRYPTO_TFM_REQ_MASK);
 315         err = crypto_hash_setkey(child, key, keylen);
 316         crypto_ahash_set_flags(parent, crypto_hash_get_flags(child) &
 317                                             CRYPTO_TFM_RES_MASK);
 318         return err;
 319 }
 320
 321 static int cryptd_hash_enqueue(struct ahash_request *req,
 322                                 crypto_completion_t complete)
 323 {
 324         struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
 325         struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
 326         struct cryptd_state *state =
 327                 cryptd_get_state(crypto_ahash_tfm(tfm));
 328         int err;
 329
 330         rctx->complete = req->base.complete;
 331         req->base.complete = complete;
 332
 333         spin_lock_bh(&state->lock);
 334         err = ahash_enqueue_request(&state->queue, req);
 335         spin_unlock_bh(&state->lock);
 336
 337         wake_up_process(state->task);
 338         return err;
 339 }
 340
 341 static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
 342 {
 343         struct cryptd_hash_ctx *ctx   = crypto_tfm_ctx(req_async->tfm);
 344         struct crypto_hash     *child = ctx->child;
 345         struct ahash_request    *req = ahash_request_cast(req_async);
 346         struct cryptd_hash_request_ctx *rctx;
 347         struct hash_desc desc;
 348
 349         rctx = ahash_request_ctx(req);
 350
 351         if (unlikely(err == -EINPROGRESS))
 352                 goto out;
 353
 354         desc.tfm = child;
 355         desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 356
 357         err = crypto_hash_crt(child)->init(&desc);
 358
 359         req->base.complete = rctx->complete;
 360
 361 out:
 362         local_bh_disable();
 363         rctx->complete(&req->base, err);
 364         local_bh_enable();
 365 }
 366
 367 static int cryptd_hash_init_enqueue(struct ahash_request *req)
 368 {
 369         return cryptd_hash_enqueue(req, cryptd_hash_init);
 370 }
 371
 372 static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
 373 {
 374         struct cryptd_hash_ctx *ctx   = crypto_tfm_ctx(req_async->tfm);
 375         struct crypto_hash     *child = ctx->child;
 376         struct ahash_request    *req = ahash_request_cast(req_async);
 377         struct cryptd_hash_request_ctx *rctx;
 378         struct hash_desc desc;
 379
 380         rctx = ahash_request_ctx(req);
 381
 382         if (unlikely(err == -EINPROGRESS))
 383                 goto out;
 384
 385         desc.tfm = child;
 386         desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 387
 388         err = crypto_hash_crt(child)->update(&desc,
 389                                                 req->src,
 390                                                 req->nbytes);
 391
 392         req->base.complete = rctx->complete;
 393
 394 out:
 395         local_bh_disable();
 396         rctx->complete(&req->base, err);
 397         local_bh_enable();
 398 }
 399
 400 static int cryptd_hash_update_enqueue(struct ahash_request *req)
 401 {
 402         return cryptd_hash_enqueue(req, cryptd_hash_update);
 403 }
 404
 405 static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
 406 {
 407         struct cryptd_hash_ctx *ctx   = crypto_tfm_ctx(req_async->tfm);
 408         struct crypto_hash     *child = ctx->child;
 409         struct ahash_request    *req = ahash_request_cast(req_async);
 410         struct cryptd_hash_request_ctx *rctx;
 411         struct hash_desc desc;
 412
 413         rctx = ahash_request_ctx(req);
 414
 415         if (unlikely(err == -EINPROGRESS))
 416                 goto out;
 417
 418         desc.tfm = child;
 419         desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 420
 421         err = crypto_hash_crt(child)->final(&desc, req->result);
 422
 423         req->base.complete = rctx->complete;
 424
 425 out:
 426         local_bh_disable();
 427         rctx->complete(&req->base, err);
 428         local_bh_enable();
 429 }
 430
 431 static int cryptd_hash_final_enqueue(struct ahash_request *req)
 432 {
 433         return cryptd_hash_enqueue(req, cryptd_hash_final);
 434 }
 435
 436 static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
 437 {
 438         struct cryptd_hash_ctx *ctx   = crypto_tfm_ctx(req_async->tfm);
 439         struct crypto_hash     *child = ctx->child;
 440         struct ahash_request    *req = ahash_request_cast(req_async);
 441         struct cryptd_hash_request_ctx *rctx;
 442         struct hash_desc desc;
 443
 444         rctx = ahash_request_ctx(req);
 445
 446         if (unlikely(err == -EINPROGRESS))
 447                 goto out;
 448
 449         desc.tfm = child;
 450         desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
 451
 452         err = crypto_hash_crt(child)->digest(&desc,
 453                                                 req->src,
 454                                                 req->nbytes,
 455                                                 req->result);
 456
 457         req->base.complete = rctx->complete;
 458
 459 out:
 460         local_bh_disable();
 461         rctx->complete(&req->base, err);
 462         local_bh_enable();
 463 }
 464
 465 static int cryptd_hash_digest_enqueue(struct ahash_request *req)
 466 {
 467         return cryptd_hash_enqueue(req, cryptd_hash_digest);
 468 }
 469
 470 static struct crypto_instance *cryptd_alloc_hash(
 471         struct rtattr **tb, struct cryptd_state *state)
 472 {
 473         struct crypto_instance *inst;
 474         struct crypto_alg *alg;
 475
 476         alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_HASH,
 477                                   CRYPTO_ALG_TYPE_HASH_MASK);
 478         if (IS_ERR(alg))
 479                 return ERR_PTR(PTR_ERR(alg));
 480
 481         inst = cryptd_alloc_instance(alg, state);
 482         if (IS_ERR(inst))
 483                 goto out_put_alg;
 484
 485         inst->alg.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC;
 486         inst->alg.cra_type = &crypto_ahash_type;
 487
 488         inst->alg.cra_ahash.digestsize = alg->cra_hash.digestsize;
 489         inst->alg.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
 490
 491         inst->alg.cra_init = cryptd_hash_init_tfm;
 492         inst->alg.cra_exit = cryptd_hash_exit_tfm;
 493
 494         inst->alg.cra_ahash.init   = cryptd_hash_init_enqueue;
 495         inst->alg.cra_ahash.update = cryptd_hash_update_enqueue;
 496         inst->alg.cra_ahash.final  = cryptd_hash_final_enqueue;
 497         inst->alg.cra_ahash.setkey = cryptd_hash_setkey;
 498         inst->alg.cra_ahash.digest = cryptd_hash_digest_enqueue;
 499
 500 out_put_alg:
 501         crypto_mod_put(alg);
 502         return inst;
 503 }
 504
 505 static struct cryptd_state state;
 506
 507 static struct crypto_instance *cryptd_alloc(struct rtattr **tb)
 508 {
 509         struct crypto_attr_type *algt;
 510
 511         algt = crypto_get_attr_type(tb);
 512         if (IS_ERR(algt))
 513                 return ERR_CAST(algt);
 514
 515         switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
 516         case CRYPTO_ALG_TYPE_BLKCIPHER:
 517                 return cryptd_alloc_blkcipher(tb, &state);
 518         case CRYPTO_ALG_TYPE_DIGEST:
 519                 return cryptd_alloc_hash(tb, &state);
 520         }
 521
 522         return ERR_PTR(-EINVAL);
 523 }
 524
 525 static void cryptd_free(struct crypto_instance *inst)
 526 {
 527         struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
 528
 529         crypto_drop_spawn(&ctx->spawn);
 530         kfree(inst);
 531 }
 532
 533 static struct crypto_template cryptd_tmpl = {
 534         .name = "cryptd",
 535         .alloc = cryptd_alloc,
 536         .free = cryptd_free,
 537         .module = THIS_MODULE,
 538 };
 539
 540 static inline int cryptd_create_thread(struct cryptd_state *state,
 541                                        int (*fn)(void *data), const char *name)
 542 {
 543         spin_lock_init(&state->lock);
 544         mutex_init(&state->mutex);
 545         crypto_init_queue(&state->queue, CRYPTD_MAX_QLEN);
 546
 547         state->task = kthread_run(fn, state, name);
 548         if (IS_ERR(state->task))
 549                 return PTR_ERR(state->task);
 550
 551         return 0;
 552 }
 553
 554 static inline void cryptd_stop_thread(struct cryptd_state *state)
 555 {
 556         BUG_ON(state->queue.qlen);
 557         kthread_stop(state->task);
 558 }
 559
 560 static int cryptd_thread(void *data)
 561 {
 562         struct cryptd_state *state = data;
 563         int stop;
 564
 565         current->flags |= PF_NOFREEZE;
 566
 567         do {
 568                 struct crypto_async_request *req, *backlog;
 569
 570                 mutex_lock(&state->mutex);
 571                 __set_current_state(TASK_INTERRUPTIBLE);
 572
 573                 spin_lock_bh(&state->lock);
 574                 backlog = crypto_get_backlog(&state->queue);
 575                 req = crypto_dequeue_request(&state->queue);
 576                 spin_unlock_bh(&state->lock);
 577
 578                 stop = kthread_should_stop();
 579
 580                 if (stop || req) {
 581                         __set_current_state(TASK_RUNNING);
 582                         if (req) {
 583                                 if (backlog)
 584                                         backlog->complete(backlog,
 585                                                           -EINPROGRESS);
 586                                 req->complete(req, 0);
 587                         }
 588                 }
 589
 590                 mutex_unlock(&state->mutex);
 591
 592                 schedule();
 593         } while (!stop);
 594
 595         return 0;
 596 }
 597
 598 static int __init cryptd_init(void)
 599 {
 600         int err;
 601
 602         err = cryptd_create_thread(&state, cryptd_thread, "cryptd");
 603         if (err)
 604                 return err;
 605
 606         err = crypto_register_template(&cryptd_tmpl);
 607         if (err)
 608                 kthread_stop(state.task);
 609
 610         return err;
 611 }
 612
 613 static void __exit cryptd_exit(void)
 614 {
 615         cryptd_stop_thread(&state);
 616         crypto_unregister_template(&cryptd_tmpl);
 617 }
 618
 619 module_init(cryptd_init);
 620 module_exit(cryptd_exit);
 621
 622 MODULE_LICENSE("GPL");
 623 MODULE_DESCRIPTION("Software async crypto daemon");