crypto/gcm.c

   1 /*
   2  * GCM: Galois/Counter Mode.
   3  *
   4  * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  */
  10
  11 #include <crypto/gf128mul.h>
  12 #include <crypto/internal/aead.h>
  13 #include <crypto/internal/skcipher.h>
  14 #include <crypto/internal/hash.h>
  15 #include <crypto/null.h>
  16 #include <crypto/scatterwalk.h>
  17 #include <crypto/hash.h>
  18 #include "internal.h"
  19 #include <linux/completion.h>
  20 #include <linux/err.h>
  21 #include <linux/init.h>
  22 #include <linux/kernel.h>
  23 #include <linux/module.h>
  24 #include <linux/slab.h>
  25
  26 struct gcm_instance_ctx {
  27         struct crypto_skcipher_spawn ctr;
  28         struct crypto_ahash_spawn ghash;
  29 };
  30
  31 struct crypto_gcm_ctx {
  32         struct crypto_ablkcipher *ctr;
  33         struct crypto_ahash *ghash;
  34 };
  35
  36 struct crypto_rfc4106_ctx {
  37         struct crypto_aead *child;
  38         u8 nonce[4];
  39 };
  40
  41 struct crypto_rfc4543_instance_ctx {
  42         struct crypto_aead_spawn aead;
  43 };
  44
  45 struct crypto_rfc4543_ctx {
  46         struct crypto_aead *child;
  47         struct crypto_blkcipher *null;
  48         u8 nonce[4];
  49 };
  50
  51 struct crypto_rfc4543_req_ctx {
  52         struct aead_request subreq;
  53 };
  54
  55 struct crypto_gcm_ghash_ctx {
  56         unsigned int cryptlen;
  57         struct scatterlist *src;
  58         int (*complete)(struct aead_request *req, u32 flags);
  59 };
  60
  61 struct crypto_gcm_req_priv_ctx {
  62         u8 iv[16];
  63         u8 auth_tag[16];
  64         u8 iauth_tag[16];
  65         struct scatterlist src[3];
  66         struct scatterlist dst[3];
  67         struct scatterlist sg;
  68         struct crypto_gcm_ghash_ctx ghash_ctx;
  69         union {
  70                 struct ahash_request ahreq;
  71                 struct ablkcipher_request abreq;
  72         } u;
  73 };
  74
  75 struct crypto_gcm_setkey_result {
  76         int err;
  77         struct completion completion;
  78 };
  79
  80 static struct {
  81         u8 buf[16];
  82         struct scatterlist sg;
  83 } *gcm_zeroes;
  84
  85 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc);
  86
  87 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
  88         struct aead_request *req)
  89 {
  90         unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
  91
  92         return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
  93 }
  94
  95 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
  96 {
  97         struct crypto_gcm_setkey_result *result = req->data;
  98
  99         if (err == -EINPROGRESS)
 100                 return;
 101
 102         result->err = err;
 103         complete(&result->completion);
 104 }
 105
 106 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
 107                              unsigned int keylen)
 108 {
 109         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 110         struct crypto_ahash *ghash = ctx->ghash;
 111         struct crypto_ablkcipher *ctr = ctx->ctr;
 112         struct {
 113                 be128 hash;
 114                 u8 iv[8];
 115
 116                 struct crypto_gcm_setkey_result result;
 117
 118                 struct scatterlist sg[1];
 119                 struct ablkcipher_request req;
 120         } *data;
 121         int err;
 122
 123         crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
 124         crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
 125                                          CRYPTO_TFM_REQ_MASK);
 126         err = crypto_ablkcipher_setkey(ctr, key, keylen);
 127         crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
 128                                     CRYPTO_TFM_RES_MASK);
 129         if (err)
 130                 return err;
 131
 132         data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
 133                        GFP_KERNEL);
 134         if (!data)
 135                 return -ENOMEM;
 136
 137         init_completion(&data->result.completion);
 138         sg_init_one(data->sg, &data->hash, sizeof(data->hash));
 139         ablkcipher_request_set_tfm(&data->req, ctr);
 140         ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
 141                                                     CRYPTO_TFM_REQ_MAY_BACKLOG,
 142                                         crypto_gcm_setkey_done,
 143                                         &data->result);
 144         ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
 145                                      sizeof(data->hash), data->iv);
 146
 147         err = crypto_ablkcipher_encrypt(&data->req);
 148         if (err == -EINPROGRESS || err == -EBUSY) {
 149                 err = wait_for_completion_interruptible(
 150                         &data->result.completion);
 151                 if (!err)
 152                         err = data->result.err;
 153         }
 154
 155         if (err)
 156                 goto out;
 157
 158         crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
 159         crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
 160                                CRYPTO_TFM_REQ_MASK);
 161         err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
 162         crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
 163                               CRYPTO_TFM_RES_MASK);
 164
 165 out:
 166         kzfree(data);
 167         return err;
 168 }
 169
 170 static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
 171                                   unsigned int authsize)
 172 {
 173         switch (authsize) {
 174         case 4:
 175         case 8:
 176         case 12:
 177         case 13:
 178         case 14:
 179         case 15:
 180         case 16:
 181                 break;
 182         default:
 183                 return -EINVAL;
 184         }
 185
 186         return 0;
 187 }
 188
 189 static void crypto_gcm_init_common(struct aead_request *req)
 190 {
 191         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 192         __be32 counter = cpu_to_be32(1);
 193         struct scatterlist *sg;
 194
 195         memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
 196         memcpy(pctx->iv, req->iv, 12);
 197         memcpy(pctx->iv + 12, &counter, 4);
 198
 199         sg_init_table(pctx->src, 3);
 200         sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
 201         sg = scatterwalk_ffwd(pctx->src + 1, req->src, req->assoclen);
 202         if (sg != pctx->src + 1)
 203                 scatterwalk_sg_chain(pctx->src, 2, sg);
 204
 205         if (req->src != req->dst) {
 206                 sg_init_table(pctx->dst, 3);
 207                 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
 208                 sg = scatterwalk_ffwd(pctx->dst + 1, req->dst, req->assoclen);
 209                 if (sg != pctx->dst + 1)
 210                         scatterwalk_sg_chain(pctx->dst, 2, sg);
 211         }
 212 }
 213
 214 static void crypto_gcm_init_crypt(struct aead_request *req,
 215                                   unsigned int cryptlen)
 216 {
 217         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 218         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 219         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 220         struct ablkcipher_request *ablk_req = &pctx->u.abreq;
 221         struct scatterlist *dst;
 222
 223         dst = req->src == req->dst ? pctx->src : pctx->dst;
 224
 225         ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
 226         ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
 227                                      cryptlen + sizeof(pctx->auth_tag),
 228                                      pctx->iv);
 229 }
 230
 231 static inline unsigned int gcm_remain(unsigned int len)
 232 {
 233         len &= 0xfU;
 234         return len ? 16 - len : 0;
 235 }
 236
 237 static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
 238
 239 static int gcm_hash_update(struct aead_request *req,
 240                            crypto_completion_t compl,
 241                            struct scatterlist *src,
 242                            unsigned int len, u32 flags)
 243 {
 244         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 245         struct ahash_request *ahreq = &pctx->u.ahreq;
 246
 247         ahash_request_set_callback(ahreq, flags, compl, req);
 248         ahash_request_set_crypt(ahreq, src, NULL, len);
 249
 250         return crypto_ahash_update(ahreq);
 251 }
 252
 253 static int gcm_hash_remain(struct aead_request *req,
 254                            unsigned int remain,
 255                            crypto_completion_t compl, u32 flags)
 256 {
 257         return gcm_hash_update(req, compl, &gcm_zeroes->sg, remain, flags);
 258 }
 259
 260 static int gcm_hash_len(struct aead_request *req, u32 flags)
 261 {
 262         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 263         struct ahash_request *ahreq = &pctx->u.ahreq;
 264         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 265         u128 lengths;
 266
 267         lengths.a = cpu_to_be64(req->assoclen * 8);
 268         lengths.b = cpu_to_be64(gctx->cryptlen * 8);
 269         memcpy(pctx->iauth_tag, &lengths, 16);
 270         sg_init_one(&pctx->sg, pctx->iauth_tag, 16);
 271         ahash_request_set_callback(ahreq, flags, gcm_hash_len_done, req);
 272         ahash_request_set_crypt(ahreq, &pctx->sg,
 273                                 pctx->iauth_tag, sizeof(lengths));
 274
 275         return crypto_ahash_finup(ahreq);
 276 }
 277
 278 static int gcm_hash_len_continue(struct aead_request *req, u32 flags)
 279 {
 280         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 281         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 282
 283         return gctx->complete(req, flags);
 284 }
 285
 286 static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
 287 {
 288         struct aead_request *req = areq->data;
 289
 290         if (err)
 291                 goto out;
 292
 293         err = gcm_hash_len_continue(req, 0);
 294         if (err == -EINPROGRESS)
 295                 return;
 296
 297 out:
 298         aead_request_complete(req, err);
 299 }
 300
 301 static int gcm_hash_crypt_remain_continue(struct aead_request *req, u32 flags)
 302 {
 303         return gcm_hash_len(req, flags) ?:
 304                gcm_hash_len_continue(req, flags);
 305 }
 306
 307 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
 308                                        int err)
 309 {
 310         struct aead_request *req = areq->data;
 311
 312         if (err)
 313                 goto out;
 314
 315         err = gcm_hash_crypt_remain_continue(req, 0);
 316         if (err == -EINPROGRESS)
 317                 return;
 318
 319 out:
 320         aead_request_complete(req, err);
 321 }
 322
 323 static int gcm_hash_crypt_continue(struct aead_request *req, u32 flags)
 324 {
 325         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 326         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 327         unsigned int remain;
 328
 329         remain = gcm_remain(gctx->cryptlen);
 330         if (remain)
 331                 return gcm_hash_remain(req, remain,
 332                                        gcm_hash_crypt_remain_done, flags) ?:
 333                        gcm_hash_crypt_remain_continue(req, flags);
 334
 335         return gcm_hash_crypt_remain_continue(req, flags);
 336 }
 337
 338 static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
 339 {
 340         struct aead_request *req = areq->data;
 341
 342         if (err)
 343                 goto out;
 344
 345         err = gcm_hash_crypt_continue(req, 0);
 346         if (err == -EINPROGRESS)
 347                 return;
 348
 349 out:
 350         aead_request_complete(req, err);
 351 }
 352
 353 static int gcm_hash_assoc_remain_continue(struct aead_request *req, u32 flags)
 354 {
 355         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 356         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 357
 358         if (gctx->cryptlen)
 359                 return gcm_hash_update(req, gcm_hash_crypt_done,
 360                                        gctx->src, gctx->cryptlen, flags) ?:
 361                        gcm_hash_crypt_continue(req, flags);
 362
 363         return gcm_hash_crypt_remain_continue(req, flags);
 364 }
 365
 366 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
 367                                        int err)
 368 {
 369         struct aead_request *req = areq->data;
 370
 371         if (err)
 372                 goto out;
 373
 374         err = gcm_hash_assoc_remain_continue(req, 0);
 375         if (err == -EINPROGRESS)
 376                 return;
 377
 378 out:
 379         aead_request_complete(req, err);
 380 }
 381
 382 static int gcm_hash_assoc_continue(struct aead_request *req, u32 flags)
 383 {
 384         unsigned int remain;
 385
 386         remain = gcm_remain(req->assoclen);
 387         if (remain)
 388                 return gcm_hash_remain(req, remain,
 389                                        gcm_hash_assoc_remain_done, flags) ?:
 390                        gcm_hash_assoc_remain_continue(req, flags);
 391
 392         return gcm_hash_assoc_remain_continue(req, flags);
 393 }
 394
 395 static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
 396 {
 397         struct aead_request *req = areq->data;
 398
 399         if (err)
 400                 goto out;
 401
 402         err = gcm_hash_assoc_continue(req, 0);
 403         if (err == -EINPROGRESS)
 404                 return;
 405
 406 out:
 407         aead_request_complete(req, err);
 408 }
 409
 410 static int gcm_hash_init_continue(struct aead_request *req, u32 flags)
 411 {
 412         if (req->assoclen)
 413                 return gcm_hash_update(req, gcm_hash_assoc_done,
 414                                        req->src, req->assoclen, flags) ?:
 415                        gcm_hash_assoc_continue(req, flags);
 416
 417         return gcm_hash_assoc_remain_continue(req, flags);
 418 }
 419
 420 static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
 421 {
 422         struct aead_request *req = areq->data;
 423
 424         if (err)
 425                 goto out;
 426
 427         err = gcm_hash_init_continue(req, 0);
 428         if (err == -EINPROGRESS)
 429                 return;
 430
 431 out:
 432         aead_request_complete(req, err);
 433 }
 434
 435 static int gcm_hash(struct aead_request *req, u32 flags)
 436 {
 437         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 438         struct ahash_request *ahreq = &pctx->u.ahreq;
 439         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 440
 441         ahash_request_set_tfm(ahreq, ctx->ghash);
 442
 443         ahash_request_set_callback(ahreq, flags, gcm_hash_init_done, req);
 444         return crypto_ahash_init(ahreq) ?:
 445                gcm_hash_init_continue(req, flags);
 446 }
 447
 448 static int gcm_enc_copy_hash(struct aead_request *req, u32 flags)
 449 {
 450         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 451         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 452         u8 *auth_tag = pctx->auth_tag;
 453
 454         crypto_xor(auth_tag, pctx->iauth_tag, 16);
 455         scatterwalk_map_and_copy(auth_tag, req->dst,
 456                                  req->assoclen + req->cryptlen,
 457                                  crypto_aead_authsize(aead), 1);
 458         return 0;
 459 }
 460
 461 static int gcm_encrypt_continue(struct aead_request *req, u32 flags)
 462 {
 463         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 464         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 465
 466         gctx->src = sg_next(req->src == req->dst ? pctx->src : pctx->dst);
 467         gctx->cryptlen = req->cryptlen;
 468         gctx->complete = gcm_enc_copy_hash;
 469
 470         return gcm_hash(req, flags);
 471 }
 472
 473 static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
 474 {
 475         struct aead_request *req = areq->data;
 476
 477         if (err)
 478                 goto out;
 479
 480         err = gcm_encrypt_continue(req, 0);
 481         if (err == -EINPROGRESS)
 482                 return;
 483
 484 out:
 485         aead_request_complete(req, err);
 486 }
 487
 488 static int crypto_gcm_encrypt(struct aead_request *req)
 489 {
 490         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 491         struct ablkcipher_request *abreq = &pctx->u.abreq;
 492         u32 flags = aead_request_flags(req);
 493
 494         crypto_gcm_init_common(req);
 495         crypto_gcm_init_crypt(req, req->cryptlen);
 496         ablkcipher_request_set_callback(abreq, flags, gcm_encrypt_done, req);
 497
 498         return crypto_ablkcipher_encrypt(abreq) ?:
 499                gcm_encrypt_continue(req, flags);
 500 }
 501
 502 static int crypto_gcm_verify(struct aead_request *req)
 503 {
 504         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 505         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 506         u8 *auth_tag = pctx->auth_tag;
 507         u8 *iauth_tag = pctx->iauth_tag;
 508         unsigned int authsize = crypto_aead_authsize(aead);
 509         unsigned int cryptlen = req->cryptlen - authsize;
 510
 511         crypto_xor(auth_tag, iauth_tag, 16);
 512         scatterwalk_map_and_copy(iauth_tag, req->src,
 513                                  req->assoclen + cryptlen, authsize, 0);
 514         return crypto_memneq(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
 515 }
 516
 517 static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
 518 {
 519         struct aead_request *req = areq->data;
 520
 521         if (!err)
 522                 err = crypto_gcm_verify(req);
 523
 524         aead_request_complete(req, err);
 525 }
 526
 527 static int gcm_dec_hash_continue(struct aead_request *req, u32 flags)
 528 {
 529         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 530         struct ablkcipher_request *abreq = &pctx->u.abreq;
 531         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 532
 533         crypto_gcm_init_crypt(req, gctx->cryptlen);
 534         ablkcipher_request_set_callback(abreq, flags, gcm_decrypt_done, req);
 535         return crypto_ablkcipher_decrypt(abreq) ?: crypto_gcm_verify(req);
 536 }
 537
 538 static int crypto_gcm_decrypt(struct aead_request *req)
 539 {
 540         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 541         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 542         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 543         unsigned int authsize = crypto_aead_authsize(aead);
 544         unsigned int cryptlen = req->cryptlen;
 545         u32 flags = aead_request_flags(req);
 546
 547         cryptlen -= authsize;
 548
 549         crypto_gcm_init_common(req);
 550
 551         gctx->src = sg_next(pctx->src);
 552         gctx->cryptlen = cryptlen;
 553         gctx->complete = gcm_dec_hash_continue;
 554
 555         return gcm_hash(req, flags);
 556 }
 557
 558 static int crypto_gcm_init_tfm(struct crypto_aead *tfm)
 559 {
 560         struct aead_instance *inst = aead_alg_instance(tfm);
 561         struct gcm_instance_ctx *ictx = aead_instance_ctx(inst);
 562         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 563         struct crypto_ablkcipher *ctr;
 564         struct crypto_ahash *ghash;
 565         unsigned long align;
 566         int err;
 567
 568         ghash = crypto_spawn_ahash(&ictx->ghash);
 569         if (IS_ERR(ghash))
 570                 return PTR_ERR(ghash);
 571
 572         ctr = crypto_spawn_skcipher(&ictx->ctr);
 573         err = PTR_ERR(ctr);
 574         if (IS_ERR(ctr))
 575                 goto err_free_hash;
 576
 577         ctx->ctr = ctr;
 578         ctx->ghash = ghash;
 579
 580         align = crypto_aead_alignmask(tfm);
 581         align &= ~(crypto_tfm_ctx_alignment() - 1);
 582         crypto_aead_set_reqsize(tfm,
 583                 align + offsetof(struct crypto_gcm_req_priv_ctx, u) +
 584                 max(sizeof(struct ablkcipher_request) +
 585                     crypto_ablkcipher_reqsize(ctr),
 586                     sizeof(struct ahash_request) +
 587                     crypto_ahash_reqsize(ghash)));
 588
 589         return 0;
 590
 591 err_free_hash:
 592         crypto_free_ahash(ghash);
 593         return err;
 594 }
 595
 596 static void crypto_gcm_exit_tfm(struct crypto_aead *tfm)
 597 {
 598         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(tfm);
 599
 600         crypto_free_ahash(ctx->ghash);
 601         crypto_free_ablkcipher(ctx->ctr);
 602 }
 603
 604 static int crypto_gcm_create_common(struct crypto_template *tmpl,
 605                                     struct rtattr **tb,
 606                                     const char *full_name,
 607                                     const char *ctr_name,
 608                                     const char *ghash_name)
 609 {
 610         struct crypto_attr_type *algt;
 611         struct aead_instance *inst;
 612         struct crypto_alg *ctr;
 613         struct crypto_alg *ghash_alg;
 614         struct hash_alg_common *ghash;
 615         struct gcm_instance_ctx *ctx;
 616         int err;
 617
 618         algt = crypto_get_attr_type(tb);
 619         if (IS_ERR(algt))
 620                 return PTR_ERR(algt);
 621
 622         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 623                 return -EINVAL;
 624
 625         ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
 626                                     CRYPTO_ALG_TYPE_HASH,
 627                                     CRYPTO_ALG_TYPE_AHASH_MASK);
 628         if (IS_ERR(ghash_alg))
 629                 return PTR_ERR(ghash_alg);
 630
 631         ghash = __crypto_hash_alg_common(ghash_alg);
 632
 633         err = -ENOMEM;
 634         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 635         if (!inst)
 636                 goto out_put_ghash;
 637
 638         ctx = aead_instance_ctx(inst);
 639         err = crypto_init_ahash_spawn(&ctx->ghash, ghash,
 640                                       aead_crypto_instance(inst));
 641         if (err)
 642                 goto err_free_inst;
 643
 644         err = -EINVAL;
 645         if (ghash->digestsize != 16)
 646                 goto err_drop_ghash;
 647
 648         crypto_set_skcipher_spawn(&ctx->ctr, aead_crypto_instance(inst));
 649         err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
 650                                    crypto_requires_sync(algt->type,
 651                                                         algt->mask));
 652         if (err)
 653                 goto err_drop_ghash;
 654
 655         ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
 656
 657         /* We only support 16-byte blocks. */
 658         if (ctr->cra_ablkcipher.ivsize != 16)
 659                 goto out_put_ctr;
 660
 661         /* Not a stream cipher? */
 662         err = -EINVAL;
 663         if (ctr->cra_blocksize != 1)
 664                 goto out_put_ctr;
 665
 666         err = -ENAMETOOLONG;
 667         if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 668                      "gcm_base(%s,%s)", ctr->cra_driver_name,
 669                      ghash_alg->cra_driver_name) >=
 670             CRYPTO_MAX_ALG_NAME)
 671                 goto out_put_ctr;
 672
 673         memcpy(inst->alg.base.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
 674
 675         inst->alg.base.cra_flags = (ghash->base.cra_flags | ctr->cra_flags) &
 676                                    CRYPTO_ALG_ASYNC;
 677         inst->alg.base.cra_priority = (ghash->base.cra_priority +
 678                                        ctr->cra_priority) / 2;
 679         inst->alg.base.cra_blocksize = 1;
 680         inst->alg.base.cra_alignmask = ghash->base.cra_alignmask |
 681                                        ctr->cra_alignmask;
 682         inst->alg.base.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
 683         inst->alg.ivsize = 12;
 684         inst->alg.maxauthsize = 16;
 685         inst->alg.init = crypto_gcm_init_tfm;
 686         inst->alg.exit = crypto_gcm_exit_tfm;
 687         inst->alg.setkey = crypto_gcm_setkey;
 688         inst->alg.setauthsize = crypto_gcm_setauthsize;
 689         inst->alg.encrypt = crypto_gcm_encrypt;
 690         inst->alg.decrypt = crypto_gcm_decrypt;
 691
 692         err = aead_register_instance(tmpl, inst);
 693         if (err)
 694                 goto out_put_ctr;
 695
 696 out_put_ghash:
 697         crypto_mod_put(ghash_alg);
 698         return err;
 699
 700 out_put_ctr:
 701         crypto_drop_skcipher(&ctx->ctr);
 702 err_drop_ghash:
 703         crypto_drop_ahash(&ctx->ghash);
 704 err_free_inst:
 705         kfree(inst);
 706         goto out_put_ghash;
 707 }
 708
 709 static int crypto_gcm_create(struct crypto_template *tmpl, struct rtattr **tb)
 710 {
 711         const char *cipher_name;
 712         char ctr_name[CRYPTO_MAX_ALG_NAME];
 713         char full_name[CRYPTO_MAX_ALG_NAME];
 714
 715         cipher_name = crypto_attr_alg_name(tb[1]);
 716         if (IS_ERR(cipher_name))
 717                 return PTR_ERR(cipher_name);
 718
 719         if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
 720             CRYPTO_MAX_ALG_NAME)
 721                 return -ENAMETOOLONG;
 722
 723         if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
 724             CRYPTO_MAX_ALG_NAME)
 725                 return -ENAMETOOLONG;
 726
 727         return crypto_gcm_create_common(tmpl, tb, full_name,
 728                                         ctr_name, "ghash");
 729 }
 730
 731 static void crypto_gcm_free(struct crypto_instance *inst)
 732 {
 733         struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
 734
 735         crypto_drop_skcipher(&ctx->ctr);
 736         crypto_drop_ahash(&ctx->ghash);
 737         kfree(aead_instance(inst));
 738 }
 739
 740 static struct crypto_template crypto_gcm_tmpl = {
 741         .name = "gcm",
 742         .create = crypto_gcm_create,
 743         .free = crypto_gcm_free,
 744         .module = THIS_MODULE,
 745 };
 746
 747 static int crypto_gcm_base_create(struct crypto_template *tmpl,
 748                                   struct rtattr **tb)
 749 {
 750         const char *ctr_name;
 751         const char *ghash_name;
 752         char full_name[CRYPTO_MAX_ALG_NAME];
 753
 754         ctr_name = crypto_attr_alg_name(tb[1]);
 755         if (IS_ERR(ctr_name))
 756                 return PTR_ERR(ctr_name);
 757
 758         ghash_name = crypto_attr_alg_name(tb[2]);
 759         if (IS_ERR(ghash_name))
 760                 return PTR_ERR(ghash_name);
 761
 762         if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
 763                      ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
 764                 return -ENAMETOOLONG;
 765
 766         return crypto_gcm_create_common(tmpl, tb, full_name,
 767                                         ctr_name, ghash_name);
 768 }
 769
 770 static struct crypto_template crypto_gcm_base_tmpl = {
 771         .name = "gcm_base",
 772         .create = crypto_gcm_base_create,
 773         .free = crypto_gcm_free,
 774         .module = THIS_MODULE,
 775 };
 776
 777 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
 778                                  unsigned int keylen)
 779 {
 780         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
 781         struct crypto_aead *child = ctx->child;
 782         int err;
 783
 784         if (keylen < 4)
 785                 return -EINVAL;
 786
 787         keylen -= 4;
 788         memcpy(ctx->nonce, key + keylen, 4);
 789
 790         crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 791         crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
 792                                      CRYPTO_TFM_REQ_MASK);
 793         err = crypto_aead_setkey(child, key, keylen);
 794         crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
 795                                       CRYPTO_TFM_RES_MASK);
 796
 797         return err;
 798 }
 799
 800 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
 801                                       unsigned int authsize)
 802 {
 803         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
 804
 805         switch (authsize) {
 806         case 8:
 807         case 12:
 808         case 16:
 809                 break;
 810         default:
 811                 return -EINVAL;
 812         }
 813
 814         return crypto_aead_setauthsize(ctx->child, authsize);
 815 }
 816
 817 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
 818 {
 819         struct aead_request *subreq = aead_request_ctx(req);
 820         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 821         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
 822         struct crypto_aead *child = ctx->child;
 823         u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
 824                            crypto_aead_alignmask(child) + 1);
 825
 826         memcpy(iv, ctx->nonce, 4);
 827         memcpy(iv + 4, req->iv, 8);
 828
 829         aead_request_set_tfm(subreq, child);
 830         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
 831                                   req->base.data);
 832         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);
 833         aead_request_set_ad(subreq, req->assoclen);
 834
 835         return subreq;
 836 }
 837
 838 static int crypto_rfc4106_encrypt(struct aead_request *req)
 839 {
 840         req = crypto_rfc4106_crypt(req);
 841
 842         return crypto_aead_encrypt(req);
 843 }
 844
 845 static int crypto_rfc4106_decrypt(struct aead_request *req)
 846 {
 847         req = crypto_rfc4106_crypt(req);
 848
 849         return crypto_aead_decrypt(req);
 850 }
 851
 852 static int crypto_rfc4106_init_tfm(struct crypto_aead *tfm)
 853 {
 854         struct aead_instance *inst = aead_alg_instance(tfm);
 855         struct crypto_aead_spawn *spawn = aead_instance_ctx(inst);
 856         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
 857         struct crypto_aead *aead;
 858         unsigned long align;
 859
 860         aead = crypto_spawn_aead(spawn);
 861         if (IS_ERR(aead))
 862                 return PTR_ERR(aead);
 863
 864         ctx->child = aead;
 865
 866         align = crypto_aead_alignmask(aead);
 867         align &= ~(crypto_tfm_ctx_alignment() - 1);
 868         crypto_aead_set_reqsize(
 869                 tfm,
 870                 sizeof(struct aead_request) +
 871                 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
 872                 align + 12);
 873
 874         return 0;
 875 }
 876
 877 static void crypto_rfc4106_exit_tfm(struct crypto_aead *tfm)
 878 {
 879         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(tfm);
 880
 881         crypto_free_aead(ctx->child);
 882 }
 883
 884 static int crypto_rfc4106_create(struct crypto_template *tmpl,
 885                                  struct rtattr **tb)
 886 {
 887         struct crypto_attr_type *algt;
 888         struct aead_instance *inst;
 889         struct crypto_aead_spawn *spawn;
 890         struct aead_alg *alg;
 891         const char *ccm_name;
 892         int err;
 893
 894         algt = crypto_get_attr_type(tb);
 895         if (IS_ERR(algt))
 896                 return PTR_ERR(algt);
 897
 898         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 899                 return -EINVAL;
 900
 901         ccm_name = crypto_attr_alg_name(tb[1]);
 902         if (IS_ERR(ccm_name))
 903                 return PTR_ERR(ccm_name);
 904
 905         inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
 906         if (!inst)
 907                 return -ENOMEM;
 908
 909         spawn = aead_instance_ctx(inst);
 910         crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
 911         err = crypto_grab_aead(spawn, ccm_name, 0,
 912                                crypto_requires_sync(algt->type, algt->mask));
 913         if (err)
 914                 goto out_free_inst;
 915
 916         alg = crypto_spawn_aead_alg(spawn);
 917
 918         err = -EINVAL;
 919
 920         /* Underlying IV size must be 12. */
 921         if (crypto_aead_alg_ivsize(alg) != 12)
 922                 goto out_drop_alg;
 923
 924         /* Not a stream cipher? */
 925         if (alg->base.cra_blocksize != 1)
 926                 goto out_drop_alg;
 927
 928         err = -ENAMETOOLONG;
 929         if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
 930                      "rfc4106(%s)", alg->base.cra_name) >=
 931             CRYPTO_MAX_ALG_NAME ||
 932             snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 933                      "rfc4106(%s)", alg->base.cra_driver_name) >=
 934             CRYPTO_MAX_ALG_NAME)
 935                 goto out_drop_alg;
 936
 937         inst->alg.base.cra_flags |= alg->base.cra_flags & CRYPTO_ALG_ASYNC;
 938         inst->alg.base.cra_priority = alg->base.cra_priority;
 939         inst->alg.base.cra_blocksize = 1;
 940         inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
 941
 942         inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
 943
 944         inst->alg.ivsize = 8;
 945         inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
 946
 947         inst->alg.init = crypto_rfc4106_init_tfm;
 948         inst->alg.exit = crypto_rfc4106_exit_tfm;
 949
 950         inst->alg.setkey = crypto_rfc4106_setkey;
 951         inst->alg.setauthsize = crypto_rfc4106_setauthsize;
 952         inst->alg.encrypt = crypto_rfc4106_encrypt;
 953         inst->alg.decrypt = crypto_rfc4106_decrypt;
 954
 955         err = aead_register_instance(tmpl, inst);
 956         if (err)
 957                 goto out_drop_alg;
 958
 959 out:
 960         return err;
 961
 962 out_drop_alg:
 963         crypto_drop_aead(spawn);
 964 out_free_inst:
 965         kfree(inst);
 966         goto out;
 967 }
 968
 969 static void crypto_rfc4106_free(struct crypto_instance *inst)
 970 {
 971         crypto_drop_aead(crypto_instance_ctx(inst));
 972         kfree(aead_instance(inst));
 973 }
 974
 975 static struct crypto_template crypto_rfc4106_tmpl = {
 976         .name = "rfc4106",
 977         .create = crypto_rfc4106_create,
 978         .free = crypto_rfc4106_free,
 979         .module = THIS_MODULE,
 980 };
 981
 982 static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key,
 983                                  unsigned int keylen)
 984 {
 985         struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
 986         struct crypto_aead *child = ctx->child;
 987         int err;
 988
 989         if (keylen < 4)
 990                 return -EINVAL;
 991
 992         keylen -= 4;
 993         memcpy(ctx->nonce, key + keylen, 4);
 994
 995         crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 996         crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
 997                                      CRYPTO_TFM_REQ_MASK);
 998         err = crypto_aead_setkey(child, key, keylen);
 999         crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
1000                                       CRYPTO_TFM_RES_MASK);
1001
1002         return err;
1003 }
1004
1005 static int crypto_rfc4543_setauthsize(struct crypto_aead *parent,
1006                                       unsigned int authsize)
1007 {
1008         struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent);
1009
1010         if (authsize != 16)
1011                 return -EINVAL;
1012
1013         return crypto_aead_setauthsize(ctx->child, authsize);
1014 }
1015
1016 static int crypto_rfc4543_crypt(struct aead_request *req, bool enc)
1017 {
1018         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1019         struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1020         struct crypto_rfc4543_req_ctx *rctx = aead_request_ctx(req);
1021         struct aead_request *subreq = &rctx->subreq;
1022         unsigned int authsize = crypto_aead_authsize(aead);
1023         u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child),
1024                            crypto_aead_alignmask(ctx->child) + 1);
1025         int err;
1026
1027         if (req->src != req->dst) {
1028                 err = crypto_rfc4543_copy_src_to_dst(req, enc);
1029                 if (err)
1030                         return err;
1031         }
1032
1033         memcpy(iv, ctx->nonce, 4);
1034         memcpy(iv + 4, req->iv, 8);
1035
1036         aead_request_set_tfm(subreq, ctx->child);
1037         aead_request_set_callback(subreq, req->base.flags,
1038                                   req->base.complete, req->base.data);
1039         aead_request_set_crypt(subreq, req->src, req->dst,
1040                                enc ? 0 : authsize, iv);
1041         aead_request_set_ad(subreq, req->assoclen + req->cryptlen -
1042                                     subreq->cryptlen);
1043
1044         return enc ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq);
1045 }
1046
1047 static int crypto_rfc4543_copy_src_to_dst(struct aead_request *req, bool enc)
1048 {
1049         struct crypto_aead *aead = crypto_aead_reqtfm(req);
1050         struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead);
1051         unsigned int authsize = crypto_aead_authsize(aead);
1052         unsigned int nbytes = req->assoclen + req->cryptlen -
1053                               (enc ? 0 : authsize);
1054         struct blkcipher_desc desc = {
1055                 .tfm = ctx->null,
1056         };
1057
1058         return crypto_blkcipher_encrypt(&desc, req->dst, req->src, nbytes);
1059 }
1060
1061 static int crypto_rfc4543_encrypt(struct aead_request *req)
1062 {
1063         return crypto_rfc4543_crypt(req, true);
1064 }
1065
1066 static int crypto_rfc4543_decrypt(struct aead_request *req)
1067 {
1068         return crypto_rfc4543_crypt(req, false);
1069 }
1070
1071 static int crypto_rfc4543_init_tfm(struct crypto_aead *tfm)
1072 {
1073         struct aead_instance *inst = aead_alg_instance(tfm);
1074         struct crypto_rfc4543_instance_ctx *ictx = aead_instance_ctx(inst);
1075         struct crypto_aead_spawn *spawn = &ictx->aead;
1076         struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1077         struct crypto_aead *aead;
1078         struct crypto_blkcipher *null;
1079         unsigned long align;
1080         int err = 0;
1081
1082         aead = crypto_spawn_aead(spawn);
1083         if (IS_ERR(aead))
1084                 return PTR_ERR(aead);
1085
1086         null = crypto_get_default_null_skcipher();
1087         err = PTR_ERR(null);
1088         if (IS_ERR(null))
1089                 goto err_free_aead;
1090
1091         ctx->child = aead;
1092         ctx->null = null;
1093
1094         align = crypto_aead_alignmask(aead);
1095         align &= ~(crypto_tfm_ctx_alignment() - 1);
1096         crypto_aead_set_reqsize(
1097                 tfm,
1098                 sizeof(struct crypto_rfc4543_req_ctx) +
1099                 ALIGN(crypto_aead_reqsize(aead), crypto_tfm_ctx_alignment()) +
1100                 align + 12);
1101
1102         return 0;
1103
1104 err_free_aead:
1105         crypto_free_aead(aead);
1106         return err;
1107 }
1108
1109 static void crypto_rfc4543_exit_tfm(struct crypto_aead *tfm)
1110 {
1111         struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(tfm);
1112
1113         crypto_free_aead(ctx->child);
1114         crypto_put_default_null_skcipher();
1115 }
1116
1117 static int crypto_rfc4543_create(struct crypto_template *tmpl,
1118                                 struct rtattr **tb)
1119 {
1120         struct crypto_attr_type *algt;
1121         struct aead_instance *inst;
1122         struct crypto_aead_spawn *spawn;
1123         struct aead_alg *alg;
1124         struct crypto_rfc4543_instance_ctx *ctx;
1125         const char *ccm_name;
1126         int err;
1127
1128         algt = crypto_get_attr_type(tb);
1129         if (IS_ERR(algt))
1130                 return PTR_ERR(algt);
1131
1132         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
1133                 return -EINVAL;
1134
1135         ccm_name = crypto_attr_alg_name(tb[1]);
1136         if (IS_ERR(ccm_name))
1137                 return PTR_ERR(ccm_name);
1138
1139         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
1140         if (!inst)
1141                 return -ENOMEM;
1142
1143         ctx = aead_instance_ctx(inst);
1144         spawn = &ctx->aead;
1145         crypto_set_aead_spawn(spawn, aead_crypto_instance(inst));
1146         err = crypto_grab_aead(spawn, ccm_name, 0,
1147                                crypto_requires_sync(algt->type, algt->mask));
1148         if (err)
1149                 goto out_free_inst;
1150
1151         alg = crypto_spawn_aead_alg(spawn);
1152
1153         err = -EINVAL;
1154
1155         /* Underlying IV size must be 12. */
1156         if (crypto_aead_alg_ivsize(alg) != 12)
1157                 goto out_drop_alg;
1158
1159         /* Not a stream cipher? */
1160         if (alg->base.cra_blocksize != 1)
1161                 goto out_drop_alg;
1162
1163         err = -ENAMETOOLONG;
1164         if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
1165                      "rfc4543(%s)", alg->base.cra_name) >=
1166             CRYPTO_MAX_ALG_NAME ||
1167             snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1168                      "rfc4543(%s)", alg->base.cra_driver_name) >=
1169             CRYPTO_MAX_ALG_NAME)
1170                 goto out_drop_alg;
1171
1172         inst->alg.base.cra_flags = alg->base.cra_flags & CRYPTO_ALG_ASYNC;
1173         inst->alg.base.cra_priority = alg->base.cra_priority;
1174         inst->alg.base.cra_blocksize = 1;
1175         inst->alg.base.cra_alignmask = alg->base.cra_alignmask;
1176
1177         inst->alg.base.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx);
1178
1179         inst->alg.ivsize = 8;
1180         inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
1181
1182         inst->alg.init = crypto_rfc4543_init_tfm;
1183         inst->alg.exit = crypto_rfc4543_exit_tfm;
1184
1185         inst->alg.setkey = crypto_rfc4543_setkey;
1186         inst->alg.setauthsize = crypto_rfc4543_setauthsize;
1187         inst->alg.encrypt = crypto_rfc4543_encrypt;
1188         inst->alg.decrypt = crypto_rfc4543_decrypt;
1189
1190         err = aead_register_instance(tmpl, inst);
1191         if (err)
1192                 goto out_drop_alg;
1193
1194 out:
1195         return err;
1196
1197 out_drop_alg:
1198         crypto_drop_aead(spawn);
1199 out_free_inst:
1200         kfree(inst);
1201         goto out;
1202 }
1203
1204 static void crypto_rfc4543_free(struct crypto_instance *inst)
1205 {
1206         struct crypto_rfc4543_instance_ctx *ctx = crypto_instance_ctx(inst);
1207
1208         crypto_drop_aead(&ctx->aead);
1209
1210         kfree(aead_instance(inst));
1211 }
1212
1213 static struct crypto_template crypto_rfc4543_tmpl = {
1214         .name = "rfc4543",
1215         .create = crypto_rfc4543_create,
1216         .free = crypto_rfc4543_free,
1217         .module = THIS_MODULE,
1218 };
1219
1220 static int __init crypto_gcm_module_init(void)
1221 {
1222         int err;
1223
1224         gcm_zeroes = kzalloc(sizeof(*gcm_zeroes), GFP_KERNEL);
1225         if (!gcm_zeroes)
1226                 return -ENOMEM;
1227
1228         sg_init_one(&gcm_zeroes->sg, gcm_zeroes->buf, sizeof(gcm_zeroes->buf));
1229
1230         err = crypto_register_template(&crypto_gcm_base_tmpl);
1231         if (err)
1232                 goto out;
1233
1234         err = crypto_register_template(&crypto_gcm_tmpl);
1235         if (err)
1236                 goto out_undo_base;
1237
1238         err = crypto_register_template(&crypto_rfc4106_tmpl);
1239         if (err)
1240                 goto out_undo_gcm;
1241
1242         err = crypto_register_template(&crypto_rfc4543_tmpl);
1243         if (err)
1244                 goto out_undo_rfc4106;
1245
1246         return 0;
1247
1248 out_undo_rfc4106:
1249         crypto_unregister_template(&crypto_rfc4106_tmpl);
1250 out_undo_gcm:
1251         crypto_unregister_template(&crypto_gcm_tmpl);
1252 out_undo_base:
1253         crypto_unregister_template(&crypto_gcm_base_tmpl);
1254 out:
1255         kfree(gcm_zeroes);
1256         return err;
1257 }
1258
1259 static void __exit crypto_gcm_module_exit(void)
1260 {
1261         kfree(gcm_zeroes);
1262         crypto_unregister_template(&crypto_rfc4543_tmpl);
1263         crypto_unregister_template(&crypto_rfc4106_tmpl);
1264         crypto_unregister_template(&crypto_gcm_tmpl);
1265         crypto_unregister_template(&crypto_gcm_base_tmpl);
1266 }
1267
1268 module_init(crypto_gcm_module_init);
1269 module_exit(crypto_gcm_module_exit);
1270
1271 MODULE_LICENSE("GPL");
1272 MODULE_DESCRIPTION("Galois/Counter Mode");
1273 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1274 MODULE_ALIAS_CRYPTO("gcm_base");
1275 MODULE_ALIAS_CRYPTO("rfc4106");
1276 MODULE_ALIAS_CRYPTO("rfc4543");
1277 MODULE_ALIAS_CRYPTO("gcm");