crypto/algif_aead.c

   1 /*
   2  * algif_aead: User-space interface for AEAD algorithms
   3  *
   4  * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de>
   5  *
   6  * This file provides the user-space API for AEAD ciphers.
   7  *
   8  * This file is derived from algif_skcipher.c.
   9  *
  10  * This program is free software; you can redistribute it and/or modify it
  11  * under the terms of the GNU General Public License as published by the Free
  12  * Software Foundation; either version 2 of the License, or (at your option)
  13  * any later version.
  14  */
  15
  16 #include <crypto/internal/aead.h>
  17 #include <crypto/scatterwalk.h>
  18 #include <crypto/if_alg.h>
  19 #include <linux/init.h>
  20 #include <linux/list.h>
  21 #include <linux/kernel.h>
  22 #include <linux/sched/signal.h>
  23 #include <linux/mm.h>
  24 #include <linux/module.h>
  25 #include <linux/net.h>
  26 #include <net/sock.h>
  27
  28 struct aead_sg_list {
  29         unsigned int cur;
  30         struct scatterlist sg[ALG_MAX_PAGES];
  31 };
  32
  33 struct aead_async_rsgl {
  34         struct af_alg_sgl sgl;
  35         struct list_head list;
  36 };
  37
  38 struct aead_async_req {
  39         struct scatterlist *tsgl;
  40         struct aead_async_rsgl first_rsgl;
  41         struct list_head list;
  42         struct kiocb *iocb;
  43         struct sock *sk;
  44         unsigned int tsgls;
  45         char iv[];
  46 };
  47
  48 struct aead_tfm {
  49         struct crypto_aead *aead;
  50         bool has_key;
  51 };
  52
  53 struct aead_ctx {
  54         struct aead_sg_list tsgl;
  55         struct aead_async_rsgl first_rsgl;
  56         struct list_head list;
  57
  58         void *iv;
  59
  60         struct af_alg_completion completion;
  61
  62         unsigned long used;
  63
  64         unsigned int len;
  65         bool more;
  66         bool merge;
  67         bool enc;
  68
  69         size_t aead_assoclen;
  70         struct aead_request aead_req;
  71 };
  72
  73 static inline int aead_sndbuf(struct sock *sk)
  74 {
  75         struct alg_sock *ask = alg_sk(sk);
  76         struct aead_ctx *ctx = ask->private;
  77
  78         return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
  79                           ctx->used, 0);
  80 }
  81
  82 static inline bool aead_writable(struct sock *sk)
  83 {
  84         return PAGE_SIZE <= aead_sndbuf(sk);
  85 }
  86
  87 static inline bool aead_sufficient_data(struct aead_ctx *ctx)
  88 {
  89         unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
  90
  91         /*
  92          * The minimum amount of memory needed for an AEAD cipher is
  93          * the AAD and in case of decryption the tag.
  94          */
  95         return ctx->used >= ctx->aead_assoclen + (ctx->enc ? 0 : as);
  96 }
  97
  98 static void aead_reset_ctx(struct aead_ctx *ctx)
  99 {
 100         struct aead_sg_list *sgl = &ctx->tsgl;
 101
 102         sg_init_table(sgl->sg, ALG_MAX_PAGES);
 103         sgl->cur = 0;
 104         ctx->used = 0;
 105         ctx->more = 0;
 106         ctx->merge = 0;
 107 }
 108
 109 static void aead_put_sgl(struct sock *sk)
 110 {
 111         struct alg_sock *ask = alg_sk(sk);
 112         struct aead_ctx *ctx = ask->private;
 113         struct aead_sg_list *sgl = &ctx->tsgl;
 114         struct scatterlist *sg = sgl->sg;
 115         unsigned int i;
 116
 117         for (i = 0; i < sgl->cur; i++) {
 118                 if (!sg_page(sg + i))
 119                         continue;
 120
 121                 put_page(sg_page(sg + i));
 122                 sg_assign_page(sg + i, NULL);
 123         }
 124         aead_reset_ctx(ctx);
 125 }
 126
 127 static void aead_wmem_wakeup(struct sock *sk)
 128 {
 129         struct socket_wq *wq;
 130
 131         if (!aead_writable(sk))
 132                 return;
 133
 134         rcu_read_lock();
 135         wq = rcu_dereference(sk->sk_wq);
 136         if (skwq_has_sleeper(wq))
 137                 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
 138                                                            POLLRDNORM |
 139                                                            POLLRDBAND);
 140         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
 141         rcu_read_unlock();
 142 }
 143
 144 static int aead_wait_for_data(struct sock *sk, unsigned flags)
 145 {
 146         DEFINE_WAIT_FUNC(wait, woken_wake_function);
 147         struct alg_sock *ask = alg_sk(sk);
 148         struct aead_ctx *ctx = ask->private;
 149         long timeout;
 150         int err = -ERESTARTSYS;
 151
 152         if (flags & MSG_DONTWAIT)
 153                 return -EAGAIN;
 154
 155         sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 156         add_wait_queue(sk_sleep(sk), &wait);
 157         for (;;) {
 158                 if (signal_pending(current))
 159                         break;
 160                 timeout = MAX_SCHEDULE_TIMEOUT;
 161                 if (sk_wait_event(sk, &timeout, !ctx->more, &wait)) {
 162                         err = 0;
 163                         break;
 164                 }
 165         }
 166         remove_wait_queue(sk_sleep(sk), &wait);
 167
 168         sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 169
 170         return err;
 171 }
 172
 173 static void aead_data_wakeup(struct sock *sk)
 174 {
 175         struct alg_sock *ask = alg_sk(sk);
 176         struct aead_ctx *ctx = ask->private;
 177         struct socket_wq *wq;
 178
 179         if (ctx->more)
 180                 return;
 181         if (!ctx->used)
 182                 return;
 183
 184         rcu_read_lock();
 185         wq = rcu_dereference(sk->sk_wq);
 186         if (skwq_has_sleeper(wq))
 187                 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
 188                                                            POLLRDNORM |
 189                                                            POLLRDBAND);
 190         sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
 191         rcu_read_unlock();
 192 }
 193
 194 static int aead_sendmsg(struct socket *sock, struct msghdr *msg, size_t size)
 195 {
 196         struct sock *sk = sock->sk;
 197         struct alg_sock *ask = alg_sk(sk);
 198         struct aead_ctx *ctx = ask->private;
 199         unsigned ivsize =
 200                 crypto_aead_ivsize(crypto_aead_reqtfm(&ctx->aead_req));
 201         struct aead_sg_list *sgl = &ctx->tsgl;
 202         struct af_alg_control con = {};
 203         long copied = 0;
 204         bool enc = 0;
 205         bool init = 0;
 206         int err = -EINVAL;
 207
 208         if (msg->msg_controllen) {
 209                 err = af_alg_cmsg_send(msg, &con);
 210                 if (err)
 211                         return err;
 212
 213                 init = 1;
 214                 switch (con.op) {
 215                 case ALG_OP_ENCRYPT:
 216                         enc = 1;
 217                         break;
 218                 case ALG_OP_DECRYPT:
 219                         enc = 0;
 220                         break;
 221                 default:
 222                         return -EINVAL;
 223                 }
 224
 225                 if (con.iv && con.iv->ivlen != ivsize)
 226                         return -EINVAL;
 227         }
 228
 229         lock_sock(sk);
 230         if (!ctx->more && ctx->used)
 231                 goto unlock;
 232
 233         if (init) {
 234                 ctx->enc = enc;
 235                 if (con.iv)
 236                         memcpy(ctx->iv, con.iv->iv, ivsize);
 237
 238                 ctx->aead_assoclen = con.aead_assoclen;
 239         }
 240
 241         while (size) {
 242                 size_t len = size;
 243                 struct scatterlist *sg = NULL;
 244
 245                 /* use the existing memory in an allocated page */
 246                 if (ctx->merge) {
 247                         sg = sgl->sg + sgl->cur - 1;
 248                         len = min_t(unsigned long, len,
 249                                     PAGE_SIZE - sg->offset - sg->length);
 250                         err = memcpy_from_msg(page_address(sg_page(sg)) +
 251                                               sg->offset + sg->length,
 252                                               msg, len);
 253                         if (err)
 254                                 goto unlock;
 255
 256                         sg->length += len;
 257                         ctx->merge = (sg->offset + sg->length) &
 258                                      (PAGE_SIZE - 1);
 259
 260                         ctx->used += len;
 261                         copied += len;
 262                         size -= len;
 263                         continue;
 264                 }
 265
 266                 if (!aead_writable(sk)) {
 267                         /* user space sent too much data */
 268                         aead_put_sgl(sk);
 269                         err = -EMSGSIZE;
 270                         goto unlock;
 271                 }
 272
 273                 /* allocate a new page */
 274                 len = min_t(unsigned long, size, aead_sndbuf(sk));
 275                 while (len) {
 276                         size_t plen = 0;
 277
 278                         if (sgl->cur >= ALG_MAX_PAGES) {
 279                                 aead_put_sgl(sk);
 280                                 err = -E2BIG;
 281                                 goto unlock;
 282                         }
 283
 284                         sg = sgl->sg + sgl->cur;
 285                         plen = min_t(size_t, len, PAGE_SIZE);
 286
 287                         sg_assign_page(sg, alloc_page(GFP_KERNEL));
 288                         err = -ENOMEM;
 289                         if (!sg_page(sg))
 290                                 goto unlock;
 291
 292                         err = memcpy_from_msg(page_address(sg_page(sg)),
 293                                               msg, plen);
 294                         if (err) {
 295                                 __free_page(sg_page(sg));
 296                                 sg_assign_page(sg, NULL);
 297                                 goto unlock;
 298                         }
 299
 300                         sg->offset = 0;
 301                         sg->length = plen;
 302                         len -= plen;
 303                         ctx->used += plen;
 304                         copied += plen;
 305                         sgl->cur++;
 306                         size -= plen;
 307                         ctx->merge = plen & (PAGE_SIZE - 1);
 308                 }
 309         }
 310
 311         err = 0;
 312
 313         ctx->more = msg->msg_flags & MSG_MORE;
 314         if (!ctx->more && !aead_sufficient_data(ctx)) {
 315                 aead_put_sgl(sk);
 316                 err = -EMSGSIZE;
 317         }
 318
 319 unlock:
 320         aead_data_wakeup(sk);
 321         release_sock(sk);
 322
 323         return err ?: copied;
 324 }
 325
 326 static ssize_t aead_sendpage(struct socket *sock, struct page *page,
 327                              int offset, size_t size, int flags)
 328 {
 329         struct sock *sk = sock->sk;
 330         struct alg_sock *ask = alg_sk(sk);
 331         struct aead_ctx *ctx = ask->private;
 332         struct aead_sg_list *sgl = &ctx->tsgl;
 333         int err = -EINVAL;
 334
 335         if (flags & MSG_SENDPAGE_NOTLAST)
 336                 flags |= MSG_MORE;
 337
 338         if (sgl->cur >= ALG_MAX_PAGES)
 339                 return -E2BIG;
 340
 341         lock_sock(sk);
 342         if (!ctx->more && ctx->used)
 343                 goto unlock;
 344
 345         if (!size)
 346                 goto done;
 347
 348         if (!aead_writable(sk)) {
 349                 /* user space sent too much data */
 350                 aead_put_sgl(sk);
 351                 err = -EMSGSIZE;
 352                 goto unlock;
 353         }
 354
 355         ctx->merge = 0;
 356
 357         get_page(page);
 358         sg_set_page(sgl->sg + sgl->cur, page, size, offset);
 359         sgl->cur++;
 360         ctx->used += size;
 361
 362         err = 0;
 363
 364 done:
 365         ctx->more = flags & MSG_MORE;
 366         if (!ctx->more && !aead_sufficient_data(ctx)) {
 367                 aead_put_sgl(sk);
 368                 err = -EMSGSIZE;
 369         }
 370
 371 unlock:
 372         aead_data_wakeup(sk);
 373         release_sock(sk);
 374
 375         return err ?: size;
 376 }
 377
 378 #define GET_ASYM_REQ(req, tfm) (struct aead_async_req *) \
 379                 ((char *)req + sizeof(struct aead_request) + \
 380                  crypto_aead_reqsize(tfm))
 381
 382  #define GET_REQ_SIZE(tfm) sizeof(struct aead_async_req) + \
 383         crypto_aead_reqsize(tfm) + crypto_aead_ivsize(tfm) + \
 384         sizeof(struct aead_request)
 385
 386 static void aead_async_cb(struct crypto_async_request *_req, int err)
 387 {
 388         struct aead_request *req = _req->data;
 389         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 390         struct aead_async_req *areq = GET_ASYM_REQ(req, tfm);
 391         struct sock *sk = areq->sk;
 392         struct scatterlist *sg = areq->tsgl;
 393         struct aead_async_rsgl *rsgl;
 394         struct kiocb *iocb = areq->iocb;
 395         unsigned int i, reqlen = GET_REQ_SIZE(tfm);
 396
 397         list_for_each_entry(rsgl, &areq->list, list) {
 398                 af_alg_free_sg(&rsgl->sgl);
 399                 if (rsgl != &areq->first_rsgl)
 400                         sock_kfree_s(sk, rsgl, sizeof(*rsgl));
 401         }
 402
 403         for (i = 0; i < areq->tsgls; i++)
 404                 put_page(sg_page(sg + i));
 405
 406         sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
 407         sock_kfree_s(sk, req, reqlen);
 408         __sock_put(sk);
 409         iocb->ki_complete(iocb, err, err);
 410 }
 411
 412 static int aead_recvmsg_async(struct socket *sock, struct msghdr *msg,
 413                               int flags)
 414 {
 415         struct sock *sk = sock->sk;
 416         struct alg_sock *ask = alg_sk(sk);
 417         struct aead_ctx *ctx = ask->private;
 418         struct crypto_aead *tfm = crypto_aead_reqtfm(&ctx->aead_req);
 419         struct aead_async_req *areq;
 420         struct aead_request *req = NULL;
 421         struct aead_sg_list *sgl = &ctx->tsgl;
 422         struct aead_async_rsgl *last_rsgl = NULL, *rsgl;
 423         unsigned int as = crypto_aead_authsize(tfm);
 424         unsigned int i, reqlen = GET_REQ_SIZE(tfm);
 425         int err = -ENOMEM;
 426         unsigned long used;
 427         size_t outlen = 0;
 428         size_t usedpages = 0;
 429
 430         lock_sock(sk);
 431         if (ctx->more) {
 432                 err = aead_wait_for_data(sk, flags);
 433                 if (err)
 434                         goto unlock;
 435         }
 436
 437         if (!aead_sufficient_data(ctx))
 438                 goto unlock;
 439
 440         used = ctx->used;
 441         if (ctx->enc)
 442                 outlen = used + as;
 443         else
 444                 outlen = used - as;
 445
 446         req = sock_kmalloc(sk, reqlen, GFP_KERNEL);
 447         if (unlikely(!req))
 448                 goto unlock;
 449
 450         areq = GET_ASYM_REQ(req, tfm);
 451         memset(&areq->first_rsgl, '\0', sizeof(areq->first_rsgl));
 452         INIT_LIST_HEAD(&areq->list);
 453         areq->iocb = msg->msg_iocb;
 454         areq->sk = sk;
 455         memcpy(areq->iv, ctx->iv, crypto_aead_ivsize(tfm));
 456         aead_request_set_tfm(req, tfm);
 457         aead_request_set_ad(req, ctx->aead_assoclen);
 458         aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 459                                   aead_async_cb, req);
 460         used -= ctx->aead_assoclen;
 461
 462         /* take over all tx sgls from ctx */
 463         areq->tsgl = sock_kmalloc(sk,
 464                                   sizeof(*areq->tsgl) * max_t(u32, sgl->cur, 1),
 465                                   GFP_KERNEL);
 466         if (unlikely(!areq->tsgl))
 467                 goto free;
 468
 469         sg_init_table(areq->tsgl, max_t(u32, sgl->cur, 1));
 470         for (i = 0; i < sgl->cur; i++)
 471                 sg_set_page(&areq->tsgl[i], sg_page(&sgl->sg[i]),
 472                             sgl->sg[i].length, sgl->sg[i].offset);
 473
 474         areq->tsgls = sgl->cur;
 475
 476         /* create rx sgls */
 477         while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
 478                 size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
 479                                       (outlen - usedpages));
 480
 481                 if (list_empty(&areq->list)) {
 482                         rsgl = &areq->first_rsgl;
 483
 484                 } else {
 485                         rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
 486                         if (unlikely(!rsgl)) {
 487                                 err = -ENOMEM;
 488                                 goto free;
 489                         }
 490                 }
 491                 rsgl->sgl.npages = 0;
 492                 list_add_tail(&rsgl->list, &areq->list);
 493
 494                 /* make one iovec available as scatterlist */
 495                 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
 496                 if (err < 0)
 497                         goto free;
 498
 499                 usedpages += err;
 500
 501                 /* chain the new scatterlist with previous one */
 502                 if (last_rsgl)
 503                         af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
 504
 505                 last_rsgl = rsgl;
 506
 507                 iov_iter_advance(&msg->msg_iter, err);
 508         }
 509
 510         /* ensure output buffer is sufficiently large */
 511         if (usedpages < outlen) {
 512                 err = -EINVAL;
 513                 goto unlock;
 514         }
 515
 516         aead_request_set_crypt(req, areq->tsgl, areq->first_rsgl.sgl.sg, used,
 517                                areq->iv);
 518         err = ctx->enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req);
 519         if (err) {
 520                 if (err == -EINPROGRESS) {
 521                         sock_hold(sk);
 522                         err = -EIOCBQUEUED;
 523                         aead_reset_ctx(ctx);
 524                         goto unlock;
 525                 } else if (err == -EBADMSG) {
 526                         aead_put_sgl(sk);
 527                 }
 528                 goto free;
 529         }
 530         aead_put_sgl(sk);
 531
 532 free:
 533         list_for_each_entry(rsgl, &areq->list, list) {
 534                 af_alg_free_sg(&rsgl->sgl);
 535                 if (rsgl != &areq->first_rsgl)
 536                         sock_kfree_s(sk, rsgl, sizeof(*rsgl));
 537         }
 538         if (areq->tsgl)
 539                 sock_kfree_s(sk, areq->tsgl, sizeof(*areq->tsgl) * areq->tsgls);
 540         if (req)
 541                 sock_kfree_s(sk, req, reqlen);
 542 unlock:
 543         aead_wmem_wakeup(sk);
 544         release_sock(sk);
 545         return err ? err : outlen;
 546 }
 547
 548 static int aead_recvmsg_sync(struct socket *sock, struct msghdr *msg, int flags)
 549 {
 550         struct sock *sk = sock->sk;
 551         struct alg_sock *ask = alg_sk(sk);
 552         struct aead_ctx *ctx = ask->private;
 553         unsigned as = crypto_aead_authsize(crypto_aead_reqtfm(&ctx->aead_req));
 554         struct aead_sg_list *sgl = &ctx->tsgl;
 555         struct aead_async_rsgl *last_rsgl = NULL;
 556         struct aead_async_rsgl *rsgl, *tmp;
 557         int err = -EINVAL;
 558         unsigned long used = 0;
 559         size_t outlen = 0;
 560         size_t usedpages = 0;
 561
 562         lock_sock(sk);
 563
 564         /*
 565          * Please see documentation of aead_request_set_crypt for the
 566          * description of the AEAD memory structure expected from the caller.
 567          */
 568
 569         if (ctx->more) {
 570                 err = aead_wait_for_data(sk, flags);
 571                 if (err)
 572                         goto unlock;
 573         }
 574
 575         /* data length provided by caller via sendmsg/sendpage */
 576         used = ctx->used;
 577
 578         /*
 579          * Make sure sufficient data is present -- note, the same check is
 580          * is also present in sendmsg/sendpage. The checks in sendpage/sendmsg
 581          * shall provide an information to the data sender that something is
 582          * wrong, but they are irrelevant to maintain the kernel integrity.
 583          * We need this check here too in case user space decides to not honor
 584          * the error message in sendmsg/sendpage and still call recvmsg. This
 585          * check here protects the kernel integrity.
 586          */
 587         if (!aead_sufficient_data(ctx))
 588                 goto unlock;
 589
 590         /*
 591          * Calculate the minimum output buffer size holding the result of the
 592          * cipher operation. When encrypting data, the receiving buffer is
 593          * larger by the tag length compared to the input buffer as the
 594          * encryption operation generates the tag. For decryption, the input
 595          * buffer provides the tag which is consumed resulting in only the
 596          * plaintext without a buffer for the tag returned to the caller.
 597          */
 598         if (ctx->enc)
 599                 outlen = used + as;
 600         else
 601                 outlen = used - as;
 602
 603         /*
 604          * The cipher operation input data is reduced by the associated data
 605          * length as this data is processed separately later on.
 606          */
 607         used -= ctx->aead_assoclen;
 608
 609         /* convert iovecs of output buffers into scatterlists */
 610         while (outlen > usedpages && iov_iter_count(&msg->msg_iter)) {
 611                 size_t seglen = min_t(size_t, iov_iter_count(&msg->msg_iter),
 612                                       (outlen - usedpages));
 613
 614                 if (list_empty(&ctx->list)) {
 615                         rsgl = &ctx->first_rsgl;
 616                 } else {
 617                         rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL);
 618                         if (unlikely(!rsgl)) {
 619                                 err = -ENOMEM;
 620                                 goto unlock;
 621                         }
 622                 }
 623                 rsgl->sgl.npages = 0;
 624                 list_add_tail(&rsgl->list, &ctx->list);
 625
 626                 /* make one iovec available as scatterlist */
 627                 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen);
 628                 if (err < 0)
 629                         goto unlock;
 630                 usedpages += err;
 631                 /* chain the new scatterlist with previous one */
 632                 if (last_rsgl)
 633                         af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
 634
 635                 last_rsgl = rsgl;
 636
 637                 iov_iter_advance(&msg->msg_iter, err);
 638         }
 639
 640         /* ensure output buffer is sufficiently large */
 641         if (usedpages < outlen) {
 642                 err = -EINVAL;
 643                 goto unlock;
 644         }
 645
 646         sg_mark_end(sgl->sg + sgl->cur - 1);
 647         aead_request_set_crypt(&ctx->aead_req, sgl->sg, ctx->first_rsgl.sgl.sg,
 648                                used, ctx->iv);
 649         aead_request_set_ad(&ctx->aead_req, ctx->aead_assoclen);
 650
 651         err = af_alg_wait_for_completion(ctx->enc ?
 652                                          crypto_aead_encrypt(&ctx->aead_req) :
 653                                          crypto_aead_decrypt(&ctx->aead_req),
 654                                          &ctx->completion);
 655
 656         if (err) {
 657                 /* EBADMSG implies a valid cipher operation took place */
 658                 if (err == -EBADMSG)
 659                         aead_put_sgl(sk);
 660
 661                 goto unlock;
 662         }
 663
 664         aead_put_sgl(sk);
 665         err = 0;
 666
 667 unlock:
 668         list_for_each_entry_safe(rsgl, tmp, &ctx->list, list) {
 669                 af_alg_free_sg(&rsgl->sgl);
 670                 list_del(&rsgl->list);
 671                 if (rsgl != &ctx->first_rsgl)
 672                         sock_kfree_s(sk, rsgl, sizeof(*rsgl));
 673         }
 674         INIT_LIST_HEAD(&ctx->list);
 675         aead_wmem_wakeup(sk);
 676         release_sock(sk);
 677
 678         return err ? err : outlen;
 679 }
 680
 681 static int aead_recvmsg(struct socket *sock, struct msghdr *msg, size_t ignored,
 682                         int flags)
 683 {
 684         return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
 685                 aead_recvmsg_async(sock, msg, flags) :
 686                 aead_recvmsg_sync(sock, msg, flags);
 687 }
 688
 689 static unsigned int aead_poll(struct file *file, struct socket *sock,
 690                               poll_table *wait)
 691 {
 692         struct sock *sk = sock->sk;
 693         struct alg_sock *ask = alg_sk(sk);
 694         struct aead_ctx *ctx = ask->private;
 695         unsigned int mask;
 696
 697         sock_poll_wait(file, sk_sleep(sk), wait);
 698         mask = 0;
 699
 700         if (!ctx->more)
 701                 mask |= POLLIN | POLLRDNORM;
 702
 703         if (aead_writable(sk))
 704                 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
 705
 706         return mask;
 707 }
 708
 709 static struct proto_ops algif_aead_ops = {
 710         .family         =       PF_ALG,
 711
 712         .connect        =       sock_no_connect,
 713         .socketpair     =       sock_no_socketpair,
 714         .getname        =       sock_no_getname,
 715         .ioctl          =       sock_no_ioctl,
 716         .listen         =       sock_no_listen,
 717         .shutdown       =       sock_no_shutdown,
 718         .getsockopt     =       sock_no_getsockopt,
 719         .mmap           =       sock_no_mmap,
 720         .bind           =       sock_no_bind,
 721         .accept         =       sock_no_accept,
 722         .setsockopt     =       sock_no_setsockopt,
 723
 724         .release        =       af_alg_release,
 725         .sendmsg        =       aead_sendmsg,
 726         .sendpage       =       aead_sendpage,
 727         .recvmsg        =       aead_recvmsg,
 728         .poll           =       aead_poll,
 729 };
 730
 731 static int aead_check_key(struct socket *sock)
 732 {
 733         int err = 0;
 734         struct sock *psk;
 735         struct alg_sock *pask;
 736         struct aead_tfm *tfm;
 737         struct sock *sk = sock->sk;
 738         struct alg_sock *ask = alg_sk(sk);
 739
 740         lock_sock(sk);
 741         if (ask->refcnt)
 742                 goto unlock_child;
 743
 744         psk = ask->parent;
 745         pask = alg_sk(ask->parent);
 746         tfm = pask->private;
 747
 748         err = -ENOKEY;
 749         lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
 750         if (!tfm->has_key)
 751                 goto unlock;
 752
 753         if (!pask->refcnt++)
 754                 sock_hold(psk);
 755
 756         ask->refcnt = 1;
 757         sock_put(psk);
 758
 759         err = 0;
 760
 761 unlock:
 762         release_sock(psk);
 763 unlock_child:
 764         release_sock(sk);
 765
 766         return err;
 767 }
 768
 769 static int aead_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
 770                                   size_t size)
 771 {
 772         int err;
 773
 774         err = aead_check_key(sock);
 775         if (err)
 776                 return err;
 777
 778         return aead_sendmsg(sock, msg, size);
 779 }
 780
 781 static ssize_t aead_sendpage_nokey(struct socket *sock, struct page *page,
 782                                        int offset, size_t size, int flags)
 783 {
 784         int err;
 785
 786         err = aead_check_key(sock);
 787         if (err)
 788                 return err;
 789
 790         return aead_sendpage(sock, page, offset, size, flags);
 791 }
 792
 793 static int aead_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
 794                                   size_t ignored, int flags)
 795 {
 796         int err;
 797
 798         err = aead_check_key(sock);
 799         if (err)
 800                 return err;
 801
 802         return aead_recvmsg(sock, msg, ignored, flags);
 803 }
 804
 805 static struct proto_ops algif_aead_ops_nokey = {
 806         .family         =       PF_ALG,
 807
 808         .connect        =       sock_no_connect,
 809         .socketpair     =       sock_no_socketpair,
 810         .getname        =       sock_no_getname,
 811         .ioctl          =       sock_no_ioctl,
 812         .listen         =       sock_no_listen,
 813         .shutdown       =       sock_no_shutdown,
 814         .getsockopt     =       sock_no_getsockopt,
 815         .mmap           =       sock_no_mmap,
 816         .bind           =       sock_no_bind,
 817         .accept         =       sock_no_accept,
 818         .setsockopt     =       sock_no_setsockopt,
 819
 820         .release        =       af_alg_release,
 821         .sendmsg        =       aead_sendmsg_nokey,
 822         .sendpage       =       aead_sendpage_nokey,
 823         .recvmsg        =       aead_recvmsg_nokey,
 824         .poll           =       aead_poll,
 825 };
 826
 827 static void *aead_bind(const char *name, u32 type, u32 mask)
 828 {
 829         struct aead_tfm *tfm;
 830         struct crypto_aead *aead;
 831
 832         tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
 833         if (!tfm)
 834                 return ERR_PTR(-ENOMEM);
 835
 836         aead = crypto_alloc_aead(name, type, mask);
 837         if (IS_ERR(aead)) {
 838                 kfree(tfm);
 839                 return ERR_CAST(aead);
 840         }
 841
 842         tfm->aead = aead;
 843
 844         return tfm;
 845 }
 846
 847 static void aead_release(void *private)
 848 {
 849         struct aead_tfm *tfm = private;
 850
 851         crypto_free_aead(tfm->aead);
 852         kfree(tfm);
 853 }
 854
 855 static int aead_setauthsize(void *private, unsigned int authsize)
 856 {
 857         struct aead_tfm *tfm = private;
 858
 859         return crypto_aead_setauthsize(tfm->aead, authsize);
 860 }
 861
 862 static int aead_setkey(void *private, const u8 *key, unsigned int keylen)
 863 {
 864         struct aead_tfm *tfm = private;
 865         int err;
 866
 867         err = crypto_aead_setkey(tfm->aead, key, keylen);
 868         tfm->has_key = !err;
 869
 870         return err;
 871 }
 872
 873 static void aead_sock_destruct(struct sock *sk)
 874 {
 875         struct alg_sock *ask = alg_sk(sk);
 876         struct aead_ctx *ctx = ask->private;
 877         unsigned int ivlen = crypto_aead_ivsize(
 878                                 crypto_aead_reqtfm(&ctx->aead_req));
 879
 880         WARN_ON(atomic_read(&sk->sk_refcnt) != 0);
 881         aead_put_sgl(sk);
 882         sock_kzfree_s(sk, ctx->iv, ivlen);
 883         sock_kfree_s(sk, ctx, ctx->len);
 884         af_alg_release_parent(sk);
 885 }
 886
 887 static int aead_accept_parent_nokey(void *private, struct sock *sk)
 888 {
 889         struct aead_ctx *ctx;
 890         struct alg_sock *ask = alg_sk(sk);
 891         struct aead_tfm *tfm = private;
 892         struct crypto_aead *aead = tfm->aead;
 893         unsigned int len = sizeof(*ctx) + crypto_aead_reqsize(aead);
 894         unsigned int ivlen = crypto_aead_ivsize(aead);
 895
 896         ctx = sock_kmalloc(sk, len, GFP_KERNEL);
 897         if (!ctx)
 898                 return -ENOMEM;
 899         memset(ctx, 0, len);
 900
 901         ctx->iv = sock_kmalloc(sk, ivlen, GFP_KERNEL);
 902         if (!ctx->iv) {
 903                 sock_kfree_s(sk, ctx, len);
 904                 return -ENOMEM;
 905         }
 906         memset(ctx->iv, 0, ivlen);
 907
 908         ctx->len = len;
 909         ctx->used = 0;
 910         ctx->more = 0;
 911         ctx->merge = 0;
 912         ctx->enc = 0;
 913         ctx->tsgl.cur = 0;
 914         ctx->aead_assoclen = 0;
 915         af_alg_init_completion(&ctx->completion);
 916         sg_init_table(ctx->tsgl.sg, ALG_MAX_PAGES);
 917         INIT_LIST_HEAD(&ctx->list);
 918
 919         ask->private = ctx;
 920
 921         aead_request_set_tfm(&ctx->aead_req, aead);
 922         aead_request_set_callback(&ctx->aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 923                                   af_alg_complete, &ctx->completion);
 924
 925         sk->sk_destruct = aead_sock_destruct;
 926
 927         return 0;
 928 }
 929
 930 static int aead_accept_parent(void *private, struct sock *sk)
 931 {
 932         struct aead_tfm *tfm = private;
 933
 934         if (!tfm->has_key)
 935                 return -ENOKEY;
 936
 937         return aead_accept_parent_nokey(private, sk);
 938 }
 939
 940 static const struct af_alg_type algif_type_aead = {
 941         .bind           =       aead_bind,
 942         .release        =       aead_release,
 943         .setkey         =       aead_setkey,
 944         .setauthsize    =       aead_setauthsize,
 945         .accept         =       aead_accept_parent,
 946         .accept_nokey   =       aead_accept_parent_nokey,
 947         .ops            =       &algif_aead_ops,
 948         .ops_nokey      =       &algif_aead_ops_nokey,
 949         .name           =       "aead",
 950         .owner          =       THIS_MODULE
 951 };
 952
 953 static int __init algif_aead_init(void)
 954 {
 955         return af_alg_register_type(&algif_type_aead);
 956 }
 957
 958 static void __exit algif_aead_exit(void)
 959 {
 960         int err = af_alg_unregister_type(&algif_type_aead);
 961         BUG_ON(err);
 962 }
 963
 964 module_init(algif_aead_init);
 965 module_exit(algif_aead_exit);
 966 MODULE_LICENSE("GPL");
 967 MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
 968 MODULE_DESCRIPTION("AEAD kernel crypto API user space interface");