cryptodev_main.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
   5  * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   6  *
   7  * This file is part of linux cryptodev.
   8  *
   9  * cryptodev is free software: you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation, either version 3 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * cryptodev is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21  */
  22
  23 /*
  24  * Device /dev/crypto provides an interface for
  25  * accessing kernel CryptoAPI algorithms (ciphers,
  26  * hashes) from userspace programs.
  27  *
  28  * /dev/crypto interface was originally introduced in
  29  * OpenBSD and this module attempts to keep the API.
  30  *
  31  */
  32
  33 #include <linux/crypto.h>
  34 #include <linux/mm.h>
  35 #include <linux/highmem.h>
  36 #include <linux/pagemap.h>
  37 #include <linux/random.h>
  38 #include "cryptodev.h"
  39 #include <asm/uaccess.h>
  40 #include <asm/ioctl.h>
  41 #include <linux/scatterlist.h>
  42 #include "cryptodev_int.h"
  43 #include "version.h"
  44
  45 MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
  46 MODULE_DESCRIPTION("CryptoDev driver");
  47 MODULE_LICENSE("GPL");
  48
  49 /* ====== Compile-time config ====== */
  50
  51 #define CRYPTODEV_STATS
  52
  53 /* ====== Module parameters ====== */
  54
  55 int cryptodev_verbosity = 0;
  56 module_param(cryptodev_verbosity, int, 0644);
  57 MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
  58
  59 #ifdef CRYPTODEV_STATS
  60 static int enable_stats = 0;
  61 module_param(enable_stats, int, 0644);
  62 MODULE_PARM_DESC(enable_stats, "collect statictics about cryptodev usage");
  63 #endif
  64
  65 /* ====== CryptoAPI ====== */
  66
  67 #define FILL_SG(sg,ptr,len)                                     \
  68         do {                                                    \
  69                 (sg)->page = virt_to_page(ptr);                 \
  70                 (sg)->offset = offset_in_page(ptr);             \
  71                 (sg)->length = len;                             \
  72                 (sg)->dma_address = 0;                          \
  73         } while (0)
  74
  75 struct csession {
  76         struct list_head entry;
  77         struct semaphore sem;
  78         struct cipher_data cdata;
  79         struct hash_data hdata;
  80         uint32_t sid;
  81 #ifdef CRYPTODEV_STATS
  82 #if ! ((COP_ENCRYPT < 2) && (COP_DECRYPT < 2))
  83 #error Struct csession.stat uses COP_{ENCRYPT,DECRYPT} as indices. Do something!
  84 #endif
  85         unsigned long long stat[2];
  86         size_t stat_max_size, stat_count;
  87 #endif
  88         int array_size;
  89         struct page **pages;
  90         struct scatterlist *sg;
  91 };
  92
  93 struct fcrypt {
  94         struct list_head list;
  95         struct semaphore sem;
  96 };
  97
  98 /* Prepare session for future use. */
  99 static int
 100 crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
 101 {
 102         struct csession *ses_new = NULL, *ses_ptr;
 103         int ret = 0;
 104         const char *alg_name=NULL;
 105         const char *hash_name=NULL;
 106         int hmac_mode = 1;
 107
 108         /* Does the request make sense? */
 109         if (unlikely(!sop->cipher && !sop->mac)) {
 110                 dprintk(1,KERN_DEBUG,"Both 'cipher' and 'mac' unset.\n");
 111                 return -EINVAL;
 112         }
 113
 114         switch (sop->cipher) {
 115                 case 0:
 116                         break;
 117                 case CRYPTO_DES_CBC:
 118                         alg_name = "cbc(des)";
 119                         break;
 120                 case CRYPTO_3DES_CBC:
 121                         alg_name = "cbc(des3_ede)";
 122                         break;
 123                 case CRYPTO_BLF_CBC:
 124                         alg_name = "cbc(blowfish)";
 125                         break;
 126                 case CRYPTO_AES_CBC:
 127                         alg_name = "cbc(aes)";
 128                         break;
 129                 case CRYPTO_CAMELLIA_CBC:
 130                         alg_name = "cbc(camelia)";
 131                         break;
 132                 case CRYPTO_NULL:
 133                         alg_name = "ecb(cipher_null)";
 134                         break;
 135                 default:
 136                         dprintk(1,KERN_DEBUG,"%s: bad cipher: %d\n", __func__, sop->cipher);
 137                         return -EINVAL;
 138         }
 139
 140         switch (sop->mac) {
 141                 case 0:
 142                         break;
 143                 case CRYPTO_MD5_HMAC:
 144                         hash_name = "hmac(md5)";
 145                         break;
 146                 case CRYPTO_RIPEMD160_HMAC:
 147                         hash_name = "hmac(rmd160)";
 148                         break;
 149                 case CRYPTO_SHA1_HMAC:
 150                         hash_name = "hmac(sha1)";
 151                         break;
 152                 case CRYPTO_SHA2_256_HMAC:
 153                         hash_name = "hmac(sha256)";
 154                         break;
 155                 case CRYPTO_SHA2_384_HMAC:
 156                         hash_name = "hmac(sha384)";
 157                         break;
 158                 case CRYPTO_SHA2_512_HMAC:
 159                         hash_name = "hmac(sha512)";
 160                         break;
 161
 162                 /* non-hmac cases */
 163                 case CRYPTO_MD5:
 164                         hash_name = "md5";
 165                         hmac_mode = 0;
 166                         break;
 167                 case CRYPTO_RIPEMD160:
 168                         hash_name = "rmd160";
 169                         hmac_mode = 0;
 170                         break;
 171                 case CRYPTO_SHA1:
 172                         hash_name = "sha1";
 173                         hmac_mode = 0;
 174                         break;
 175                 case CRYPTO_SHA2_256:
 176                         hash_name = "sha256";
 177                         hmac_mode = 0;
 178                         break;
 179                 case CRYPTO_SHA2_384:
 180                         hash_name = "sha384";
 181                         hmac_mode = 0;
 182                         break;
 183                 case CRYPTO_SHA2_512:
 184                         hash_name = "sha512";
 185                         hmac_mode = 0;
 186                         break;
 187
 188                 default:
 189                         dprintk(1,KERN_DEBUG,"%s: bad mac: %d\n", __func__, sop->mac);
 190                         return -EINVAL;
 191         }
 192
 193         /* Create a session and put it to the list. */
 194         ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
 195         if(!ses_new) {
 196                 return -ENOMEM;
 197         }
 198
 199         /* Set-up crypto transform. */
 200         if (alg_name) {
 201                 uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
 202
 203                 if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
 204                         dprintk(1,KERN_DEBUG,"Setting key failed for %s-%zu.\n",
 205                                 alg_name, (size_t)sop->keylen*8);
 206                         ret = -EINVAL;
 207                         goto error_cipher;
 208                 }
 209
 210                 ret = copy_from_user(keyp, sop->key, sop->keylen);
 211                 if (unlikely(ret)) {
 212                         goto error_cipher;
 213                 }
 214
 215                 ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp, sop->keylen);
 216
 217                 if (ret < 0) {
 218                         dprintk(1,KERN_DEBUG,"%s: Failed to load cipher for %s\n", __func__,
 219                                    alg_name);
 220                         ret = -EINVAL;
 221                         goto error_cipher;
 222                 }
 223         }
 224
 225         if (hash_name) {
 226                 uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
 227
 228                 if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
 229                         dprintk(1,KERN_DEBUG,"Setting key failed for %s-%zu.\n",
 230                                 alg_name, (size_t)sop->mackeylen*8);
 231                         ret = -EINVAL;
 232                         goto error_hash;
 233                 }
 234
 235                 ret = copy_from_user(keyp, sop->mackey, sop->mackeylen);
 236                 if (unlikely(ret)) {
 237                         goto error_hash;
 238                 }
 239
 240                 ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode, keyp, sop->mackeylen);
 241                 if (ret != 0) {
 242                         dprintk(1,KERN_DEBUG,"%s: Failed to load hash for %s\n", __func__,
 243                                    hash_name);
 244                         ret = -EINVAL;
 245                         goto error_hash;
 246                 }
 247         }
 248
 249         ses_new->array_size = 16;
 250         dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
 251                         __func__, ses_new->array_size);
 252         ses_new->pages = kzalloc(ses_new->array_size *
 253                         sizeof(struct page *), GFP_KERNEL);
 254         ses_new->sg = kzalloc(ses_new->array_size *
 255                         sizeof(struct scatterlist), GFP_KERNEL);
 256
 257         /* put the new session to the list */
 258         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 259         init_MUTEX(&ses_new->sem);
 260
 261         down(&fcr->sem);
 262 restart:
 263         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 264                 /* Check for duplicate SID */
 265                 if (unlikely(ses_new->sid == ses_ptr->sid)) {
 266                         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 267                         /* Unless we have a broken RNG this
 268                            shouldn't loop forever... ;-) */
 269                         goto restart;
 270                 }
 271         }
 272
 273         list_add(&ses_new->entry, &fcr->list);
 274         up(&fcr->sem);
 275
 276         /* Fill in some values for the user. */
 277         sop->ses = ses_new->sid;
 278
 279         return 0;
 280
 281 error_hash:
 282         cryptodev_cipher_deinit( &ses_new->cdata);
 283 error_cipher:
 284         if (ses_new) kfree(ses_new);
 285
 286         return ret;
 287
 288 }
 289
 290 /* Everything that needs to be done when remowing a session. */
 291 static inline void
 292 crypto_destroy_session(struct csession *ses_ptr)
 293 {
 294         if(down_trylock(&ses_ptr->sem)) {
 295                 dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
 296                         ses_ptr->sid);
 297                 down(&ses_ptr->sem);
 298         }
 299         dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
 300 #if defined(CRYPTODEV_STATS)
 301         if(enable_stats)
 302                 dprintk(2, KERN_DEBUG,
 303                         "Usage in Bytes: enc=%llu, dec=%llu, max=%zu, avg=%lu, cnt=%zu\n",
 304                         ses_ptr->stat[COP_ENCRYPT], ses_ptr->stat[COP_DECRYPT],
 305                         ses_ptr->stat_max_size, ses_ptr->stat_count > 0
 306                                 ? ((unsigned long)(ses_ptr->stat[COP_ENCRYPT]+
 307                                                    ses_ptr->stat[COP_DECRYPT]) /
 308                                    ses_ptr->stat_count) : 0,
 309                         ses_ptr->stat_count);
 310 #endif
 311         cryptodev_cipher_deinit(&ses_ptr->cdata);
 312         cryptodev_hash_deinit(&ses_ptr->hdata);
 313         dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
 314                         __func__, ses_ptr->array_size);
 315         kfree(ses_ptr->pages);
 316         kfree(ses_ptr->sg);
 317         up(&ses_ptr->sem);
 318         kfree(ses_ptr);
 319 }
 320
 321 /* Look up a session by ID and remove. */
 322 static int
 323 crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
 324 {
 325         struct csession *tmp, *ses_ptr;
 326         struct list_head *head;
 327         int ret = 0;
 328
 329         down(&fcr->sem);
 330         head = &fcr->list;
 331         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 332                 if(ses_ptr->sid == sid) {
 333                         list_del(&ses_ptr->entry);
 334                         crypto_destroy_session(ses_ptr);
 335                         break;
 336                 }
 337         }
 338
 339         if (unlikely(!ses_ptr)) {
 340                 dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n", sid);
 341                 ret = -ENOENT;
 342         }
 343         up(&fcr->sem);
 344
 345         return ret;
 346 }
 347
 348 /* Remove all sessions when closing the file */
 349 static int
 350 crypto_finish_all_sessions(struct fcrypt *fcr)
 351 {
 352         struct csession *tmp, *ses_ptr;
 353         struct list_head *head;
 354
 355         down(&fcr->sem);
 356
 357         head = &fcr->list;
 358         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 359                 list_del(&ses_ptr->entry);
 360                 crypto_destroy_session(ses_ptr);
 361         }
 362         up(&fcr->sem);
 363
 364         return 0;
 365 }
 366
 367 /* Look up session by session ID. The returned session is locked. */
 368 static struct csession *
 369 crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
 370 {
 371         struct csession *ses_ptr;
 372
 373         down(&fcr->sem);
 374         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 375                 if(ses_ptr->sid == sid) {
 376                         down(&ses_ptr->sem);
 377                         break;
 378                 }
 379         }
 380         up(&fcr->sem);
 381
 382         return ses_ptr;
 383 }
 384
 385 static int
 386 hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
 387                 struct scatterlist *src_sg, struct scatterlist *dst_sg, uint32_t len)
 388 {
 389         int ret;
 390
 391         /* Always hash before encryption and after decryption. Maybe
 392          * we should introduce a flag to switch... TBD later on.
 393          */
 394         if (cop->op == COP_ENCRYPT) {
 395                 if (ses_ptr->hdata.init != 0) {
 396                         ret = cryptodev_hash_update(&ses_ptr->hdata, src_sg, len);
 397                         if (unlikely(ret))
 398                                 goto out_err;
 399                 }
 400                 if (ses_ptr->cdata.init != 0) {
 401                         ret = cryptodev_cipher_encrypt( &ses_ptr->cdata, src_sg, dst_sg, len);
 402
 403                         if (unlikely(ret))
 404                                 goto out_err;
 405                 }
 406         } else {
 407                 if (ses_ptr->cdata.init != 0) {
 408                         ret = cryptodev_cipher_decrypt( &ses_ptr->cdata, src_sg, dst_sg, len);
 409
 410                         if (unlikely(ret))
 411                                 goto out_err;
 412                 }
 413
 414                 if (ses_ptr->hdata.init != 0) {
 415                         ret = cryptodev_hash_update(&ses_ptr->hdata, dst_sg, len);
 416                         if (unlikely(ret))
 417                                 goto out_err;
 418                 }
 419         }
 420         return 0;
 421 out_err:
 422         dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n",ret);
 423         return ret;
 424 }
 425
 426
 427 /* This is the main crypto function - feed it with plaintext
 428    and get a ciphertext (or vice versa :-) */
 429 static int
 430 __crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
 431 {
 432         char *data;
 433         char __user *src, __user *dst;
 434         struct scatterlist sg;
 435         size_t nbytes, bufsize;
 436         int ret = 0;
 437
 438         nbytes = cop->len;
 439         data = (char*)__get_free_page(GFP_KERNEL);
 440
 441         if (unlikely(!data)) {
 442                 return -ENOMEM;
 443         }
 444         bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
 445
 446         src = cop->src;
 447         dst = cop->dst;
 448
 449         while(nbytes > 0) {
 450                 size_t current_len = nbytes > bufsize ? bufsize : nbytes;
 451
 452                 ret = copy_from_user(data, src, current_len);
 453                 if (unlikely(ret))
 454                         break;
 455
 456                 sg_init_one(&sg, data, current_len);
 457
 458                 ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
 459
 460                 if (unlikely(ret))
 461                         break;
 462
 463                 if (ses_ptr->cdata.init != 0) {
 464                         ret = copy_to_user(dst, data, current_len);
 465                         if (unlikely(ret))
 466                                 break;
 467                 }
 468
 469                 dst += current_len;
 470                 nbytes -= current_len;
 471                 src += current_len;
 472         }
 473
 474         free_page((unsigned long)data);
 475         return ret;
 476 }
 477
 478 #ifndef DISABLE_ZCOPY
 479
 480 static void release_user_pages(struct page **pg, int pagecount)
 481 {
 482         while (pagecount--) {
 483                 if (!PageReserved(pg[pagecount]))
 484                         SetPageDirty(pg[pagecount]);
 485                 page_cache_release(pg[pagecount]);
 486         }
 487 }
 488
 489 /* last page - first page + 1 */
 490 #define PAGECOUNT(buf, buflen) \
 491         ((((unsigned long)(buf + buflen - 1) & PAGE_MASK) >> PAGE_SHIFT) - \
 492          (((unsigned long) buf               & PAGE_MASK) >> PAGE_SHIFT) + 1)
 493
 494 /* offset of buf in it's first page */
 495 #define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
 496
 497 #define MIN(a, b)       ((a > b) ? b : a)
 498
 499 /* fetch the pages addr resides in into pg and initialise sg with them */
 500 static int __get_userbuf(uint8_t *addr, uint32_t len,
 501                 int pgcount, struct page **pg, struct scatterlist *sg)
 502 {
 503         int ret, pglen, i = 0;
 504         struct scatterlist *sgp;
 505
 506         down_write(&current->mm->mmap_sem);
 507         ret = get_user_pages(current, current->mm,
 508                         (unsigned long)addr, pgcount, 1, 0, pg, NULL);
 509         up_write(&current->mm->mmap_sem);
 510         if (ret != pgcount)
 511                 return -EINVAL;
 512
 513         sg_init_table(sg, pgcount);
 514
 515         pglen = MIN(PAGE_SIZE - PAGEOFFSET(addr), len);
 516         sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
 517
 518         len -= pglen;
 519         for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
 520                 pglen = MIN(PAGE_SIZE, len);
 521                 sg_set_page(sgp, pg[i++], pglen, 0);
 522                 len -= pglen;
 523         }
 524         sg_mark_end(sg_last(sg, pgcount));
 525         return 0;
 526 }
 527
 528 /* make cop->src and cop->dst available in scatterlists */
 529 static int get_userbuf(struct csession *ses,
 530                 struct crypt_op *cop, struct scatterlist **src_sg,
 531                 struct scatterlist **dst_sg, int *tot_pages)
 532 {
 533         int src_pagecount, dst_pagecount = 0, pagecount;
 534
 535         src_pagecount = PAGECOUNT(cop->src, cop->len);
 536         if (cop->src != cop->dst) {     /* non-in-situ transformation */
 537                 dst_pagecount = PAGECOUNT(cop->dst, cop->len);
 538         }
 539         (*tot_pages) = pagecount = src_pagecount + dst_pagecount;
 540
 541         if (pagecount > ses->array_size) {
 542                 while (ses->array_size < pagecount)
 543                         ses->array_size *= 2;
 544
 545                 dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
 546                                 __func__, ses->array_size);
 547                 ses->pages = krealloc(ses->pages, ses->array_size *
 548                                 sizeof(struct page *), GFP_KERNEL);
 549                 ses->sg = krealloc(ses->sg, ses->array_size *
 550                                 sizeof(struct scatterlist), GFP_KERNEL);
 551
 552                 if (ses->sg == NULL || ses->pages == NULL) {
 553                         return -ENOMEM;
 554                 }
 555         }
 556
 557         if (__get_userbuf(cop->src, cop->len,
 558                         src_pagecount, ses->pages, ses->sg)) {
 559                 printk(KERN_ERR "%s: failed to get user pages\n", __func__);
 560                 return -EINVAL;
 561         }
 562         (*src_sg) = (*dst_sg) = ses->sg;
 563
 564         if (dst_pagecount) {
 565                 (*dst_sg) = ses->sg + src_pagecount;
 566
 567                 if (__get_userbuf(cop->dst, cop->len, dst_pagecount,
 568                                         ses->pages + src_pagecount, *dst_sg)) {
 569                         printk(KERN_ERR "%s: failed to get user pages\n", __func__);
 570                         release_user_pages(ses->pages, src_pagecount);
 571                         return -EINVAL;
 572                 }
 573         }
 574         return 0;
 575 }
 576
 577 /* This is the main crypto function - zero-copy edition */
 578 static int
 579 __crypto_run_zc(struct csession *ses_ptr, struct crypt_op *cop)
 580 {
 581         struct scatterlist *src_sg, *dst_sg;
 582         int ret = 0, pagecount;
 583
 584         ret = get_userbuf(ses_ptr, cop, &src_sg, &dst_sg, &pagecount);
 585         if (unlikely(ret)) {
 586                 dprintk(1, KERN_ERR, "Error getting user pages. Falling back to non zero copy.\n");
 587                 return __crypto_run_std(ses_ptr, cop);
 588         }
 589
 590         ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
 591
 592         release_user_pages(ses_ptr->pages, pagecount);
 593         return ret;
 594 }
 595
 596 #endif /* DISABLE_ZCOPY */
 597
 598 static int crypto_run(struct fcrypt *fcr, struct crypt_op *cop)
 599 {
 600         struct csession *ses_ptr;
 601         uint8_t hash_output[AALG_MAX_RESULT_LEN];
 602         int ret;
 603
 604         if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
 605                 dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", cop->op);
 606                 return -EINVAL;
 607         }
 608
 609         /* this also enters ses_ptr->sem */
 610         ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
 611         if (unlikely(!ses_ptr)) {
 612                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
 613                 return -EINVAL;
 614         }
 615
 616         if (ses_ptr->hdata.init != 0) {
 617                 ret = cryptodev_hash_reset(&ses_ptr->hdata);
 618                 if (unlikely(ret)) {
 619                         dprintk(1, KERN_ERR,
 620                                 "error in cryptodev_hash_reset()\n");
 621                         goto out_unlock;
 622                 }
 623         }
 624
 625         if (ses_ptr->cdata.init != 0) {
 626                 int blocksize = ses_ptr->cdata.blocksize;
 627
 628                 if (unlikely(cop->len % blocksize)) {
 629                         dprintk(1, KERN_ERR,
 630                                 "data size (%u) isn't a multiple of block size (%u)\n",
 631                                 cop->len, blocksize);
 632                         ret = -EINVAL;
 633                         goto out_unlock;
 634                 }
 635
 636                 if (cop->iv) {
 637                         uint8_t iv[EALG_MAX_BLOCK_LEN];
 638
 639                         ret = copy_from_user(iv, cop->iv, min( (int)sizeof(iv), (ses_ptr->cdata.ivsize)));
 640                         if (unlikely(ret)) {
 641                                 dprintk(1, KERN_ERR, "error copying IV (%d bytes)\n", min( (int)sizeof(iv), (ses_ptr->cdata.ivsize)));
 642                                 goto out_unlock;
 643                         }
 644
 645                         cryptodev_cipher_set_iv(&ses_ptr->cdata, iv, ses_ptr->cdata.ivsize);
 646                 }
 647         }
 648
 649 #ifdef DISABLE_ZCOPY
 650         ret = __crypto_run_std(ses_ptr, cop);
 651 #else /* normal */
 652         ret = __crypto_run_zc(ses_ptr, cop);
 653 #endif
 654         if (unlikely(ret))
 655                 goto out_unlock;
 656
 657         if (ses_ptr->hdata.init != 0) {
 658                 ret = cryptodev_hash_final(&ses_ptr->hdata, hash_output);
 659                 if (unlikely(ret)) {
 660                         dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n",ret);
 661                         goto out_unlock;
 662                 }
 663
 664                 if (unlikely(copy_to_user(cop->mac, hash_output, ses_ptr->hdata.digestsize)))
 665                         goto out_unlock;
 666         }
 667
 668 #if defined(CRYPTODEV_STATS)
 669         if (enable_stats) {
 670                 /* this is safe - we check cop->op at the function entry */
 671                 ses_ptr->stat[cop->op] += cop->len;
 672                 if (ses_ptr->stat_max_size < cop->len)
 673                         ses_ptr->stat_max_size = cop->len;
 674                 ses_ptr->stat_count++;
 675         }
 676 #endif
 677
 678 out_unlock:
 679         up(&ses_ptr->sem);
 680         return ret;
 681 }
 682
 683 /* ====== /dev/crypto ====== */
 684
 685 static int
 686 cryptodev_open(struct inode *inode, struct file *filp)
 687 {
 688         struct fcrypt *fcr;
 689
 690         fcr = kmalloc(sizeof(*fcr), GFP_KERNEL);
 691         if(!fcr)
 692                 return -ENOMEM;
 693
 694         memset(fcr, 0, sizeof(*fcr));
 695         init_MUTEX(&fcr->sem);
 696         INIT_LIST_HEAD(&fcr->list);
 697         filp->private_data = fcr;
 698
 699         return 0;
 700 }
 701
 702 static int
 703 cryptodev_release(struct inode *inode, struct file *filp)
 704 {
 705         struct fcrypt *fcr = filp->private_data;
 706
 707         if(fcr) {
 708                 crypto_finish_all_sessions(fcr);
 709                 kfree(fcr);
 710                 filp->private_data = NULL;
 711         }
 712
 713         return 0;
 714 }
 715
 716 static int
 717 clonefd(struct file *filp)
 718 {
 719         struct fdtable *fdt = files_fdtable(current->files);
 720         int ret;
 721         ret = get_unused_fd();
 722         if (ret >= 0) {
 723                         get_file(filp);
 724                         FD_SET(ret, fdt->open_fds);
 725                         fd_install(ret, filp);
 726         }
 727
 728         return ret;
 729 }
 730
 731 static int
 732 cryptodev_ioctl(struct inode *inode, struct file *filp,
 733                 unsigned int cmd, unsigned long arg)
 734 {
 735         int __user *p = (void __user *)arg;
 736         struct session_op sop;
 737         struct crypt_op cop;
 738         struct fcrypt *fcr = filp->private_data;
 739         uint32_t ses;
 740         int ret, fd;
 741
 742         if (unlikely(!fcr))
 743                 BUG();
 744
 745         switch (cmd) {
 746                 case CIOCASYMFEAT:
 747                         put_user(0, p);
 748                         return 0;
 749                 case CRIOGET:
 750                         fd = clonefd(filp);
 751                         put_user(fd, p);
 752                         return 0;
 753
 754                 case CIOCGSESSION:
 755                         ret = copy_from_user(&sop, (void*)arg, sizeof(sop));
 756                         if (unlikely(ret))
 757                                 return ret;
 758
 759                         ret = crypto_create_session(fcr, &sop);
 760                         if (unlikely(ret))
 761                                 return ret;
 762                         return copy_to_user((void*)arg, &sop, sizeof(sop));
 763
 764                 case CIOCFSESSION:
 765                         ret = get_user(ses, (uint32_t*)arg);
 766                         if (unlikely(ret))
 767                                 return ret;
 768
 769                         return crypto_finish_session(fcr, ses);
 770
 771                 case CIOCCRYPT:
 772                         ret = copy_from_user(&cop, (void*)arg, sizeof(cop));
 773                         if (unlikely(ret))
 774                                 return ret;
 775
 776                         ret = crypto_run(fcr, &cop);
 777                         if (unlikely(ret))
 778                                 return ret;
 779                         return copy_to_user((void*)arg, &cop, sizeof(cop));
 780
 781                 default:
 782                         return -EINVAL;
 783         }
 784 }
 785
 786 /* compatibility code for 32bit userlands */
 787 #ifdef CONFIG_COMPAT
 788
 789 static inline void
 790 compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
 791 {
 792         sop->cipher = compat->cipher;
 793         sop->mac = compat->mac;
 794         sop->keylen = compat->keylen;
 795
 796         sop->key       = compat_ptr(compat->key);
 797         sop->mackeylen = compat->mackeylen;
 798         sop->mackey    = compat_ptr(compat->mackey);
 799         sop->ses       = compat->ses;
 800 }
 801
 802 static inline void
 803 session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
 804 {
 805         compat->cipher = sop->cipher;
 806         compat->mac = sop->mac;
 807         compat->keylen = sop->keylen;
 808
 809         compat->key       = ptr_to_compat(sop->key);
 810         compat->mackeylen = sop->mackeylen;
 811         compat->mackey    = ptr_to_compat(sop->mackey);
 812         compat->ses       = sop->ses;
 813 }
 814
 815 static inline void
 816 compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
 817 {
 818         cop->ses = compat->ses;
 819         cop->op = compat->op;
 820         cop->flags = compat->flags;
 821         cop->len = compat->len;
 822
 823         cop->src = compat_ptr(compat->src);
 824         cop->dst = compat_ptr(compat->dst);
 825         cop->mac = compat_ptr(compat->mac);
 826         cop->iv  = compat_ptr(compat->iv);
 827 }
 828
 829 static inline void
 830 crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
 831 {
 832         compat->ses = cop->ses;
 833         compat->op = cop->op;
 834         compat->flags = cop->flags;
 835         compat->len = cop->len;
 836
 837         compat->src = ptr_to_compat(cop->src);
 838         compat->dst = ptr_to_compat(cop->dst);
 839         compat->mac = ptr_to_compat(cop->mac);
 840         compat->iv  = ptr_to_compat(cop->iv);
 841 }
 842
 843 static long
 844 cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
 845 {
 846         struct fcrypt *fcr = file->private_data;
 847         struct session_op sop;
 848         struct compat_session_op compat_sop;
 849         struct crypt_op cop;
 850         struct compat_crypt_op compat_cop;
 851         int ret;
 852
 853         if (unlikely(!fcr))
 854                 BUG();
 855
 856         switch (cmd) {
 857         case CIOCASYMFEAT:
 858         case CRIOGET:
 859         case CIOCFSESSION:
 860                 return cryptodev_ioctl(NULL, file, cmd, arg);
 861
 862         case COMPAT_CIOCGSESSION:
 863                 ret = copy_from_user(&compat_sop,
 864                                 (void *)arg, sizeof(compat_sop));
 865                 if (unlikely(ret))
 866                         return ret;
 867
 868                 compat_to_session_op(&compat_sop, &sop);
 869
 870                 ret = crypto_create_session(fcr, &sop);
 871                 if (unlikely(ret))
 872                         return ret;
 873
 874                 session_op_to_compat(&sop, &compat_sop);
 875                 return copy_to_user((void*)arg,
 876                                 &compat_sop, sizeof(compat_sop));
 877
 878         case COMPAT_CIOCCRYPT:
 879                 ret = copy_from_user(&compat_cop,
 880                                 (void*)arg, sizeof(compat_cop));
 881                 if (unlikely(ret))
 882                         return ret;
 883
 884                 compat_to_crypt_op(&compat_cop, &cop);
 885
 886                 ret = crypto_run(fcr, &cop);
 887                 if (unlikely(ret))
 888                         return ret;
 889
 890                 crypt_op_to_compat(&cop, &compat_cop);
 891                 return copy_to_user((void*)arg,
 892                                 &compat_cop, sizeof(compat_cop));
 893
 894         default:
 895                 return -EINVAL;
 896         }
 897 }
 898
 899 #endif /* CONFIG_COMPAT */
 900
 901 struct file_operations cryptodev_fops = {
 902         .owner = THIS_MODULE,
 903         .open = cryptodev_open,
 904         .release = cryptodev_release,
 905         .ioctl = cryptodev_ioctl,
 906 #ifdef CONFIG_COMPAT
 907         .compat_ioctl = cryptodev_compat_ioctl,
 908 #endif /* CONFIG_COMPAT */
 909 };
 910
 911 struct miscdevice cryptodev = {
 912         .minor = MISC_DYNAMIC_MINOR,
 913         .name = "crypto",
 914         .fops = &cryptodev_fops,
 915 };
 916
 917 static int
 918 cryptodev_register(void)
 919 {
 920         int rc;
 921
 922         rc = misc_register (&cryptodev);
 923         if (unlikely(rc)) {
 924                 printk(KERN_ERR PFX "registeration of /dev/crypto failed\n");
 925                 return rc;
 926         }
 927
 928         return 0;
 929 }
 930
 931 static void
 932 cryptodev_deregister(void)
 933 {
 934         misc_deregister(&cryptodev);
 935 }
 936
 937 /* ====== Module init/exit ====== */
 938
 939 int __init init_cryptodev(void)
 940 {
 941         int rc;
 942
 943         rc = cryptodev_register();
 944         if (unlikely(rc))
 945                 return rc;
 946
 947         printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
 948
 949         return 0;
 950 }
 951
 952 void __exit exit_cryptodev(void)
 953 {
 954         cryptodev_deregister();
 955         printk(KERN_INFO PFX "driver unloaded.\n");
 956 }
 957
 958 module_init(init_cryptodev);
 959 module_exit(exit_cryptodev);