ioctl.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
   5  * Copyright (c) 2009,2010,2011 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   6  * Copyright (c) 2010 Phil Sutter
   7  *
   8  * This file is part of linux cryptodev.
   9  *
  10  * This program is free software; you can redistribute it and/or
  11  * modify it under the terms of the GNU General Public License
  12  * as published by the Free Software Foundation; either version 2
  13  * of the License, or (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18  * GNU General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU General Public License
  21  * along with this program; if not, write to the Free Software
  22  * Foundation, Inc.,
  23  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  24  */
  25
  26 /*
  27  * Device /dev/crypto provides an interface for
  28  * accessing kernel CryptoAPI algorithms (ciphers,
  29  * hashes) from userspace programs.
  30  *
  31  * /dev/crypto interface was originally introduced in
  32  * OpenBSD and this module attempts to keep the API.
  33  *
  34  */
  35
  36 #include <crypto/hash.h>
  37 #include <linux/crypto.h>
  38 #include <linux/mm.h>
  39 #include <linux/highmem.h>
  40 #include <linux/ioctl.h>
  41 #include <linux/random.h>
  42 #include <linux/syscalls.h>
  43 #include <linux/pagemap.h>
  44 #include <linux/poll.h>
  45 #include <linux/uaccess.h>
  46 #include <crypto/cryptodev.h>
  47 #include <linux/scatterlist.h>
  48 #include "cryptodev_int.h"
  49 #include "zc.h"
  50 #include "version.h"
  51
  52 MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
  53 MODULE_DESCRIPTION("CryptoDev driver");
  54 MODULE_LICENSE("GPL");
  55
  56 /* ====== Compile-time config ====== */
  57
  58 /* Default (pre-allocated) and maximum size of the job queue.
  59  * These are free, pending and done items all together. */
  60 #define DEF_COP_RINGSIZE 16
  61 #define MAX_COP_RINGSIZE 64
  62
  63 /* ====== Module parameters ====== */
  64
  65 int cryptodev_verbosity;
  66 module_param(cryptodev_verbosity, int, 0644);
  67 MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
  68
  69 /* ====== CryptoAPI ====== */
  70 struct todo_list_item {
  71         struct list_head __hook;
  72         struct kernel_crypt_op kcop;
  73         int result;
  74 };
  75
  76 struct locked_list {
  77         struct list_head list;
  78         struct mutex lock;
  79 };
  80
  81 struct crypt_priv {
  82         struct fcrypt fcrypt;
  83         struct locked_list free, todo, done;
  84         int itemcount;
  85         struct work_struct cryptask;
  86         wait_queue_head_t user_waiter;
  87 };
  88
  89 #define FILL_SG(sg, ptr, len)                                   \
  90         do {                                                    \
  91                 (sg)->page = virt_to_page(ptr);                 \
  92                 (sg)->offset = offset_in_page(ptr);             \
  93                 (sg)->length = len;                             \
  94                 (sg)->dma_address = 0;                          \
  95         } while (0)
  96
  97 /* cryptodev's own workqueue, keeps crypto tasks from disturbing the force */
  98 static struct workqueue_struct *cryptodev_wq;
  99
 100 /* Prepare session for future use. */
 101 static int
 102 crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
 103 {
 104         struct csession *ses_new = NULL, *ses_ptr;
 105         int ret = 0;
 106         const char *alg_name = NULL;
 107         const char *hash_name = NULL;
 108         int hmac_mode = 1, stream = 0, aead = 0;
 109
 110         /* Does the request make sense? */
 111         if (unlikely(!sop->cipher && !sop->mac)) {
 112                 dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
 113                 return -EINVAL;
 114         }
 115
 116         switch (sop->cipher) {
 117         case 0:
 118                 break;
 119         case CRYPTO_DES_CBC:
 120                 alg_name = "cbc(des)";
 121                 break;
 122         case CRYPTO_3DES_CBC:
 123                 alg_name = "cbc(des3_ede)";
 124                 break;
 125         case CRYPTO_BLF_CBC:
 126                 alg_name = "cbc(blowfish)";
 127                 break;
 128         case CRYPTO_AES_CBC:
 129                 alg_name = "cbc(aes)";
 130                 break;
 131         case CRYPTO_AES_ECB:
 132                 alg_name = "ecb(aes)";
 133                 break;
 134         case CRYPTO_CAMELLIA_CBC:
 135                 alg_name = "cbc(camellia)";
 136                 break;
 137         case CRYPTO_AES_CTR:
 138                 alg_name = "ctr(aes)";
 139                 stream = 1;
 140                 break;
 141         case CRYPTO_AES_GCM:
 142                 alg_name = "gcm(aes)";
 143                 stream = 1;
 144                 aead = 1;
 145                 break;
 146         case CRYPTO_NULL:
 147                 alg_name = "ecb(cipher_null)";
 148                 stream = 1;
 149                 break;
 150         default:
 151                 dprintk(1, KERN_DEBUG, "bad cipher: %d\n", sop->cipher);
 152                 return -EINVAL;
 153         }
 154
 155         switch (sop->mac) {
 156         case 0:
 157                 break;
 158         case CRYPTO_MD5_HMAC:
 159                 hash_name = "hmac(md5)";
 160                 break;
 161         case CRYPTO_RIPEMD160_HMAC:
 162                 hash_name = "hmac(rmd160)";
 163                 break;
 164         case CRYPTO_SHA1_HMAC:
 165                 hash_name = "hmac(sha1)";
 166                 break;
 167         case CRYPTO_SHA2_224_HMAC:
 168                 hash_name = "hmac(sha224)";
 169                 break;
 170
 171         case CRYPTO_SHA2_256_HMAC:
 172                 hash_name = "hmac(sha256)";
 173                 break;
 174         case CRYPTO_SHA2_384_HMAC:
 175                 hash_name = "hmac(sha384)";
 176                 break;
 177         case CRYPTO_SHA2_512_HMAC:
 178                 hash_name = "hmac(sha512)";
 179                 break;
 180
 181         /* non-hmac cases */
 182         case CRYPTO_MD5:
 183                 hash_name = "md5";
 184                 hmac_mode = 0;
 185                 break;
 186         case CRYPTO_RIPEMD160:
 187                 hash_name = "rmd160";
 188                 hmac_mode = 0;
 189                 break;
 190         case CRYPTO_SHA1:
 191                 hash_name = "sha1";
 192                 hmac_mode = 0;
 193                 break;
 194         case CRYPTO_SHA2_224:
 195                 hash_name = "sha224";
 196                 hmac_mode = 0;
 197                 break;
 198         case CRYPTO_SHA2_256:
 199                 hash_name = "sha256";
 200                 hmac_mode = 0;
 201                 break;
 202         case CRYPTO_SHA2_384:
 203                 hash_name = "sha384";
 204                 hmac_mode = 0;
 205                 break;
 206         case CRYPTO_SHA2_512:
 207                 hash_name = "sha512";
 208                 hmac_mode = 0;
 209                 break;
 210         default:
 211                 dprintk(1, KERN_DEBUG, "bad mac: %d\n", sop->mac);
 212                 return -EINVAL;
 213         }
 214
 215         /* Create a session and put it to the list. */
 216         ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
 217         if (!ses_new)
 218                 return -ENOMEM;
 219
 220         /* Set-up crypto transform. */
 221         if (alg_name) {
 222                 uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
 223
 224                 if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
 225                         dprintk(1, KERN_DEBUG,
 226                                 "Setting key failed for %s-%zu.\n",
 227                                 alg_name, (size_t)sop->keylen*8);
 228                         ret = -EINVAL;
 229                         goto error_cipher;
 230                 }
 231
 232                 if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
 233                         ret = -EFAULT;
 234                         goto error_cipher;
 235                 }
 236
 237                 ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
 238                                                 sop->keylen, stream, aead);
 239                 if (ret < 0) {
 240                         dprintk(1, KERN_DEBUG,
 241                                 "Failed to load cipher for %s\n", alg_name);
 242                         ret = -EINVAL;
 243                         goto error_cipher;
 244                 }
 245         }
 246
 247         if (hash_name && aead == 0) {
 248                 uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
 249
 250                 if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
 251                         dprintk(1, KERN_DEBUG,
 252                                 "Setting key failed for %s-%zu.\n",
 253                                 alg_name, (size_t)sop->mackeylen*8);
 254                         ret = -EINVAL;
 255                         goto error_hash;
 256                 }
 257
 258                 if (sop->mackey && unlikely(copy_from_user(keyp, sop->mackey,
 259                                             sop->mackeylen))) {
 260                         ret = -EFAULT;
 261                         goto error_hash;
 262                 }
 263
 264                 ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
 265                                                         keyp, sop->mackeylen);
 266                 if (ret != 0) {
 267                         dprintk(1, KERN_DEBUG, "Failed to load hash for %s\n", hash_name);
 268                         ret = -EINVAL;
 269                         goto error_hash;
 270                 }
 271         }
 272
 273         ses_new->alignmask = max(ses_new->cdata.alignmask,
 274                                                   ses_new->hdata.alignmask);
 275         dprintk(2, KERN_DEBUG, "got alignmask %d\n", ses_new->alignmask);
 276
 277         ses_new->array_size = DEFAULT_PREALLOC_PAGES;
 278         dprintk(2, KERN_DEBUG, "preallocating for %d user pages\n",
 279                         ses_new->array_size);
 280         ses_new->pages = kzalloc(ses_new->array_size *
 281                         sizeof(struct page *), GFP_KERNEL);
 282         ses_new->sg = kzalloc(ses_new->array_size *
 283                         sizeof(struct scatterlist), GFP_KERNEL);
 284         if (ses_new->sg == NULL || ses_new->pages == NULL) {
 285                 dprintk(0, KERN_DEBUG, "Memory error\n");
 286                 ret = -ENOMEM;
 287                 goto error_hash;
 288         }
 289
 290         /* put the new session to the list */
 291         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 292         mutex_init(&ses_new->sem);
 293
 294         mutex_lock(&fcr->sem);
 295 restart:
 296         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 297                 /* Check for duplicate SID */
 298                 if (unlikely(ses_new->sid == ses_ptr->sid)) {
 299                         get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
 300                         /* Unless we have a broken RNG this
 301                            shouldn't loop forever... ;-) */
 302                         goto restart;
 303                 }
 304         }
 305
 306         list_add(&ses_new->entry, &fcr->list);
 307         mutex_unlock(&fcr->sem);
 308
 309         /* Fill in some values for the user. */
 310         sop->ses = ses_new->sid;
 311
 312         return 0;
 313
 314 error_hash:
 315         cryptodev_cipher_deinit(&ses_new->cdata);
 316         kfree(ses_new->sg);
 317         kfree(ses_new->pages);
 318 error_cipher:
 319         kfree(ses_new);
 320
 321         return ret;
 322
 323 }
 324
 325 /* Everything that needs to be done when remowing a session. */
 326 static inline void
 327 crypto_destroy_session(struct csession *ses_ptr)
 328 {
 329         if (!mutex_trylock(&ses_ptr->sem)) {
 330                 dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
 331                         ses_ptr->sid);
 332                 mutex_lock(&ses_ptr->sem);
 333         }
 334         dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
 335         cryptodev_cipher_deinit(&ses_ptr->cdata);
 336         cryptodev_hash_deinit(&ses_ptr->hdata);
 337         dprintk(2, KERN_DEBUG, "freeing space for %d user pages\n",
 338                         ses_ptr->array_size);
 339         kfree(ses_ptr->pages);
 340         kfree(ses_ptr->sg);
 341         mutex_unlock(&ses_ptr->sem);
 342         kfree(ses_ptr);
 343 }
 344
 345 /* Look up a session by ID and remove. */
 346 static int
 347 crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
 348 {
 349         struct csession *tmp, *ses_ptr;
 350         struct list_head *head;
 351         int ret = 0;
 352
 353         mutex_lock(&fcr->sem);
 354         head = &fcr->list;
 355         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 356                 if (ses_ptr->sid == sid) {
 357                         list_del(&ses_ptr->entry);
 358                         crypto_destroy_session(ses_ptr);
 359                         break;
 360                 }
 361         }
 362
 363         if (unlikely(!ses_ptr)) {
 364                 dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
 365                         sid);
 366                 ret = -ENOENT;
 367         }
 368         mutex_unlock(&fcr->sem);
 369
 370         return ret;
 371 }
 372
 373 /* Remove all sessions when closing the file */
 374 static int
 375 crypto_finish_all_sessions(struct fcrypt *fcr)
 376 {
 377         struct csession *tmp, *ses_ptr;
 378         struct list_head *head;
 379
 380         mutex_lock(&fcr->sem);
 381
 382         head = &fcr->list;
 383         list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
 384                 list_del(&ses_ptr->entry);
 385                 crypto_destroy_session(ses_ptr);
 386         }
 387         mutex_unlock(&fcr->sem);
 388
 389         return 0;
 390 }
 391
 392 /* Look up session by session ID. The returned session is locked. */
 393 struct csession *
 394 crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
 395 {
 396         struct csession *ses_ptr, *retval = NULL;
 397
 398         if (unlikely(fcr == NULL))
 399                 return NULL;
 400
 401         mutex_lock(&fcr->sem);
 402         list_for_each_entry(ses_ptr, &fcr->list, entry) {
 403                 if (ses_ptr->sid == sid) {
 404                         mutex_lock(&ses_ptr->sem);
 405                         retval = ses_ptr;
 406                         break;
 407                 }
 408         }
 409         mutex_unlock(&fcr->sem);
 410
 411         return retval;
 412 }
 413
 414 static void cryptask_routine(struct work_struct *work)
 415 {
 416         struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask);
 417         struct todo_list_item *item;
 418         LIST_HEAD(tmp);
 419
 420         /* fetch all pending jobs into the temporary list */
 421         mutex_lock(&pcr->todo.lock);
 422         list_cut_position(&tmp, &pcr->todo.list, pcr->todo.list.prev);
 423         mutex_unlock(&pcr->todo.lock);
 424
 425         /* handle each job locklessly */
 426         list_for_each_entry(item, &tmp, __hook) {
 427                 item->result = crypto_run(&pcr->fcrypt, &item->kcop);
 428                 if (unlikely(item->result))
 429                         dprintk(0, KERN_ERR, "crypto_run() failed: %d\n",
 430                                         item->result);
 431         }
 432
 433         /* push all handled jobs to the done list at once */
 434         mutex_lock(&pcr->done.lock);
 435         list_splice_tail(&tmp, &pcr->done.list);
 436         mutex_unlock(&pcr->done.lock);
 437
 438         /* wake for POLLIN */
 439         wake_up_interruptible(&pcr->user_waiter);
 440 }
 441
 442 /* ====== /dev/crypto ====== */
 443
 444 static int
 445 cryptodev_open(struct inode *inode, struct file *filp)
 446 {
 447         struct todo_list_item *tmp;
 448         struct crypt_priv *pcr;
 449         int i;
 450
 451         pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
 452         if (!pcr)
 453                 return -ENOMEM;
 454
 455         memset(pcr, 0, sizeof(*pcr));
 456         mutex_init(&pcr->fcrypt.sem);
 457         INIT_LIST_HEAD(&pcr->fcrypt.list);
 458
 459         INIT_LIST_HEAD(&pcr->free.list);
 460         INIT_LIST_HEAD(&pcr->todo.list);
 461         INIT_LIST_HEAD(&pcr->done.list);
 462         INIT_WORK(&pcr->cryptask, cryptask_routine);
 463         mutex_init(&pcr->free.lock);
 464         mutex_init(&pcr->todo.lock);
 465         mutex_init(&pcr->done.lock);
 466         init_waitqueue_head(&pcr->user_waiter);
 467
 468         for (i = 0; i < DEF_COP_RINGSIZE; i++) {
 469                 tmp = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
 470                 if (!tmp)
 471                         return -ENOMEM;
 472                 pcr->itemcount++;
 473                 dprintk(2, KERN_DEBUG, "allocated new item at %lx\n",
 474                                 (unsigned long)tmp);
 475                 list_add(&tmp->__hook, &pcr->free.list);
 476         }
 477
 478         filp->private_data = pcr;
 479         dprintk(2, KERN_DEBUG,
 480                 "Cryptodev handle initialised, %d elements in queue\n",
 481                 DEF_COP_RINGSIZE);
 482         return 0;
 483 }
 484
 485 static int
 486 cryptodev_release(struct inode *inode, struct file *filp)
 487 {
 488         struct crypt_priv *pcr = filp->private_data;
 489         struct todo_list_item *item, *item_safe;
 490         int items_freed = 0;
 491
 492         if (!pcr)
 493                 return 0;
 494
 495         cancel_work_sync(&pcr->cryptask);
 496
 497         mutex_destroy(&pcr->todo.lock);
 498         mutex_destroy(&pcr->done.lock);
 499         mutex_destroy(&pcr->free.lock);
 500
 501         list_splice_tail(&pcr->todo.list, &pcr->free.list);
 502         list_splice_tail(&pcr->done.list, &pcr->free.list);
 503
 504         list_for_each_entry_safe(item, item_safe, &pcr->free.list, __hook) {
 505                 dprintk(2, KERN_DEBUG, "freeing item at %lx\n",
 506                                 (unsigned long)item);
 507                 list_del(&item->__hook);
 508                 kfree(item);
 509                 items_freed++;
 510
 511         }
 512         if (items_freed != pcr->itemcount) {
 513                 dprintk(0, KERN_ERR,
 514                         "freed %d items, but %d should exist!\n",
 515                         items_freed, pcr->itemcount);
 516         }
 517
 518         crypto_finish_all_sessions(&pcr->fcrypt);
 519         kfree(pcr);
 520         filp->private_data = NULL;
 521
 522         dprintk(2, KERN_DEBUG,
 523                 "Cryptodev handle deinitialised, %d elements freed\n",
 524                 items_freed);
 525         return 0;
 526 }
 527
 528 static int
 529 clonefd(struct file *filp)
 530 {
 531         int ret;
 532         ret = get_unused_fd();
 533         if (ret >= 0) {
 534                         get_file(filp);
 535                         fd_install(ret, filp);
 536         }
 537
 538         return ret;
 539 }
 540
 541 /* enqueue a job for asynchronous completion
 542  *
 543  * returns:
 544  * -EBUSY when there are no free queue slots left
 545  *        (and the number of slots has reached it MAX_COP_RINGSIZE)
 546  * -EFAULT when there was a memory allocation error
 547  * 0 on success */
 548 static int crypto_async_run(struct crypt_priv *pcr, struct kernel_crypt_op *kcop)
 549 {
 550         struct todo_list_item *item = NULL;
 551
 552         mutex_lock(&pcr->free.lock);
 553         if (likely(!list_empty(&pcr->free.list))) {
 554                 item = list_first_entry(&pcr->free.list,
 555                                 struct todo_list_item, __hook);
 556                 list_del(&item->__hook);
 557         } else if (pcr->itemcount < MAX_COP_RINGSIZE) {
 558                 pcr->itemcount++;
 559         } else {
 560                 mutex_unlock(&pcr->free.lock);
 561                 return -EBUSY;
 562         }
 563         mutex_unlock(&pcr->free.lock);
 564
 565         if (unlikely(!item)) {
 566                 item = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
 567                 if (unlikely(!item))
 568                         return -EFAULT;
 569                 dprintk(1, KERN_INFO, "increased item count to %d\n",
 570                                 pcr->itemcount);
 571         }
 572
 573         memcpy(&item->kcop, kcop, sizeof(struct kernel_crypt_op));
 574
 575         mutex_lock(&pcr->todo.lock);
 576         list_add_tail(&item->__hook, &pcr->todo.list);
 577         mutex_unlock(&pcr->todo.lock);
 578
 579         queue_work(cryptodev_wq, &pcr->cryptask);
 580         return 0;
 581 }
 582
 583 /* get the first completed job from the "done" queue
 584  *
 585  * returns:
 586  * -EBUSY if no completed jobs are ready (yet)
 587  * the return value of crypto_run() otherwise */
 588 static int crypto_async_fetch(struct crypt_priv *pcr,
 589                 struct kernel_crypt_op *kcop)
 590 {
 591         struct todo_list_item *item;
 592         int retval;
 593
 594         mutex_lock(&pcr->done.lock);
 595         if (list_empty(&pcr->done.list)) {
 596                 mutex_unlock(&pcr->done.lock);
 597                 return -EBUSY;
 598         }
 599         item = list_first_entry(&pcr->done.list, struct todo_list_item, __hook);
 600         list_del(&item->__hook);
 601         mutex_unlock(&pcr->done.lock);
 602
 603         memcpy(kcop, &item->kcop, sizeof(struct kernel_crypt_op));
 604         retval = item->result;
 605
 606         mutex_lock(&pcr->free.lock);
 607         list_add_tail(&item->__hook, &pcr->free.list);
 608         mutex_unlock(&pcr->free.lock);
 609
 610         /* wake for POLLOUT */
 611         wake_up_interruptible(&pcr->user_waiter);
 612
 613         return retval;
 614 }
 615
 616 /* this function has to be called from process context */
 617 static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
 618 {
 619         struct crypt_op *cop = &kcop->cop;
 620         struct csession *ses_ptr;
 621         int rc;
 622
 623         /* this also enters ses_ptr->sem */
 624         ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
 625         if (unlikely(!ses_ptr)) {
 626                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
 627                 return -EINVAL;
 628         }
 629         kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
 630         kcop->digestsize = 0; /* will be updated during operation */
 631
 632         crypto_put_session(ses_ptr);
 633
 634         kcop->task = current;
 635         kcop->mm = current->mm;
 636
 637         if (cop->iv) {
 638                 rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen);
 639                 if (unlikely(rc)) {
 640                         dprintk(1, KERN_ERR,
 641                                 "error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
 642                                 kcop->ivlen, rc, (unsigned long)cop->iv);
 643                         return -EFAULT;
 644                 }
 645         }
 646
 647         return 0;
 648 }
 649
 650 /* this function has to be called from process context */
 651 static int fill_cop_from_kcop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
 652 {
 653         int ret;
 654
 655         if (kcop->digestsize) {
 656                 ret = copy_to_user(kcop->cop.mac,
 657                                 kcop->hash_output, kcop->digestsize);
 658                 if (unlikely(ret))
 659                         return -EFAULT;
 660         }
 661         if (kcop->ivlen && kcop->cop.flags & COP_FLAG_WRITE_IV) {
 662                 ret = copy_to_user(kcop->cop.iv,
 663                                 kcop->iv, kcop->ivlen);
 664                 if (unlikely(ret))
 665                         return -EFAULT;
 666         }
 667         return 0;
 668 }
 669
 670 static int kcop_from_user(struct kernel_crypt_op *kcop,
 671                         struct fcrypt *fcr, void __user *arg)
 672 {
 673         if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
 674                 return -EFAULT;
 675
 676         return fill_kcop_from_cop(kcop, fcr);
 677 }
 678
 679 static int kcop_to_user(struct kernel_crypt_op *kcop,
 680                         struct fcrypt *fcr, void __user *arg)
 681 {
 682         int ret;
 683
 684         ret = fill_cop_from_kcop(kcop, fcr);
 685         if (unlikely(ret)) {
 686                 dprintk(1, KERN_ERR, "Error in fill_cop_from_kcop\n");
 687                 return ret;
 688         }
 689
 690         if (unlikely(copy_to_user(arg, &kcop->cop, sizeof(kcop->cop)))) {
 691                 dprintk(1, KERN_ERR, "Cannot copy to userspace\n");
 692                 return -EFAULT;
 693         }
 694         return 0;
 695 }
 696
 697 static inline void tfm_info_to_alg_info(struct alg_info *dst, struct crypto_tfm *tfm)
 698 {
 699         snprintf(dst->cra_name, CRYPTODEV_MAX_ALG_NAME,
 700                         "%s", crypto_tfm_alg_name(tfm));
 701         snprintf(dst->cra_driver_name, CRYPTODEV_MAX_ALG_NAME,
 702                         "%s", crypto_tfm_alg_driver_name(tfm));
 703 }
 704
 705 static unsigned int is_known_accelerated(struct crypto_tfm *tfm)
 706 {
 707 const char* name = crypto_tfm_alg_driver_name(tfm);
 708
 709         if (name == NULL)
 710           return 1; /* assume accelerated */
 711
 712         if (strstr(name, "-talitos"))
 713           return 1;
 714         else if (strncmp(name, "mv-", 3) == 0)
 715           return 1;
 716         else if (strstr(name, "geode"))
 717           return 1;
 718         else if (strstr(name, "hifn"))
 719           return 1;
 720         else if (strstr(name, "-ixp4xx"))
 721           return 1;
 722         else if (strstr(name, "-omap"))
 723           return 1;
 724         else if (strstr(name, "-picoxcell"))
 725           return 1;
 726         else if (strstr(name, "-s5p"))
 727           return 1;
 728         else if (strstr(name, "-ppc4xx"))
 729           return 1;
 730         else if (strstr(name, "-caam"))
 731           return 1;
 732         else if (strstr(name, "-n2"))
 733           return 1;
 734
 735         return 0;
 736 }
 737
 738 static int get_session_info(struct fcrypt *fcr, struct session_info_op *siop)
 739 {
 740         struct csession *ses_ptr;
 741         struct crypto_tfm *tfm;
 742
 743         /* this also enters ses_ptr->sem */
 744         ses_ptr = crypto_get_session_by_sid(fcr, siop->ses);
 745         if (unlikely(!ses_ptr)) {
 746                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", siop->ses);
 747                 return -EINVAL;
 748         }
 749
 750         siop->flags = 0;
 751
 752         if (ses_ptr->cdata.init) {
 753                 if (ses_ptr->cdata.aead == 0) {
 754                         tfm = crypto_ablkcipher_tfm(ses_ptr->cdata.async.s);
 755                 } else {
 756                         tfm = crypto_aead_tfm(ses_ptr->cdata.async.as);
 757                 }
 758                 tfm_info_to_alg_info(&siop->cipher_info, tfm);
 759 #ifdef CRYPTO_ALG_KERN_DRIVER_ONLY
 760                 if (tfm->__crt_alg->cra_flags & CRYPTO_ALG_KERN_DRIVER_ONLY)
 761                         siop->flags |= SIOP_FLAG_KERNEL_DRIVER_ONLY;
 762 #else
 763                 if (is_known_accelerated(tfm))
 764                         siop->flags |= SIOP_FLAG_KERNEL_DRIVER_ONLY;
 765 #endif
 766         }
 767         if (ses_ptr->hdata.init) {
 768                 tfm = crypto_ahash_tfm(ses_ptr->hdata.async.s);
 769                 tfm_info_to_alg_info(&siop->hash_info, tfm);
 770 #ifdef CRYPTO_ALG_KERN_DRIVER_ONLY
 771                 if (tfm->__crt_alg->cra_flags & CRYPTO_ALG_KERN_DRIVER_ONLY)
 772                         siop->flags |= SIOP_FLAG_KERNEL_DRIVER_ONLY;
 773 #else
 774                 if (is_known_accelerated(tfm))
 775                         siop->flags |= SIOP_FLAG_KERNEL_DRIVER_ONLY;
 776 #endif
 777         }
 778
 779         siop->alignmask = ses_ptr->alignmask;
 780
 781         crypto_put_session(ses_ptr);
 782         return 0;
 783 }
 784
 785 static long
 786 cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
 787 {
 788         void __user *arg = (void __user *)arg_;
 789         int __user *p = arg;
 790         struct session_op sop;
 791         struct kernel_crypt_op kcop;
 792         struct kernel_crypt_auth_op kcaop;
 793         struct crypt_priv *pcr = filp->private_data;
 794         struct fcrypt *fcr;
 795         struct session_info_op siop;
 796         uint32_t ses;
 797         int ret, fd;
 798
 799         if (unlikely(!pcr))
 800                 BUG();
 801
 802         fcr = &pcr->fcrypt;
 803
 804         switch (cmd) {
 805         case CIOCASYMFEAT:
 806                 return put_user(0, p);
 807         case CRIOGET:
 808                 fd = clonefd(filp);
 809                 ret = put_user(fd, p);
 810                 if (unlikely(ret)) {
 811                         sys_close(fd);
 812                         return ret;
 813                 }
 814                 return ret;
 815         case CIOCGSESSION:
 816                 if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
 817                         return -EFAULT;
 818
 819                 ret = crypto_create_session(fcr, &sop);
 820                 if (unlikely(ret))
 821                         return ret;
 822                 ret = copy_to_user(arg, &sop, sizeof(sop));
 823                 if (unlikely(ret)) {
 824                         crypto_finish_session(fcr, sop.ses);
 825                         return -EFAULT;
 826                 }
 827                 return ret;
 828         case CIOCFSESSION:
 829                 ret = get_user(ses, (uint32_t __user *)arg);
 830                 if (unlikely(ret))
 831                         return ret;
 832                 ret = crypto_finish_session(fcr, ses);
 833                 return ret;
 834         case CIOCGSESSINFO:
 835                 if (unlikely(copy_from_user(&siop, arg, sizeof(siop))))
 836                         return -EFAULT;
 837
 838                 ret = get_session_info(fcr, &siop);
 839                 if (unlikely(ret))
 840                         return ret;
 841                 return copy_to_user(arg, &siop, sizeof(siop));
 842         case CIOCCRYPT:
 843                 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg))) {
 844                         dprintk(1, KERN_WARNING, "Error copying from user\n");
 845                         return ret;
 846                 }
 847
 848                 ret = crypto_run(fcr, &kcop);
 849                 if (unlikely(ret)) {
 850                         dprintk(1, KERN_WARNING, "Error in crypto_run\n");
 851                         return ret;
 852                 }
 853
 854                 return kcop_to_user(&kcop, fcr, arg);
 855         case CIOCAUTHCRYPT:
 856                 if (unlikely(ret = kcaop_from_user(&kcaop, fcr, arg))) {
 857                         dprintk(1, KERN_WARNING, "Error copying from user\n");
 858                         return ret;
 859                 }
 860
 861                 ret = crypto_auth_run(fcr, &kcaop);
 862                 if (unlikely(ret)) {
 863                         dprintk(1, KERN_WARNING, "Error in crypto_auth_run\n");
 864                         return ret;
 865                 }
 866                 return kcaop_to_user(&kcaop, fcr, arg);
 867         case CIOCASYNCCRYPT:
 868                 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
 869                         return ret;
 870
 871                 return crypto_async_run(pcr, &kcop);
 872         case CIOCASYNCFETCH:
 873                 ret = crypto_async_fetch(pcr, &kcop);
 874                 if (unlikely(ret))
 875                         return ret;
 876
 877                 return kcop_to_user(&kcop, fcr, arg);
 878         default:
 879                 return -EINVAL;
 880         }
 881 }
 882
 883 /* compatibility code for 32bit userlands */
 884 #ifdef CONFIG_COMPAT
 885
 886 static inline void
 887 compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
 888 {
 889         sop->cipher = compat->cipher;
 890         sop->mac = compat->mac;
 891         sop->keylen = compat->keylen;
 892
 893         sop->key       = compat_ptr(compat->key);
 894         sop->mackeylen = compat->mackeylen;
 895         sop->mackey    = compat_ptr(compat->mackey);
 896         sop->ses       = compat->ses;
 897 }
 898
 899 static inline void
 900 session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
 901 {
 902         compat->cipher = sop->cipher;
 903         compat->mac = sop->mac;
 904         compat->keylen = sop->keylen;
 905
 906         compat->key       = ptr_to_compat(sop->key);
 907         compat->mackeylen = sop->mackeylen;
 908         compat->mackey    = ptr_to_compat(sop->mackey);
 909         compat->ses       = sop->ses;
 910 }
 911
 912 static inline void
 913 compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
 914 {
 915         cop->ses = compat->ses;
 916         cop->op = compat->op;
 917         cop->flags = compat->flags;
 918         cop->len = compat->len;
 919
 920         cop->src = compat_ptr(compat->src);
 921         cop->dst = compat_ptr(compat->dst);
 922         cop->mac = compat_ptr(compat->mac);
 923         cop->iv  = compat_ptr(compat->iv);
 924 }
 925
 926 static inline void
 927 crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
 928 {
 929         compat->ses = cop->ses;
 930         compat->op = cop->op;
 931         compat->flags = cop->flags;
 932         compat->len = cop->len;
 933
 934         compat->src = ptr_to_compat(cop->src);
 935         compat->dst = ptr_to_compat(cop->dst);
 936         compat->mac = ptr_to_compat(cop->mac);
 937         compat->iv  = ptr_to_compat(cop->iv);
 938 }
 939
 940 static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
 941                                  struct fcrypt *fcr, void __user *arg)
 942 {
 943         struct compat_crypt_op compat_cop;
 944
 945         if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
 946                 return -EFAULT;
 947         compat_to_crypt_op(&compat_cop, &kcop->cop);
 948
 949         return fill_kcop_from_cop(kcop, fcr);
 950 }
 951
 952 static int compat_kcop_to_user(struct kernel_crypt_op *kcop,
 953                                  struct fcrypt *fcr, void __user *arg)
 954 {
 955         int ret;
 956         struct compat_crypt_op compat_cop;
 957
 958         ret = fill_cop_from_kcop(kcop, fcr);
 959         if (unlikely(ret)) {
 960                 dprintk(1, KERN_WARNING, "Error in fill_cop_from_kcop\n");
 961                 return ret;
 962         }
 963         crypt_op_to_compat(&kcop->cop, &compat_cop);
 964
 965         if (unlikely(copy_to_user(arg, &compat_cop, sizeof(compat_cop)))) {
 966                 dprintk(1, KERN_WARNING, "Error copying to user\n");
 967                 return -EFAULT;
 968         }
 969         return 0;
 970 }
 971
 972 static long
 973 cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
 974 {
 975         void __user *arg = (void __user *)arg_;
 976         struct crypt_priv *pcr = file->private_data;
 977         struct fcrypt *fcr;
 978         struct session_op sop;
 979         struct compat_session_op compat_sop;
 980         struct kernel_crypt_op kcop;
 981         int ret;
 982
 983         if (unlikely(!pcr))
 984                 BUG();
 985
 986         fcr = &pcr->fcrypt;
 987
 988         switch (cmd) {
 989         case CIOCASYMFEAT:
 990         case CRIOGET:
 991         case CIOCFSESSION:
 992         case CIOCGSESSINFO:
 993                 return cryptodev_ioctl(file, cmd, arg_);
 994
 995         case COMPAT_CIOCGSESSION:
 996                 if (unlikely(copy_from_user(&compat_sop, arg,
 997                                             sizeof(compat_sop))))
 998                         return -EFAULT;
 999                 compat_to_session_op(&compat_sop, &sop);
1000
1001                 ret = crypto_create_session(fcr, &sop);
1002                 if (unlikely(ret))
1003                         return ret;
1004
1005                 session_op_to_compat(&sop, &compat_sop);
1006                 ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
1007                 if (unlikely(ret)) {
1008                         crypto_finish_session(fcr, sop.ses);
1009                         return -EFAULT;
1010                 }
1011                 return ret;
1012
1013         case COMPAT_CIOCCRYPT:
1014                 ret = compat_kcop_from_user(&kcop, fcr, arg);
1015                 if (unlikely(ret))
1016                         return ret;
1017
1018                 ret = crypto_run(fcr, &kcop);
1019                 if (unlikely(ret))
1020                         return ret;
1021
1022                 return compat_kcop_to_user(&kcop, fcr, arg);
1023         case COMPAT_CIOCASYNCCRYPT:
1024                 if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg)))
1025                         return ret;
1026
1027                 return crypto_async_run(pcr, &kcop);
1028         case COMPAT_CIOCASYNCFETCH:
1029                 ret = crypto_async_fetch(pcr, &kcop);
1030                 if (unlikely(ret))
1031                         return ret;
1032
1033                 return compat_kcop_to_user(&kcop, fcr, arg);
1034
1035         default:
1036                 return -EINVAL;
1037         }
1038 }
1039
1040 #endif /* CONFIG_COMPAT */
1041
1042 static unsigned int cryptodev_poll(struct file *file, poll_table *wait)
1043 {
1044         struct crypt_priv *pcr = file->private_data;
1045         int ret = 0;
1046
1047         poll_wait(file, &pcr->user_waiter, wait);
1048
1049         if (!list_empty_careful(&pcr->done.list))
1050                 ret |= POLLIN | POLLRDNORM;
1051         if (!list_empty_careful(&pcr->free.list) || pcr->itemcount < MAX_COP_RINGSIZE)
1052                 ret |= POLLOUT | POLLWRNORM;
1053
1054         return ret;
1055 }
1056
1057 static const struct file_operations cryptodev_fops = {
1058         .owner = THIS_MODULE,
1059         .open = cryptodev_open,
1060         .release = cryptodev_release,
1061         .unlocked_ioctl = cryptodev_ioctl,
1062 #ifdef CONFIG_COMPAT
1063         .compat_ioctl = cryptodev_compat_ioctl,
1064 #endif /* CONFIG_COMPAT */
1065         .poll = cryptodev_poll,
1066 };
1067
1068 static struct miscdevice cryptodev = {
1069         .minor = MISC_DYNAMIC_MINOR,
1070         .name = "crypto",
1071         .fops = &cryptodev_fops,
1072         .mode = S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH,
1073 };
1074
1075 static int __init
1076 cryptodev_register(void)
1077 {
1078         int rc;
1079
1080         rc = misc_register(&cryptodev);
1081         if (unlikely(rc)) {
1082                 printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
1083                 return rc;
1084         }
1085
1086         return 0;
1087 }
1088
1089 static void __exit
1090 cryptodev_deregister(void)
1091 {
1092         misc_deregister(&cryptodev);
1093 }
1094
1095 /* ====== Module init/exit ====== */
1096 static int __init init_cryptodev(void)
1097 {
1098         int rc;
1099
1100         cryptodev_wq = create_workqueue("cryptodev_queue");
1101         if (unlikely(!cryptodev_wq)) {
1102                 printk(KERN_ERR PFX "failed to allocate the cryptodev workqueue\n");
1103                 return -EFAULT;
1104         }
1105
1106         rc = cryptodev_register();
1107         if (unlikely(rc)) {
1108                 destroy_workqueue(cryptodev_wq);
1109                 return rc;
1110         }
1111
1112         printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
1113
1114         return 0;
1115 }
1116
1117 static void __exit exit_cryptodev(void)
1118 {
1119         flush_workqueue(cryptodev_wq);
1120         destroy_workqueue(cryptodev_wq);
1121
1122         cryptodev_deregister();
1123         printk(KERN_INFO PFX "driver unloaded.\n");
1124 }
1125
1126 module_init(init_cryptodev);
1127 module_exit(exit_cryptodev);
1128