?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
allow updating the IV in userspace
[cryptodev-linux.git] / cryptodev_main.c
blobcb81107c119599a0497ea9a79c180cdeaab2af57
1 /*
2 * Driver for /dev/crypto device (aka CryptoDev)
3 *
4 * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
5 * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
6 *
7 * This file is part of linux cryptodev.
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License
11 * as published by the Free Software Foundation; either version 2
12 * of the License, or (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc.,
22 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
23 */
24
25 /*
26 * Device /dev/crypto provides an interface for
27 * accessing kernel CryptoAPI algorithms (ciphers,
28 * hashes) from userspace programs.
29 *
30 * /dev/crypto interface was originally introduced in
31 * OpenBSD and this module attempts to keep the API.
32 *
33 */
34
35 #include <crypto/hash.h>
36 #include <linux/crypto.h>
37 #include <linux/mm.h>
38 #include <linux/highmem.h>
39 #include <linux/ioctl.h>
40 #include <linux/random.h>
41 #include <linux/syscalls.h>
42 #include <linux/pagemap.h>
43 #include <linux/poll.h>
44 #include <linux/uaccess.h>
45 #include "cryptodev.h"
46 #include <linux/scatterlist.h>
47 #include "cryptodev_int.h"
48 #include "version.h"
49
50 MODULE_AUTHOR("Nikos Mavrogiannopoulos <nmav@gnutls.org>");
51 MODULE_DESCRIPTION("CryptoDev driver");
52 MODULE_LICENSE("GPL");
53
54 /* ====== Compile-time config ====== */
55
56 #define CRYPTODEV_STATS
57
58 /* Default (pre-allocated) and maximum size of the job queue.
59 * These are free, pending and done items all together. */
60 #define DEF_COP_RINGSIZE 16
61 #define MAX_COP_RINGSIZE 64
62
63 /* ====== Module parameters ====== */
64
65 int cryptodev_verbosity;
66 module_param(cryptodev_verbosity, int, 0644);
67 MODULE_PARM_DESC(cryptodev_verbosity, "0: normal, 1: verbose, 2: debug");
68
69 #ifdef CRYPTODEV_STATS
70 static int enable_stats;
71 module_param(enable_stats, int, 0644);
72 MODULE_PARM_DESC(enable_stats, "collect statictics about cryptodev usage");
73 #endif
74
75 /* ====== CryptoAPI ====== */
76 struct fcrypt {
77 struct list_head list;
78 struct mutex sem;
79 };
80
81 struct todo_list_item {
82 struct list_head __hook;
83 struct kernel_crypt_op kcop;
84 int result;
85 };
86
87 struct locked_list {
88 struct list_head list;
89 struct mutex lock;
90 };
91
92 struct crypt_priv {
93 struct fcrypt fcrypt;
94 struct locked_list free, todo, done;
95 int itemcount;
96 struct work_struct cryptask;
97 wait_queue_head_t user_waiter;
98 };
99
100 #define FILL_SG(sg, ptr, len) \
101 do { \
102 (sg)->page = virt_to_page(ptr); \
103 (sg)->offset = offset_in_page(ptr); \
104 (sg)->length = len; \
105 (sg)->dma_address = 0; \
106 } while (0)
107
108 struct csession {
109 struct list_head entry;
110 struct mutex sem;
111 struct cipher_data cdata;
112 struct hash_data hdata;
113 uint32_t sid;
114 uint32_t alignmask;
115 #ifdef CRYPTODEV_STATS
116 #if !((COP_ENCRYPT < 2) && (COP_DECRYPT < 2))
117 #error Struct csession.stat uses COP_{ENCRYPT,DECRYPT} as indices. Do something!
118 #endif
119 unsigned long long stat[2];
120 size_t stat_max_size, stat_count;
121 #endif
122 int array_size;
123 struct page **pages;
124 struct scatterlist *sg;
125 };
126
127 /* cryptodev's own workqueue, keeps crypto tasks from disturbing the force */
128 static struct workqueue_struct *cryptodev_wq;
129
130 /* Prepare session for future use. */
131 static int
132 crypto_create_session(struct fcrypt *fcr, struct session_op *sop)
133 {
134 struct csession *ses_new = NULL, *ses_ptr;
135 int ret = 0;
136 const char *alg_name = NULL;
137 const char *hash_name = NULL;
138 int hmac_mode = 1;
139
140 /* Does the request make sense? */
141 if (unlikely(!sop->cipher && !sop->mac)) {
142 dprintk(1, KERN_DEBUG, "Both 'cipher' and 'mac' unset.\n");
143 return -EINVAL;
144 }
145
146 switch (sop->cipher) {
147 case 0:
148 break;
149 case CRYPTO_DES_CBC:
150 alg_name = "cbc(des)";
151 break;
152 case CRYPTO_3DES_CBC:
153 alg_name = "cbc(des3_ede)";
154 break;
155 case CRYPTO_BLF_CBC:
156 alg_name = "cbc(blowfish)";
157 break;
158 case CRYPTO_AES_CBC:
159 alg_name = "cbc(aes)";
160 break;
161 case CRYPTO_AES_ECB:
162 alg_name = "ecb(aes)";
163 break;
164 case CRYPTO_CAMELLIA_CBC:
165 alg_name = "cbc(camelia)";
166 break;
167 case CRYPTO_AES_CTR:
168 alg_name = "ctr(aes)";
169 break;
170 case CRYPTO_NULL:
171 alg_name = "ecb(cipher_null)";
172 break;
173 default:
174 dprintk(1, KERN_DEBUG, "%s: bad cipher: %d\n", __func__,
175 sop->cipher);
176 return -EINVAL;
177 }
178
179 switch (sop->mac) {
180 case 0:
181 break;
182 case CRYPTO_MD5_HMAC:
183 hash_name = "hmac(md5)";
184 break;
185 case CRYPTO_RIPEMD160_HMAC:
186 hash_name = "hmac(rmd160)";
187 break;
188 case CRYPTO_SHA1_HMAC:
189 hash_name = "hmac(sha1)";
190 break;
191 case CRYPTO_SHA2_256_HMAC:
192 hash_name = "hmac(sha256)";
193 break;
194 case CRYPTO_SHA2_384_HMAC:
195 hash_name = "hmac(sha384)";
196 break;
197 case CRYPTO_SHA2_512_HMAC:
198 hash_name = "hmac(sha512)";
199 break;
200
201 /* non-hmac cases */
202 case CRYPTO_MD5:
203 hash_name = "md5";
204 hmac_mode = 0;
205 break;
206 case CRYPTO_RIPEMD160:
207 hash_name = "rmd160";
208 hmac_mode = 0;
209 break;
210 case CRYPTO_SHA1:
211 hash_name = "sha1";
212 hmac_mode = 0;
213 break;
214 case CRYPTO_SHA2_256:
215 hash_name = "sha256";
216 hmac_mode = 0;
217 break;
218 case CRYPTO_SHA2_384:
219 hash_name = "sha384";
220 hmac_mode = 0;
221 break;
222 case CRYPTO_SHA2_512:
223 hash_name = "sha512";
224 hmac_mode = 0;
225 break;
226
227 default:
228 dprintk(1, KERN_DEBUG, "%s: bad mac: %d\n", __func__,
229 sop->mac);
230 return -EINVAL;
231 }
232
233 /* Create a session and put it to the list. */
234 ses_new = kzalloc(sizeof(*ses_new), GFP_KERNEL);
235 if (!ses_new)
236 return -ENOMEM;
237
238 /* Set-up crypto transform. */
239 if (alg_name) {
240 uint8_t keyp[CRYPTO_CIPHER_MAX_KEY_LEN];
241
242 if (unlikely(sop->keylen > CRYPTO_CIPHER_MAX_KEY_LEN)) {
243 dprintk(1, KERN_DEBUG,
244 "Setting key failed for %s-%zu.\n",
245 alg_name, (size_t)sop->keylen*8);
246 ret = -EINVAL;
247 goto error_cipher;
248 }
249
250 if (unlikely(copy_from_user(keyp, sop->key, sop->keylen))) {
251 ret = -EFAULT;
252 goto error_cipher;
253 }
254
255 ret = cryptodev_cipher_init(&ses_new->cdata, alg_name, keyp,
256 sop->keylen);
257 if (ret < 0) {
258 dprintk(1, KERN_DEBUG,
259 "%s: Failed to load cipher for %s\n",
260 __func__, alg_name);
261 ret = -EINVAL;
262 goto error_cipher;
263 }
264 }
265
266 if (hash_name) {
267 uint8_t keyp[CRYPTO_HMAC_MAX_KEY_LEN];
268
269 if (unlikely(sop->mackeylen > CRYPTO_HMAC_MAX_KEY_LEN)) {
270 dprintk(1, KERN_DEBUG,
271 "Setting key failed for %s-%zu.\n",
272 alg_name, (size_t)sop->mackeylen*8);
273 ret = -EINVAL;
274 goto error_hash;
275 }
276
277 if (sop->mackey && unlikely(copy_from_user(keyp, sop->mackey,
278 sop->mackeylen))) {
279 ret = -EFAULT;
280 goto error_hash;
281 }
282
283 ret = cryptodev_hash_init(&ses_new->hdata, hash_name, hmac_mode,
284 keyp, sop->mackeylen);
285 if (ret != 0) {
286 dprintk(1, KERN_DEBUG,
287 "%s: Failed to load hash for %s\n",
288 __func__, hash_name);
289 ret = -EINVAL;
290 goto error_hash;
291 }
292 }
293
294 ses_new->alignmask = max(ses_new->cdata.alignmask,
295 ses_new->hdata.alignmask);
296 dprintk(2, KERN_DEBUG, "%s: got alignmask %d\n", __func__, ses_new->alignmask);
297
298 ses_new->array_size = DEFAULT_PREALLOC_PAGES;
299 dprintk(2, KERN_DEBUG, "%s: preallocating for %d user pages\n",
300 __func__, ses_new->array_size);
301 ses_new->pages = kzalloc(ses_new->array_size *
302 sizeof(struct page *), GFP_KERNEL);
303 ses_new->sg = kzalloc(ses_new->array_size *
304 sizeof(struct scatterlist), GFP_KERNEL);
305 if (ses_new->sg == NULL || ses_new->pages == NULL) {
306 dprintk(0, KERN_DEBUG, "Memory error\n");
307 ret = -ENOMEM;
308 goto error_hash;
309 }
310
311 /* put the new session to the list */
312 get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
313 mutex_init(&ses_new->sem);
314
315 mutex_lock(&fcr->sem);
316 restart:
317 list_for_each_entry(ses_ptr, &fcr->list, entry) {
318 /* Check for duplicate SID */
319 if (unlikely(ses_new->sid == ses_ptr->sid)) {
320 get_random_bytes(&ses_new->sid, sizeof(ses_new->sid));
321 /* Unless we have a broken RNG this
322 shouldn't loop forever... ;-) */
323 goto restart;
324 }
325 }
326
327 list_add(&ses_new->entry, &fcr->list);
328 mutex_unlock(&fcr->sem);
329
330 /* Fill in some values for the user. */
331 sop->ses = ses_new->sid;
332
333 return 0;
334
335 error_hash:
336 cryptodev_cipher_deinit(&ses_new->cdata);
337 kfree(ses_new->sg);
338 kfree(ses_new->pages);
339 error_cipher:
340 kfree(ses_new);
341
342 return ret;
343
344 }
345
346 /* Everything that needs to be done when remowing a session. */
347 static inline void
348 crypto_destroy_session(struct csession *ses_ptr)
349 {
350 if (!mutex_trylock(&ses_ptr->sem)) {
351 dprintk(2, KERN_DEBUG, "Waiting for semaphore of sid=0x%08X\n",
352 ses_ptr->sid);
353 mutex_lock(&ses_ptr->sem);
354 }
355 dprintk(2, KERN_DEBUG, "Removed session 0x%08X\n", ses_ptr->sid);
356 #if defined(CRYPTODEV_STATS)
357 if (enable_stats)
358 dprintk(2, KERN_DEBUG,
359 "Usage in Bytes: enc=%llu, dec=%llu, "
360 "max=%zu, avg=%lu, cnt=%zu\n",
361 ses_ptr->stat[COP_ENCRYPT], ses_ptr->stat[COP_DECRYPT],
362 ses_ptr->stat_max_size, ses_ptr->stat_count > 0
363 ? ((unsigned long)(ses_ptr->stat[COP_ENCRYPT]+
364 ses_ptr->stat[COP_DECRYPT]) /
365 ses_ptr->stat_count) : 0,
366 ses_ptr->stat_count);
367 #endif
368 cryptodev_cipher_deinit(&ses_ptr->cdata);
369 cryptodev_hash_deinit(&ses_ptr->hdata);
370 dprintk(2, KERN_DEBUG, "%s: freeing space for %d user pages\n",
371 __func__, ses_ptr->array_size);
372 kfree(ses_ptr->pages);
373 kfree(ses_ptr->sg);
374 mutex_unlock(&ses_ptr->sem);
375 kfree(ses_ptr);
376 }
377
378 /* Look up a session by ID and remove. */
379 static int
380 crypto_finish_session(struct fcrypt *fcr, uint32_t sid)
381 {
382 struct csession *tmp, *ses_ptr;
383 struct list_head *head;
384 int ret = 0;
385
386 mutex_lock(&fcr->sem);
387 head = &fcr->list;
388 list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
389 if (ses_ptr->sid == sid) {
390 list_del(&ses_ptr->entry);
391 crypto_destroy_session(ses_ptr);
392 break;
393 }
394 }
395
396 if (unlikely(!ses_ptr)) {
397 dprintk(1, KERN_ERR, "Session with sid=0x%08X not found!\n",
398 sid);
399 ret = -ENOENT;
400 }
401 mutex_unlock(&fcr->sem);
402
403 return ret;
404 }
405
406 /* Remove all sessions when closing the file */
407 static int
408 crypto_finish_all_sessions(struct fcrypt *fcr)
409 {
410 struct csession *tmp, *ses_ptr;
411 struct list_head *head;
412
413 mutex_lock(&fcr->sem);
414
415 head = &fcr->list;
416 list_for_each_entry_safe(ses_ptr, tmp, head, entry) {
417 list_del(&ses_ptr->entry);
418 crypto_destroy_session(ses_ptr);
419 }
420 mutex_unlock(&fcr->sem);
421
422 return 0;
423 }
424
425 /* Look up session by session ID. The returned session is locked. */
426 static struct csession *
427 crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid)
428 {
429 struct csession *ses_ptr, *retval = 0;
430
431 mutex_lock(&fcr->sem);
432 list_for_each_entry(ses_ptr, &fcr->list, entry) {
433 if (ses_ptr->sid == sid) {
434 mutex_lock(&ses_ptr->sem);
435 retval = ses_ptr;
436 break;
437 }
438 }
439 mutex_unlock(&fcr->sem);
440
441 return retval;
442 }
443
444 static int
445 hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
446 struct scatterlist *src_sg, struct scatterlist *dst_sg,
447 uint32_t len)
448 {
449 int ret;
450
451 /* Always hash before encryption and after decryption. Maybe
452 * we should introduce a flag to switch... TBD later on.
453 */
454 if (cop->op == COP_ENCRYPT) {
455 if (ses_ptr->hdata.init != 0) {
456 ret = cryptodev_hash_update(&ses_ptr->hdata,
457 src_sg, len);
458 if (unlikely(ret))
459 goto out_err;
460 }
461 if (ses_ptr->cdata.init != 0) {
462 ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
463 src_sg, dst_sg, len);
464
465 if (unlikely(ret))
466 goto out_err;
467 }
468 } else {
469 if (ses_ptr->cdata.init != 0) {
470 ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
471 src_sg, dst_sg, len);
472
473 if (unlikely(ret))
474 goto out_err;
475 }
476
477 if (ses_ptr->hdata.init != 0) {
478 ret = cryptodev_hash_update(&ses_ptr->hdata,
479 dst_sg, len);
480 if (unlikely(ret))
481 goto out_err;
482 }
483 }
484 return 0;
485 out_err:
486 dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
487 return ret;
488 }
489
490
491 /* This is the main crypto function - feed it with plaintext
492 and get a ciphertext (or vice versa :-) */
493 static int
494 __crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
495 {
496 char *data;
497 char __user *src, *dst;
498 struct scatterlist sg;
499 size_t nbytes, bufsize;
500 int ret = 0;
501
502 nbytes = cop->len;
503 data = (char *)__get_free_page(GFP_KERNEL);
504
505 if (unlikely(!data))
506 return -ENOMEM;
507
508 bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
509
510 src = cop->src;
511 dst = cop->dst;
512
513 while (nbytes > 0) {
514 size_t current_len = nbytes > bufsize ? bufsize : nbytes;
515
516 if (unlikely(copy_from_user(data, src, current_len))) {
517 ret = -EFAULT;
518 break;
519 }
520
521 sg_init_one(&sg, data, current_len);
522
523 ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
524
525 if (unlikely(ret))
526 break;
527
528 if (ses_ptr->cdata.init != 0) {
529 if (unlikely(copy_to_user(dst, data, current_len))) {
530 ret = -EFAULT;
531 break;
532 }
533 }
534
535 dst += current_len;
536 nbytes -= current_len;
537 src += current_len;
538 }
539
540 free_page((unsigned long)data);
541 return ret;
542 }
543
544 void release_user_pages(struct page **pg, int pagecount)
545 {
546 while (pagecount--) {
547 if (!PageReserved(pg[pagecount]))
548 SetPageDirty(pg[pagecount]);
549 page_cache_release(pg[pagecount]);
550 }
551 }
552
553 /* offset of buf in it's first page */
554 #define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
555
556 /* fetch the pages addr resides in into pg and initialise sg with them */
557 int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
558 int pgcount, struct page **pg, struct scatterlist *sg,
559 struct task_struct *task, struct mm_struct *mm)
560 {
561 int ret, pglen, i = 0;
562 struct scatterlist *sgp;
563
564 down_write(&mm->mmap_sem);
565 ret = get_user_pages(task, mm,
566 (unsigned long)addr, pgcount, write, 0, pg, NULL);
567 up_write(&mm->mmap_sem);
568 if (ret != pgcount)
569 return -EINVAL;
570
571 sg_init_table(sg, pgcount);
572
573 pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
574 sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
575
576 len -= pglen;
577 for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
578 pglen = min((uint32_t)PAGE_SIZE, len);
579 sg_set_page(sgp, pg[i++], pglen, 0);
580 len -= pglen;
581 }
582 sg_mark_end(sg_last(sg, pgcount));
583 return 0;
584 }
585
586 /* make cop->src and cop->dst available in scatterlists */
587 static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop,
588 struct scatterlist **src_sg, struct scatterlist **dst_sg,
589 int *tot_pages)
590 {
591 int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1;
592 struct crypt_op *cop = &kcop->cop;
593 int rc;
594
595 if (cop->src == NULL)
596 return -EINVAL;
597
598 if (!IS_ALIGNED((unsigned long)cop->src, ses->alignmask)) {
599 dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
600 __func__, (unsigned long)cop->src, ses->alignmask + 1);
601 }
602
603 src_pagecount = PAGECOUNT(cop->src, cop->len);
604 if (!ses->cdata.init) { /* hashing only */
605 write_src = 0;
606 } else if (cop->src != cop->dst) { /* non-in-situ transformation */
607 if (cop->dst == NULL)
608 return -EINVAL;
609
610 dst_pagecount = PAGECOUNT(cop->dst, cop->len);
611 write_src = 0;
612
613 if (!IS_ALIGNED((unsigned long)cop->dst, ses->alignmask)) {
614 dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n",
615 __func__, (unsigned long)cop->dst, ses->alignmask + 1);
616 }
617
618 }
619 (*tot_pages) = pagecount = src_pagecount + dst_pagecount;
620
621 if (pagecount > ses->array_size) {
622 struct scatterlist *sg;
623 struct page **pages;
624 int array_size;
625
626 for (array_size = ses->array_size; array_size < pagecount;
627 array_size *= 2)
628 ;
629
630 dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
631 __func__, array_size);
632 pages = krealloc(ses->pages, array_size * sizeof(struct page *),
633 GFP_KERNEL);
634 if (unlikely(!pages))
635 return -ENOMEM;
636 ses->pages = pages;
637 sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
638 GFP_KERNEL);
639 if (unlikely(!sg))
640 return -ENOMEM;
641 ses->sg = sg;
642 ses->array_size = array_size;
643 }
644
645 rc = __get_userbuf(cop->src, cop->len, write_src, src_pagecount,
646 ses->pages, ses->sg, kcop->task, kcop->mm);
647 if (unlikely(rc)) {
648 dprintk(1, KERN_ERR,
649 "failed to get user pages for data input\n");
650 return -EINVAL;
651 }
652 (*src_sg) = (*dst_sg) = ses->sg;
653
654 if (!dst_pagecount)
655 return 0;
656
657 (*dst_sg) = ses->sg + src_pagecount;
658
659 rc = __get_userbuf(cop->dst, cop->len, 1, dst_pagecount,
660 ses->pages + src_pagecount, *dst_sg,
661 kcop->task, kcop->mm);
662 if (unlikely(rc)) {
663 dprintk(1, KERN_ERR,
664 "failed to get user pages for data output\n");
665 release_user_pages(ses->pages, src_pagecount);
666 return -EINVAL;
667 }
668 return 0;
669 }
670
671 /* This is the main crypto function - zero-copy edition */
672 static int
673 __crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
674 {
675 struct scatterlist *src_sg, *dst_sg;
676 struct crypt_op *cop = &kcop->cop;
677 int ret = 0, pagecount;
678
679 ret = get_userbuf(ses_ptr, kcop, &src_sg, &dst_sg, &pagecount);
680 if (unlikely(ret)) {
681 dprintk(1, KERN_ERR, "Error getting user pages. "
682 "Falling back to non zero copy.\n");
683 return __crypto_run_std(ses_ptr, cop);
684 }
685
686 ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
687
688 release_user_pages(ses_ptr->pages, pagecount);
689 return ret;
690 }
691
692 static int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
693 {
694 struct csession *ses_ptr;
695 struct crypt_op *cop = &kcop->cop;
696 int ret;
697
698 if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
699 dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", cop->op);
700 return -EINVAL;
701 }
702
703 /* this also enters ses_ptr->sem */
704 ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
705 if (unlikely(!ses_ptr)) {
706 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
707 return -EINVAL;
708 }
709
710 if (ses_ptr->hdata.init != 0 && !(cop->flags & (COP_FLAG_UPDATE | COP_FLAG_FINAL))) {
711 ret = cryptodev_hash_reset(&ses_ptr->hdata);
712 if (unlikely(ret)) {
713 dprintk(1, KERN_ERR,
714 "error in cryptodev_hash_reset()\n");
715 goto out_unlock;
716 }
717 }
718
719 if (ses_ptr->cdata.init != 0) {
720 int blocksize = ses_ptr->cdata.blocksize;
721
722 if (unlikely(cop->len % blocksize)) {
723 dprintk(1, KERN_ERR,
724 "data size (%u) isn't a multiple "
725 "of block size (%u)\n",
726 cop->len, blocksize);
727 ret = -EINVAL;
728 goto out_unlock;
729 }
730
731 cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
732 min(ses_ptr->cdata.ivsize, kcop->ivlen));
733 }
734
735 if (likely(cop->len)) {
736 ret = __crypto_run_zc(ses_ptr, kcop);
737 if (unlikely(ret))
738 goto out_unlock;
739 }
740
741 if (ses_ptr->cdata.init != 0) {
742 cryptodev_cipher_get_iv(&ses_ptr->cdata, kcop->iv,
743 min(ses_ptr->cdata.ivsize, kcop->ivlen));
744 }
745
746 if (ses_ptr->hdata.init != 0 &&
747 ((cop->flags & COP_FLAG_FINAL) ||
748 (!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
749
750 ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
751 if (unlikely(ret)) {
752 dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
753 goto out_unlock;
754 }
755 kcop->digestsize = ses_ptr->hdata.digestsize;
756 }
757
758 #if defined(CRYPTODEV_STATS)
759 if (enable_stats) {
760 /* this is safe - we check cop->op at the function entry */
761 ses_ptr->stat[cop->op] += cop->len;
762 if (ses_ptr->stat_max_size < cop->len)
763 ses_ptr->stat_max_size = cop->len;
764 ses_ptr->stat_count++;
765 }
766 #endif
767
768 out_unlock:
769 mutex_unlock(&ses_ptr->sem);
770 return ret;
771 }
772
773 static void cryptask_routine(struct work_struct *work)
774 {
775 struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask);
776 struct todo_list_item *item;
777 LIST_HEAD(tmp);
778
779 /* fetch all pending jobs into the temporary list */
780 mutex_lock(&pcr->todo.lock);
781 list_cut_position(&tmp, &pcr->todo.list, pcr->todo.list.prev);
782 mutex_unlock(&pcr->todo.lock);
783
784 /* handle each job locklessly */
785 list_for_each_entry(item, &tmp, __hook) {
786 item->result = crypto_run(&pcr->fcrypt, &item->kcop);
787 if (unlikely(item->result))
788 dprintk(0, KERN_ERR, "%s: crypto_run() failed: %d\n",
789 __func__, item->result);
790 }
791
792 /* push all handled jobs to the done list at once */
793 mutex_lock(&pcr->done.lock);
794 list_splice_tail(&tmp, &pcr->done.list);
795 mutex_unlock(&pcr->done.lock);
796
797 /* wake for POLLIN */
798 wake_up_interruptible(&pcr->user_waiter);
799 }
800
801 /* ====== /dev/crypto ====== */
802
803 static int
804 cryptodev_open(struct inode *inode, struct file *filp)
805 {
806 struct todo_list_item *tmp;
807 struct crypt_priv *pcr;
808 int i;
809
810 pcr = kmalloc(sizeof(*pcr), GFP_KERNEL);
811 if (!pcr)
812 return -ENOMEM;
813
814 memset(pcr, 0, sizeof(*pcr));
815 mutex_init(&pcr->fcrypt.sem);
816 INIT_LIST_HEAD(&pcr->fcrypt.list);
817
818 INIT_LIST_HEAD(&pcr->free.list);
819 INIT_LIST_HEAD(&pcr->todo.list);
820 INIT_LIST_HEAD(&pcr->done.list);
821 INIT_WORK(&pcr->cryptask, cryptask_routine);
822 mutex_init(&pcr->free.lock);
823 mutex_init(&pcr->todo.lock);
824 mutex_init(&pcr->done.lock);
825 init_waitqueue_head(&pcr->user_waiter);
826
827 for (i = 0; i < DEF_COP_RINGSIZE; i++) {
828 tmp = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
829 pcr->itemcount++;
830 dprintk(2, KERN_DEBUG, "%s: allocated new item at %lx\n",
831 __func__, (unsigned long)tmp);
832 list_add(&tmp->__hook, &pcr->free.list);
833 }
834
835 filp->private_data = pcr;
836 dprintk(2, KERN_DEBUG,
837 "Cryptodev handle initialised, %d elements in queue\n",
838 DEF_COP_RINGSIZE);
839 return 0;
840 }
841
842 static int
843 cryptodev_release(struct inode *inode, struct file *filp)
844 {
845 struct crypt_priv *pcr = filp->private_data;
846 struct todo_list_item *item, *item_safe;
847 int items_freed = 0;
848
849 if (!pcr)
850 return 0;
851
852 cancel_work_sync(&pcr->cryptask);
853
854 mutex_destroy(&pcr->todo.lock);
855 mutex_destroy(&pcr->done.lock);
856 mutex_destroy(&pcr->free.lock);
857
858 list_splice_tail(&pcr->todo.list, &pcr->free.list);
859 list_splice_tail(&pcr->done.list, &pcr->free.list);
860
861 list_for_each_entry_safe(item, item_safe, &pcr->free.list, __hook) {
862 dprintk(2, KERN_DEBUG, "%s: freeing item at %lx\n",
863 __func__, (unsigned long)item);
864 list_del(&item->__hook);
865 kfree(item);
866 items_freed++;
867
868 }
869 if (items_freed != pcr->itemcount) {
870 dprintk(0, KERN_ERR,
871 "%s: freed %d items, but %d should exist!\n",
872 __func__, items_freed, pcr->itemcount);
873 }
874
875 crypto_finish_all_sessions(&pcr->fcrypt);
876 kfree(pcr);
877 filp->private_data = NULL;
878
879 dprintk(2, KERN_DEBUG,
880 "Cryptodev handle deinitialised, %d elements freed\n",
881 items_freed);
882 return 0;
883 }
884
885 static int
886 clonefd(struct file *filp)
887 {
888 int ret;
889 ret = get_unused_fd();
890 if (ret >= 0) {
891 get_file(filp);
892 fd_install(ret, filp);
893 }
894
895 return ret;
896 }
897
898 /* enqueue a job for asynchronous completion
899 *
900 * returns:
901 * -EBUSY when there are no free queue slots left
902 * (and the number of slots has reached it MAX_COP_RINGSIZE)
903 * -EFAULT when there was a memory allocation error
904 * 0 on success */
905 static int crypto_async_run(struct crypt_priv *pcr, struct kernel_crypt_op *kcop)
906 {
907 struct todo_list_item *item = NULL;
908
909 mutex_lock(&pcr->free.lock);
910 if (likely(!list_empty(&pcr->free.list))) {
911 item = list_first_entry(&pcr->free.list,
912 struct todo_list_item, __hook);
913 list_del(&item->__hook);
914 } else if (pcr->itemcount < MAX_COP_RINGSIZE) {
915 pcr->itemcount++;
916 } else {
917 mutex_unlock(&pcr->free.lock);
918 return -EBUSY;
919 }
920 mutex_unlock(&pcr->free.lock);
921
922 if (unlikely(!item)) {
923 item = kzalloc(sizeof(struct todo_list_item), GFP_KERNEL);
924 if (unlikely(!item))
925 return -EFAULT;
926 dprintk(1, KERN_INFO, "%s: increased item count to %d\n",
927 __func__, pcr->itemcount);
928 }
929
930 memcpy(&item->kcop, kcop, sizeof(struct kernel_crypt_op));
931
932 mutex_lock(&pcr->todo.lock);
933 list_add_tail(&item->__hook, &pcr->todo.list);
934 mutex_unlock(&pcr->todo.lock);
935
936 queue_work(cryptodev_wq, &pcr->cryptask);
937 return 0;
938 }
939
940 /* get the first completed job from the "done" queue
941 *
942 * returns:
943 * -EBUSY if no completed jobs are ready (yet)
944 * the return value of crypto_run() otherwise */
945 static int crypto_async_fetch(struct crypt_priv *pcr,
946 struct kernel_crypt_op *kcop)
947 {
948 struct todo_list_item *item;
949 int retval;
950
951 mutex_lock(&pcr->done.lock);
952 if (list_empty(&pcr->done.list)) {
953 mutex_unlock(&pcr->done.lock);
954 return -EBUSY;
955 }
956 item = list_first_entry(&pcr->done.list, struct todo_list_item, __hook);
957 list_del(&item->__hook);
958 mutex_unlock(&pcr->done.lock);
959
960 memcpy(kcop, &item->kcop, sizeof(struct kernel_crypt_op));
961 retval = item->result;
962
963 mutex_lock(&pcr->free.lock);
964 list_add_tail(&item->__hook, &pcr->free.list);
965 mutex_unlock(&pcr->free.lock);
966
967 /* wake for POLLOUT */
968 wake_up_interruptible(&pcr->user_waiter);
969
970 return retval;
971 }
972
973 /* this function has to be called from process context */
974 static int fill_kcop_from_cop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
975 {
976 struct crypt_op *cop = &kcop->cop;
977 struct csession *ses_ptr;
978 int rc;
979
980 /* this also enters ses_ptr->sem */
981 ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
982 if (unlikely(!ses_ptr)) {
983 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
984 return -EINVAL;
985 }
986 kcop->ivlen = cop->iv ? ses_ptr->cdata.ivsize : 0;
987 kcop->digestsize = 0; /* will be updated during operation */
988
989 mutex_unlock(&ses_ptr->sem);
990
991 kcop->task = current;
992 kcop->mm = current->mm;
993
994 if (cop->iv) {
995 rc = copy_from_user(kcop->iv, cop->iv, kcop->ivlen);
996 if (unlikely(rc)) {
997 dprintk(1, KERN_ERR,
998 "error copying IV (%d bytes), copy_from_user returned %d for address %lx\n",
999 kcop->ivlen, rc, (unsigned long)cop->iv);
1000 return -EFAULT;
1001 }
1002 }
1003
1004 return 0;
1005 }
1006
1007 /* this function has to be called from process context */
1008 static int fill_cop_from_kcop(struct kernel_crypt_op *kcop, struct fcrypt *fcr)
1009 {
1010 int ret;
1011
1012 if (kcop->digestsize) {
1013 ret = copy_to_user(kcop->cop.mac,
1014 kcop->hash_output, kcop->digestsize);
1015 if (unlikely(ret))
1016 return -EFAULT;
1017 }
1018 if (kcop->ivlen && kcop->cop.flags & COP_FLAG_WRITE_IV) {
1019 ret = copy_to_user(kcop->cop.iv,
1020 kcop->iv, kcop->ivlen);
1021 if (unlikely(ret))
1022 return -EFAULT;
1023 }
1024 return 0;
1025 }
1026
1027 static int kcop_from_user(struct kernel_crypt_op *kcop,
1028 struct fcrypt *fcr, void __user *arg)
1029 {
1030 if (unlikely(copy_from_user(&kcop->cop, arg, sizeof(kcop->cop))))
1031 return -EFAULT;
1032
1033 return fill_kcop_from_cop(kcop, fcr);
1034 }
1035
1036 static int kcop_to_user(struct kernel_crypt_op *kcop,
1037 struct fcrypt *fcr, void __user *arg)
1038 {
1039 int ret;
1040
1041 ret = fill_cop_from_kcop(kcop, fcr);
1042 if (unlikely(ret))
1043 return ret;
1044
1045 if (unlikely(copy_to_user(arg, &kcop->cop, sizeof(kcop->cop))))
1046 return -EFAULT;
1047 return 0;
1048 }
1049
1050 static inline void tfm_info_to_alg_info(struct alg_info *dst, struct crypto_tfm *tfm)
1051 {
1052 snprintf(dst->cra_name, CRYPTODEV_MAX_ALG_NAME,
1053 "%s", crypto_tfm_alg_name(tfm));
1054 snprintf(dst->cra_driver_name, CRYPTODEV_MAX_ALG_NAME,
1055 "%s", crypto_tfm_alg_driver_name(tfm));
1056 }
1057
1058 static int get_session_info(struct fcrypt *fcr, struct session_info_op *siop)
1059 {
1060 struct csession *ses_ptr;
1061
1062 /* this also enters ses_ptr->sem */
1063 ses_ptr = crypto_get_session_by_sid(fcr, siop->ses);
1064 if (unlikely(!ses_ptr)) {
1065 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", siop->ses);
1066 return -EINVAL;
1067 }
1068
1069 if (ses_ptr->cdata.init) {
1070 tfm_info_to_alg_info(&siop->cipher_info,
1071 crypto_ablkcipher_tfm(ses_ptr->cdata.async.s));
1072 }
1073 if (ses_ptr->hdata.init) {
1074 tfm_info_to_alg_info(&siop->hash_info,
1075 crypto_ahash_tfm(ses_ptr->hdata.async.s));
1076 }
1077
1078 siop->alignmask = ses_ptr->alignmask;
1079
1080 mutex_unlock(&ses_ptr->sem);
1081 return 0;
1082 }
1083
1084 static long
1085 cryptodev_ioctl(struct file *filp, unsigned int cmd, unsigned long arg_)
1086 {
1087 void __user *arg = (void __user *)arg_;
1088 int __user *p = arg;
1089 struct session_op sop;
1090 struct kernel_crypt_op kcop;
1091 struct crypt_priv *pcr = filp->private_data;
1092 struct fcrypt *fcr;
1093 struct session_info_op siop;
1094 uint32_t ses;
1095 int ret, fd;
1096
1097 if (unlikely(!pcr))
1098 BUG();
1099
1100 fcr = &pcr->fcrypt;
1101
1102 switch (cmd) {
1103 case CIOCASYMFEAT:
1104 return put_user(0, p);
1105 case CRIOGET:
1106 fd = clonefd(filp);
1107 ret = put_user(fd, p);
1108 if (unlikely(ret)) {
1109 sys_close(fd);
1110 return ret;
1111 }
1112 return ret;
1113 case CIOCGSESSION:
1114 if (unlikely(copy_from_user(&sop, arg, sizeof(sop))))
1115 return -EFAULT;
1116
1117 ret = crypto_create_session(fcr, &sop);
1118 if (unlikely(ret))
1119 return ret;
1120 ret = copy_to_user(arg, &sop, sizeof(sop));
1121 if (unlikely(ret)) {
1122 crypto_finish_session(fcr, sop.ses);
1123 return -EFAULT;
1124 }
1125 return ret;
1126 case CIOCFSESSION:
1127 ret = get_user(ses, (uint32_t __user *)arg);
1128 if (unlikely(ret))
1129 return ret;
1130 ret = crypto_finish_session(fcr, ses);
1131 return ret;
1132 case CIOCGSESSINFO:
1133 if (unlikely(copy_from_user(&siop, arg, sizeof(siop))))
1134 return -EFAULT;
1135
1136 ret = get_session_info(fcr, &siop);
1137 if (unlikely(ret))
1138 return ret;
1139 return copy_to_user(arg, &siop, sizeof(siop));
1140 case CIOCCRYPT:
1141 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
1142 return ret;
1143
1144 ret = crypto_run(fcr, &kcop);
1145 if (unlikely(ret))
1146 return ret;
1147
1148 return kcop_to_user(&kcop, fcr, arg);
1149 case CIOCASYNCCRYPT:
1150 if (unlikely(ret = kcop_from_user(&kcop, fcr, arg)))
1151 return ret;
1152
1153 return crypto_async_run(pcr, &kcop);
1154 case CIOCASYNCFETCH:
1155 ret = crypto_async_fetch(pcr, &kcop);
1156 if (unlikely(ret))
1157 return ret;
1158
1159 return kcop_to_user(&kcop, fcr, arg);
1160 default:
1161 return -EINVAL;
1162 }
1163 }
1164
1165 /* compatibility code for 32bit userlands */
1166 #ifdef CONFIG_COMPAT
1167
1168 static inline void
1169 compat_to_session_op(struct compat_session_op *compat, struct session_op *sop)
1170 {
1171 sop->cipher = compat->cipher;
1172 sop->mac = compat->mac;
1173 sop->keylen = compat->keylen;
1174
1175 sop->key = compat_ptr(compat->key);
1176 sop->mackeylen = compat->mackeylen;
1177 sop->mackey = compat_ptr(compat->mackey);
1178 sop->ses = compat->ses;
1179 }
1180
1181 static inline void
1182 session_op_to_compat(struct session_op *sop, struct compat_session_op *compat)
1183 {
1184 compat->cipher = sop->cipher;
1185 compat->mac = sop->mac;
1186 compat->keylen = sop->keylen;
1187
1188 compat->key = ptr_to_compat(sop->key);
1189 compat->mackeylen = sop->mackeylen;
1190 compat->mackey = ptr_to_compat(sop->mackey);
1191 compat->ses = sop->ses;
1192 }
1193
1194 static inline void
1195 compat_to_crypt_op(struct compat_crypt_op *compat, struct crypt_op *cop)
1196 {
1197 cop->ses = compat->ses;
1198 cop->op = compat->op;
1199 cop->flags = compat->flags;
1200 cop->len = compat->len;
1201
1202 cop->src = compat_ptr(compat->src);
1203 cop->dst = compat_ptr(compat->dst);
1204 cop->mac = compat_ptr(compat->mac);
1205 cop->iv = compat_ptr(compat->iv);
1206 }
1207
1208 static inline void
1209 crypt_op_to_compat(struct crypt_op *cop, struct compat_crypt_op *compat)
1210 {
1211 compat->ses = cop->ses;
1212 compat->op = cop->op;
1213 compat->flags = cop->flags;
1214 compat->len = cop->len;
1215
1216 compat->src = ptr_to_compat(cop->src);
1217 compat->dst = ptr_to_compat(cop->dst);
1218 compat->mac = ptr_to_compat(cop->mac);
1219 compat->iv = ptr_to_compat(cop->iv);
1220 }
1221
1222 static int compat_kcop_from_user(struct kernel_crypt_op *kcop,
1223 struct fcrypt *fcr, void __user *arg)
1224 {
1225 struct compat_crypt_op compat_cop;
1226
1227 if (unlikely(copy_from_user(&compat_cop, arg, sizeof(compat_cop))))
1228 return -EFAULT;
1229 compat_to_crypt_op(&compat_cop, &kcop->cop);
1230
1231 return fill_kcop_from_cop(kcop, fcr);
1232 }
1233
1234 static int compat_kcop_to_user(struct kernel_crypt_op *kcop,
1235 struct fcrypt *fcr, void __user *arg)
1236 {
1237 int ret;
1238 struct compat_crypt_op compat_cop;
1239
1240 ret = fill_cop_from_kcop(kcop, fcr);
1241 if (unlikely(ret))
1242 return ret;
1243 crypt_op_to_compat(&kcop->cop, &compat_cop);
1244
1245 if (unlikely(copy_to_user(arg, &compat_cop, sizeof(compat_cop))))
1246 return -EFAULT;
1247 return 0;
1248 }
1249
1250 static long
1251 cryptodev_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg_)
1252 {
1253 void __user *arg = (void __user *)arg_;
1254 struct crypt_priv *pcr = file->private_data;
1255 struct fcrypt *fcr;
1256 struct session_op sop;
1257 struct compat_session_op compat_sop;
1258 struct kernel_crypt_op kcop;
1259 int ret;
1260
1261 if (unlikely(!pcr))
1262 BUG();
1263
1264 fcr = &pcr->fcrypt;
1265
1266 switch (cmd) {
1267 case CIOCASYMFEAT:
1268 case CRIOGET:
1269 case CIOCFSESSION:
1270 return cryptodev_ioctl(file, cmd, arg_);
1271
1272 case COMPAT_CIOCGSESSION:
1273 if (unlikely(copy_from_user(&compat_sop, arg,
1274 sizeof(compat_sop))))
1275 return -EFAULT;
1276 compat_to_session_op(&compat_sop, &sop);
1277
1278 ret = crypto_create_session(fcr, &sop);
1279 if (unlikely(ret))
1280 return ret;
1281
1282 session_op_to_compat(&sop, &compat_sop);
1283 ret = copy_to_user(arg, &compat_sop, sizeof(compat_sop));
1284 if (unlikely(ret)) {
1285 crypto_finish_session(fcr, sop.ses);
1286 return -EFAULT;
1287 }
1288 return ret;
1289
1290 case COMPAT_CIOCCRYPT:
1291 ret = compat_kcop_from_user(&kcop, fcr, arg);
1292 if (unlikely(ret))
1293 return ret;
1294
1295 ret = crypto_run(fcr, &kcop);
1296 if (unlikely(ret))
1297 return ret;
1298
1299 return compat_kcop_to_user(&kcop, fcr, arg);
1300 case COMPAT_CIOCASYNCCRYPT:
1301 if (unlikely(ret = compat_kcop_from_user(&kcop, fcr, arg)))
1302 return ret;
1303
1304 return crypto_async_run(pcr, &kcop);
1305 case COMPAT_CIOCASYNCFETCH:
1306 ret = crypto_async_fetch(pcr, &kcop);
1307 if (unlikely(ret))
1308 return ret;
1309
1310 return compat_kcop_to_user(&kcop, fcr, arg);
1311
1312 default:
1313 return -EINVAL;
1314 }
1315 }
1316
1317 #endif /* CONFIG_COMPAT */
1318
1319 static unsigned int cryptodev_poll(struct file *file, poll_table *wait)
1320 {
1321 struct crypt_priv *pcr = file->private_data;
1322 int ret = 0;
1323
1324 poll_wait(file, &pcr->user_waiter, wait);
1325
1326 if (!list_empty_careful(&pcr->done.list))
1327 ret |= POLLIN | POLLRDNORM;
1328 if (!list_empty_careful(&pcr->free.list) || pcr->itemcount < MAX_COP_RINGSIZE)
1329 ret |= POLLOUT | POLLWRNORM;
1330
1331 return ret;
1332 }
1333
1334 static const struct file_operations cryptodev_fops = {
1335 .owner = THIS_MODULE,
1336 .open = cryptodev_open,
1337 .release = cryptodev_release,
1338 .unlocked_ioctl = cryptodev_ioctl,
1339 #ifdef CONFIG_COMPAT
1340 .compat_ioctl = cryptodev_compat_ioctl,
1341 #endif /* CONFIG_COMPAT */
1342 .poll = cryptodev_poll,
1343 };
1344
1345 static struct miscdevice cryptodev = {
1346 .minor = MISC_DYNAMIC_MINOR,
1347 .name = "crypto",
1348 .fops = &cryptodev_fops,
1349 };
1350
1351 static int __init
1352 cryptodev_register(void)
1353 {
1354 int rc;
1355
1356 rc = misc_register(&cryptodev);
1357 if (unlikely(rc)) {
1358 printk(KERN_ERR PFX "registration of /dev/crypto failed\n");
1359 return rc;
1360 }
1361
1362 return 0;
1363 }
1364
1365 static void __exit
1366 cryptodev_deregister(void)
1367 {
1368 misc_deregister(&cryptodev);
1369 }
1370
1371 /* ====== Module init/exit ====== */
1372 static int __init init_cryptodev(void)
1373 {
1374 int rc;
1375
1376 cryptodev_wq = create_workqueue("cryptodev_queue");
1377 if (unlikely(!cryptodev_wq)) {
1378 printk(KERN_ERR PFX "failed to allocate the cryptodev workqueue\n");
1379 return -EFAULT;
1380 }
1381
1382 rc = cryptodev_register();
1383 if (unlikely(rc)) {
1384 destroy_workqueue(cryptodev_wq);
1385 return rc;
1386 }
1387
1388 printk(KERN_INFO PFX "driver %s loaded.\n", VERSION);
1389
1390 return 0;
1391 }
1392
1393 static void __exit exit_cryptodev(void)
1394 {
1395 flush_workqueue(cryptodev_wq);
1396 destroy_workqueue(cryptodev_wq);
1397
1398 cryptodev_deregister();
1399 printk(KERN_INFO PFX "driver unloaded.\n");
1400 }
1401
1402 module_init(init_cryptodev);
1403 module_exit(exit_cryptodev);
1404
Cryptodev for linux