?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Initial commit
[wrt350n-kernel.git] / arch / cris / arch-v32 / drivers / cryptocop.c
blob9fb58202be99b8d54d729c161dba9983cc29352d
1 /*
2 * Stream co-processor driver for the ETRAX FS
3 *
4 * Copyright (C) 2003-2007 Axis Communications AB
5 */
6
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <asm/atomic.h>
20
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50
51 #define DESCR_ALLOC_PAD (31)
52
53 struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61 };
62
63
64 struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82 };
83
84
85 struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120 };
121
122
123 struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126 };
127
128 /* Session list. */
129
130 struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136 };
137
138
139 struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145 };
146
147 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150 typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156
157 struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160 };
161
162 struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168 };
169
170 struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172 };
173
174
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197 of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206 * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211
212
213 /** Local functions. **/
214
215 static int cryptocop_open(struct inode *, struct file *);
216
217 static int cryptocop_release(struct inode *, struct file *);
218
219 static int cryptocop_ioctl(struct inode *inode, struct file *file,
220 unsigned int cmd, unsigned long arg);
221
222 static void cryptocop_start_job(void);
223
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
243
244 static int init_stream_coprocessor(void);
245
246 static void __exit exit_stream_coprocessor(void);
247
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263
264
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH (8)
267 #define AES_BLOCK_LENGTH (16)
268 #define MD5_BLOCK_LENGTH (64)
269 #define SHA1_BLOCK_LENGTH (64)
270 #define CSUM_BLOCK_LENGTH (2)
271 #define MD5_STATE_LENGTH (16)
272 #define SHA1_STATE_LENGTH (20)
273
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR (254)
276 #define CRYPTOCOP_MINOR (0)
277
278
279
280 const struct file_operations cryptocop_fops = {
281 .owner = THIS_MODULE,
282 .open = cryptocop_open,
283 .release = cryptocop_release,
284 .ioctl = cryptocop_ioctl
285 };
286
287
288 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
289 {
290 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
291 kfree(cdesc->free_buf);
292
293 if (cdesc->from_pool) {
294 unsigned long int flags;
295 spin_lock_irqsave(&descr_pool_lock, flags);
296 cdesc->next = descr_pool_free_list;
297 descr_pool_free_list = cdesc;
298 ++descr_pool_no_free;
299 spin_unlock_irqrestore(&descr_pool_lock, flags);
300 } else {
301 kfree(cdesc);
302 }
303 }
304
305
306 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
307 {
308 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
309 struct cryptocop_dma_desc *cdesc;
310
311 if (use_pool) {
312 unsigned long int flags;
313 spin_lock_irqsave(&descr_pool_lock, flags);
314 if (!descr_pool_free_list) {
315 spin_unlock_irqrestore(&descr_pool_lock, flags);
316 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
317 return NULL;
318 }
319 cdesc = descr_pool_free_list;
320 descr_pool_free_list = descr_pool_free_list->next;
321 --descr_pool_no_free;
322 spin_unlock_irqrestore(&descr_pool_lock, flags);
323 cdesc->from_pool = 1;
324 } else {
325 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
326 if (!cdesc) {
327 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
328 return NULL;
329 }
330 cdesc->from_pool = 0;
331 }
332 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
333
334 cdesc->next = NULL;
335
336 cdesc->free_buf = NULL;
337 cdesc->dma_descr->out_eop = 0;
338 cdesc->dma_descr->in_eop = 0;
339 cdesc->dma_descr->intr = 0;
340 cdesc->dma_descr->eol = 0;
341 cdesc->dma_descr->wait = 0;
342 cdesc->dma_descr->buf = NULL;
343 cdesc->dma_descr->after = NULL;
344
345 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
346 return cdesc;
347 }
348
349
350 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
351 {
352 DEBUG(printk("setup_descr_chain: entering\n"));
353 while (cd) {
354 if (cd->next) {
355 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
356 } else {
357 cd->dma_descr->next = NULL;
358 }
359 cd = cd->next;
360 }
361 DEBUG(printk("setup_descr_chain: exit\n"));
362 }
363
364
365 /* Create a pad descriptor for the transform.
366 * Return -1 for error, 0 if pad created. */
367 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
368 {
369 struct cryptocop_dma_desc *cdesc = NULL;
370 int error = 0;
371 struct strcop_meta_out mo = {
372 .ciphsel = src_none,
373 .hashsel = src_none,
374 .csumsel = src_none
375 };
376 char *pad;
377 size_t plen;
378
379 DEBUG(printk("create_pad_descriptor: start.\n"));
380 /* Setup pad descriptor. */
381
382 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
383 cdesc = alloc_cdesc(alloc_flag);
384 if (!cdesc){
385 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
386 goto error_cleanup;
387 }
388 switch (tc->unit_no) {
389 case src_md5:
390 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
391 if (error){
392 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
393 goto error_cleanup;
394 }
395 cdesc->free_buf = pad;
396 mo.hashsel = src_dma;
397 mo.hashconf = tc->hash_conf;
398 mo.hashmode = tc->hash_mode;
399 break;
400 case src_sha1:
401 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
402 if (error){
403 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
404 goto error_cleanup;
405 }
406 cdesc->free_buf = pad;
407 mo.hashsel = src_dma;
408 mo.hashconf = tc->hash_conf;
409 mo.hashmode = tc->hash_mode;
410 break;
411 case src_csum:
412 if (tc->consumed % tc->blocklength){
413 pad = (char*)csum_zero_pad;
414 plen = 1;
415 } else {
416 pad = (char*)cdesc; /* Use any pointer. */
417 plen = 0;
418 }
419 mo.csumsel = src_dma;
420 break;
421 }
422 cdesc->dma_descr->wait = 1;
423 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
424 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
425 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
426
427 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
428 *pad_desc = cdesc;
429
430 return 0;
431
432 error_cleanup:
433 if (cdesc) free_cdesc(cdesc);
434 return -1;
435 }
436
437
438 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
439 {
440 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
441 struct strcop_meta_out mo = {0};
442
443 DEBUG(printk("setup_key_dl_desc\n"));
444
445 if (!key_desc) {
446 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
447 return -ENOMEM;
448 }
449
450 /* Download key. */
451 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
452 /* Precook the AES decrypt key. */
453 if (!tc->tctx->dec_key_set){
454 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
455 tc->tctx->dec_key_set = 1;
456 }
457 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
458 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
459 } else {
460 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
461 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
462 }
463 /* Setup metadata. */
464 mo.dlkey = 1;
465 switch (tc->tctx->init.keylen) {
466 case 64:
467 mo.decrypt = 0;
468 mo.hashmode = 0;
469 break;
470 case 128:
471 mo.decrypt = 0;
472 mo.hashmode = 1;
473 break;
474 case 192:
475 mo.decrypt = 1;
476 mo.hashmode = 0;
477 break;
478 case 256:
479 mo.decrypt = 1;
480 mo.hashmode = 1;
481 break;
482 default:
483 break;
484 }
485 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
486 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
487
488 key_desc->dma_descr->out_eop = 1;
489 key_desc->dma_descr->wait = 1;
490 key_desc->dma_descr->intr = 0;
491
492 *kd = key_desc;
493 return 0;
494 }
495
496 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
497 {
498 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
499 struct strcop_meta_out mo = {0};
500
501 DEBUG(printk("setup_cipher_iv_desc\n"));
502
503 if (!iv_desc) {
504 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
505 return -ENOMEM;
506 }
507 /* Download IV. */
508 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
509 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
510
511 /* Setup metadata. */
512 mo.hashsel = mo.csumsel = src_none;
513 mo.ciphsel = src_dma;
514 mo.ciphconf = tc->ciph_conf;
515 mo.cbcmode = tc->cbcmode;
516
517 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
518
519 iv_desc->dma_descr->out_eop = 0;
520 iv_desc->dma_descr->wait = 1;
521 iv_desc->dma_descr->intr = 0;
522
523 *id = iv_desc;
524 return 0;
525 }
526
527 /* Map the ouput length of the transform to operation output starting on the inject index. */
528 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
529 {
530 int err = 0;
531 struct cryptocop_dma_desc head = {0};
532 struct cryptocop_dma_desc *outdesc = &head;
533 size_t iov_offset = 0;
534 size_t out_ix = 0;
535 int outiov_ix = 0;
536 struct strcop_meta_in mi = {0};
537
538 size_t out_length = tc->produced;
539 int rem_length;
540 int dlength;
541
542 assert(out_length != 0);
543 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
544 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
545 return -EINVAL;
546 }
547 /* Traverse the out iovec until the result inject index is reached. */
548 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
549 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
550 outiov_ix++;
551 }
552 if (outiov_ix >= operation->tfrm_op.outcount){
553 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
554 return -EINVAL;
555 }
556 iov_offset = tc->tcfg->inject_ix - out_ix;
557 mi.dmasel = tc->unit_no;
558
559 /* Setup the output descriptors. */
560 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
561 outdesc->next = alloc_cdesc(alloc_flag);
562 if (!outdesc->next) {
563 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
564 err = -ENOMEM;
565 goto error_cleanup;
566 }
567 outdesc = outdesc->next;
568 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
569 dlength = (out_length < rem_length) ? out_length : rem_length;
570
571 DEBUG(printk("create_input_descriptors:\n"
572 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
573 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
574 "outcount=%d, outiov_ix=%d\n",
575 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
576
577 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
578 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
579 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
580
581 out_length -= dlength;
582 iov_offset += dlength;
583 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
584 iov_offset = 0;
585 ++outiov_ix;
586 }
587 }
588 if (out_length > 0){
589 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
590 err = -EINVAL;
591 goto error_cleanup;
592 }
593 /* Set sync in last descriptor. */
594 mi.sync = 1;
595 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
596
597 *id = head.next;
598 return 0;
599
600 error_cleanup:
601 while (head.next) {
602 outdesc = head.next->next;
603 free_cdesc(head.next);
604 head.next = outdesc;
605 }
606 return err;
607 }
608
609
610 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
611 {
612 while (desc_len != 0) {
613 struct cryptocop_dma_desc *cdesc;
614 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
615 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
616
617 cdesc = alloc_cdesc(alloc_flag);
618 if (!cdesc) {
619 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
620 return -ENOMEM;
621 }
622 (*current_out_cdesc)->next = cdesc;
623 (*current_out_cdesc) = cdesc;
624
625 cdesc->free_buf = NULL;
626
627 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
628 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
629
630 desc_len -= dlength;
631 *iniov_offset += dlength;
632 assert(desc_len >= 0);
633 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
634 *iniov_offset = 0;
635 ++(*iniov_ix);
636 if (*iniov_ix > operation->tfrm_op.incount) {
637 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
638 return -EINVAL;
639 }
640 }
641 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
642 } /* while (desc_len != 0) */
643 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
644 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
645
646 return 0;
647 }
648
649
650 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
651 {
652 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
653 if (tc->tcfg) {
654 int failed = 0;
655 struct cryptocop_dma_desc *idescs = NULL;
656 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
657 if (tc->pad_descs) {
658 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
659 while (tc->pad_descs) {
660 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
661 (*current_out_cdesc)->next = tc->pad_descs;
662 tc->pad_descs = tc->pad_descs->next;
663 (*current_out_cdesc) = (*current_out_cdesc)->next;
664 }
665 }
666
667 /* Setup and append output descriptors to DMA in list. */
668 if (tc->unit_no == src_dma){
669 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
670 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
671 unsigned int start_ix = tc->start_ix;
672 while (start_ix){
673 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
674 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
675 if (!(*current_in_cdesc)->next){
676 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
677 return -ENOMEM;
678 }
679 (*current_in_cdesc) = (*current_in_cdesc)->next;
680 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
681 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
682 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
683 start_ix -= desclen;
684 }
685 mi.sync = 1;
686 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
687 }
688
689 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
690 if (failed){
691 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
692 return failed;
693 }
694 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
695 while (idescs) {
696 DEBUG(printk("append descriptor 0x%p\n", idescs));
697 (*current_in_cdesc)->next = idescs;
698 idescs = idescs->next;
699 (*current_in_cdesc) = (*current_in_cdesc)->next;
700 }
701 }
702 return 0;
703 }
704
705
706
707 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
708 {
709 struct cryptocop_session *sess;
710 struct cryptocop_transform_ctx *tctx;
711
712 struct cryptocop_tfrm_ctx digest_ctx = {
713 .previous_src = src_none,
714 .current_src = src_none,
715 .start_ix = 0,
716 .requires_padding = 1,
717 .strict_block_length = 0,
718 .hash_conf = 0,
719 .hash_mode = 0,
720 .ciph_conf = 0,
721 .cbcmode = 0,
722 .decrypt = 0,
723 .consumed = 0,
724 .produced = 0,
725 .pad_descs = NULL,
726 .active = 0,
727 .done = 0,
728 .prev_src = NULL,
729 .curr_src = NULL,
730 .tcfg = NULL};
731 struct cryptocop_tfrm_ctx cipher_ctx = {
732 .previous_src = src_none,
733 .current_src = src_none,
734 .start_ix = 0,
735 .requires_padding = 0,
736 .strict_block_length = 1,
737 .hash_conf = 0,
738 .hash_mode = 0,
739 .ciph_conf = 0,
740 .cbcmode = 0,
741 .decrypt = 0,
742 .consumed = 0,
743 .produced = 0,
744 .pad_descs = NULL,
745 .active = 0,
746 .done = 0,
747 .prev_src = NULL,
748 .curr_src = NULL,
749 .tcfg = NULL};
750 struct cryptocop_tfrm_ctx csum_ctx = {
751 .previous_src = src_none,
752 .current_src = src_none,
753 .start_ix = 0,
754 .blocklength = 2,
755 .requires_padding = 1,
756 .strict_block_length = 0,
757 .hash_conf = 0,
758 .hash_mode = 0,
759 .ciph_conf = 0,
760 .cbcmode = 0,
761 .decrypt = 0,
762 .consumed = 0,
763 .produced = 0,
764 .pad_descs = NULL,
765 .active = 0,
766 .done = 0,
767 .tcfg = NULL,
768 .prev_src = NULL,
769 .curr_src = NULL,
770 .unit_no = src_csum};
771 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
772
773 unsigned int indata_ix = 0;
774
775 /* iovec accounting. */
776 int iniov_ix = 0;
777 int iniov_offset = 0;
778
779 /* Operation descriptor cfg traversal pointer. */
780 struct cryptocop_desc *odsc;
781
782 int failed = 0;
783 /* List heads for allocated descriptors. */
784 struct cryptocop_dma_desc out_cdesc_head = {0};
785 struct cryptocop_dma_desc in_cdesc_head = {0};
786
787 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
788 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
789
790 struct cryptocop_tfrm_ctx *output_tc = NULL;
791 void *iop_alloc_ptr;
792
793 assert(operation != NULL);
794 assert(int_op != NULL);
795
796 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
797 DEBUG(print_cryptocop_operation(operation));
798
799 sess = get_session(operation->sid);
800 if (!sess) {
801 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
802 failed = -EINVAL;
803 goto error_cleanup;
804 }
805 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
806 if (!iop_alloc_ptr) {
807 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
808 failed = -ENOMEM;
809 goto error_cleanup;
810 }
811 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
812 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
813 (*int_op)->alloc_ptr = iop_alloc_ptr;
814 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
815
816 (*int_op)->sid = operation->sid;
817 (*int_op)->cdesc_out = NULL;
818 (*int_op)->cdesc_in = NULL;
819 (*int_op)->tdes_mode = cryptocop_3des_ede;
820 (*int_op)->csum_mode = cryptocop_csum_le;
821 (*int_op)->ddesc_out = NULL;
822 (*int_op)->ddesc_in = NULL;
823
824 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
825 if (!tcfg) {
826 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
827 failed = -EINVAL;
828 goto error_cleanup;
829 }
830 while (tcfg) {
831 tctx = get_transform_ctx(sess, tcfg->tid);
832 if (!tctx) {
833 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
834 failed = -EINVAL;
835 goto error_cleanup;
836 }
837 if (tcfg->inject_ix > operation->tfrm_op.outlen){
838 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
839 failed = -EINVAL;
840 goto error_cleanup;
841 }
842 switch (tctx->init.alg){
843 case cryptocop_alg_mem2mem:
844 if (cipher_ctx.tcfg != NULL){
845 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
846 failed = -EINVAL;
847 goto error_cleanup;
848 }
849 /* mem2mem is handled as a NULL cipher. */
850 cipher_ctx.cbcmode = 0;
851 cipher_ctx.decrypt = 0;
852 cipher_ctx.blocklength = 1;
853 cipher_ctx.ciph_conf = 0;
854 cipher_ctx.unit_no = src_dma;
855 cipher_ctx.tcfg = tcfg;
856 cipher_ctx.tctx = tctx;
857 break;
858 case cryptocop_alg_des:
859 case cryptocop_alg_3des:
860 case cryptocop_alg_aes:
861 /* cipher */
862 if (cipher_ctx.tcfg != NULL){
863 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
864 failed = -EINVAL;
865 goto error_cleanup;
866 }
867 cipher_ctx.tcfg = tcfg;
868 cipher_ctx.tctx = tctx;
869 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
870 cipher_ctx.decrypt = 1;
871 }
872 switch (tctx->init.cipher_mode) {
873 case cryptocop_cipher_mode_ecb:
874 cipher_ctx.cbcmode = 0;
875 break;
876 case cryptocop_cipher_mode_cbc:
877 cipher_ctx.cbcmode = 1;
878 break;
879 default:
880 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
881 failed = -EINVAL;
882 goto error_cleanup;
883 }
884 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
885 switch (tctx->init.alg){
886 case cryptocop_alg_des:
887 cipher_ctx.ciph_conf = 0;
888 cipher_ctx.unit_no = src_des;
889 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
890 break;
891 case cryptocop_alg_3des:
892 cipher_ctx.ciph_conf = 1;
893 cipher_ctx.unit_no = src_des;
894 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
895 break;
896 case cryptocop_alg_aes:
897 cipher_ctx.ciph_conf = 2;
898 cipher_ctx.unit_no = src_aes;
899 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
900 break;
901 default:
902 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
903 }
904 (*int_op)->tdes_mode = tctx->init.tdes_mode;
905 break;
906 case cryptocop_alg_md5:
907 case cryptocop_alg_sha1:
908 /* digest */
909 if (digest_ctx.tcfg != NULL){
910 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
911 failed = -EINVAL;
912 goto error_cleanup;
913 }
914 digest_ctx.tcfg = tcfg;
915 digest_ctx.tctx = tctx;
916 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
917 switch (tctx->init.alg){
918 case cryptocop_alg_md5:
919 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
920 digest_ctx.unit_no = src_md5;
921 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
922 break;
923 case cryptocop_alg_sha1:
924 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
925 digest_ctx.unit_no = src_sha1;
926 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
927 break;
928 default:
929 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
930 }
931 break;
932 case cryptocop_alg_csum:
933 /* digest */
934 if (csum_ctx.tcfg != NULL){
935 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
936 failed = -EINVAL;
937 goto error_cleanup;
938 }
939 (*int_op)->csum_mode = tctx->init.csum_mode;
940 csum_ctx.tcfg = tcfg;
941 csum_ctx.tctx = tctx;
942 break;
943 default:
944 /* no algorithm. */
945 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
946 failed = -EINVAL;
947 goto error_cleanup;
948 }
949 tcfg = tcfg->next;
950 }
951 /* Download key if a cipher is used. */
952 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
953 struct cryptocop_dma_desc *key_desc = NULL;
954
955 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
956 if (failed) {
957 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
958 goto error_cleanup;
959 }
960 current_out_cdesc->next = key_desc;
961 current_out_cdesc = key_desc;
962 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
963
964 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
965 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
966 struct cryptocop_dma_desc *iv_desc = NULL;
967
968 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
969
970 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
971 if (failed) {
972 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
973 goto error_cleanup;
974 }
975 current_out_cdesc->next = iv_desc;
976 current_out_cdesc = iv_desc;
977 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
978 }
979 }
980
981 /* Process descriptors. */
982 odsc = operation->tfrm_op.desc;
983 while (odsc) {
984 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
985 struct strcop_meta_out meta_out = {0};
986 size_t desc_len = odsc->length;
987 int active_count, eop_needed_count;
988
989 output_tc = NULL;
990
991 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
992
993 while (dcfg) {
994 struct cryptocop_tfrm_ctx *tc = NULL;
995
996 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
997 /* Get the local context for the transform and mark it as the output unit if it produces output. */
998 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
999 tc = &digest_ctx;
1000 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1001 tc = &cipher_ctx;
1002 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1003 tc = &csum_ctx;
1004 }
1005 if (!tc) {
1006 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1007 failed = -EINVAL;
1008 goto error_cleanup;
1009 }
1010 if (tc->done) {
1011 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1012 failed = -EINVAL;
1013 goto error_cleanup;
1014 }
1015 if (!tc->active) {
1016 tc->start_ix = indata_ix;
1017 tc->active = 1;
1018 }
1019
1020 tc->previous_src = tc->current_src;
1021 tc->prev_src = tc->curr_src;
1022 /* Map source unit id to DMA source config. */
1023 switch (dcfg->src){
1024 case cryptocop_source_dma:
1025 tc->current_src = src_dma;
1026 break;
1027 case cryptocop_source_des:
1028 tc->current_src = src_des;
1029 break;
1030 case cryptocop_source_3des:
1031 tc->current_src = src_des;
1032 break;
1033 case cryptocop_source_aes:
1034 tc->current_src = src_aes;
1035 break;
1036 case cryptocop_source_md5:
1037 case cryptocop_source_sha1:
1038 case cryptocop_source_csum:
1039 case cryptocop_source_none:
1040 default:
1041 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1042 */
1043 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1044 failed = -EINVAL;
1045 goto error_cleanup;
1046 }
1047 if (tc->current_src != src_dma) {
1048 /* Find the unit we are sourcing from. */
1049 if (digest_ctx.unit_no == tc->current_src){
1050 tc->curr_src = &digest_ctx;
1051 } else if (cipher_ctx.unit_no == tc->current_src){
1052 tc->curr_src = &cipher_ctx;
1053 } else if (csum_ctx.unit_no == tc->current_src){
1054 tc->curr_src = &csum_ctx;
1055 }
1056 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1057 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1058 failed = -EINVAL;
1059 goto error_cleanup;
1060 }
1061 } else {
1062 tc->curr_src = NULL;
1063 }
1064
1065 /* Detect source switch. */
1066 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1067 if (tc->active && (tc->current_src != tc->previous_src)) {
1068 /* Only allow source switch when both the old source unit and the new one have
1069 * no pending data to process (i.e. the consumed length must be a multiple of the
1070 * transform blocklength). */
1071 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1072 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1073 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1074 {
1075 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1076 failed = -EINVAL;
1077 goto error_cleanup;
1078 }
1079 }
1080 /* Detect unit deactivation. */
1081 if (dcfg->last) {
1082 /* Length check of this is handled below. */
1083 tc->done = 1;
1084 }
1085 dcfg = dcfg->next;
1086 } /* while (dcfg) */
1087 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1088
1089 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1090 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1091 failed = -EINVAL;
1092 goto error_cleanup;
1093 }
1094 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1095 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1096 failed = -EINVAL;
1097 goto error_cleanup;
1098 }
1099 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1100 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1101 failed = -EINVAL;
1102 goto error_cleanup;
1103 }
1104
1105 /* Update consumed and produced lengths.
1106
1107 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1108 other unit that process data in blocks of one octet) it is correct, but if it source from a
1109 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1110 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1111 unit has processed an exact multiple of its block length the end result will be correct.
1112 Beware that if the source change restriction change this code will need to be (much) reworked.
1113 */
1114 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1115
1116 if (csum_ctx.active) {
1117 csum_ctx.consumed += desc_len;
1118 if (csum_ctx.done) {
1119 csum_ctx.produced = 2;
1120 }
1121 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1122 }
1123 if (digest_ctx.active) {
1124 digest_ctx.consumed += desc_len;
1125 if (digest_ctx.done) {
1126 if (digest_ctx.unit_no == src_md5) {
1127 digest_ctx.produced = MD5_STATE_LENGTH;
1128 } else {
1129 digest_ctx.produced = SHA1_STATE_LENGTH;
1130 }
1131 }
1132 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1133 }
1134 if (cipher_ctx.active) {
1135 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1136 assert(cipher_ctx.current_src == src_dma);
1137 cipher_ctx.consumed += desc_len;
1138 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1139 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1140 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1141 }
1142 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1143 }
1144
1145 /* Setup the DMA out descriptors. */
1146 /* Configure the metadata. */
1147 active_count = 0;
1148 eop_needed_count = 0;
1149 if (cipher_ctx.active) {
1150 ++active_count;
1151 if (cipher_ctx.unit_no == src_dma){
1152 /* mem2mem */
1153 meta_out.ciphsel = src_none;
1154 } else {
1155 meta_out.ciphsel = cipher_ctx.current_src;
1156 }
1157 meta_out.ciphconf = cipher_ctx.ciph_conf;
1158 meta_out.cbcmode = cipher_ctx.cbcmode;
1159 meta_out.decrypt = cipher_ctx.decrypt;
1160 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1161 if (cipher_ctx.done) ++eop_needed_count;
1162 } else {
1163 meta_out.ciphsel = src_none;
1164 }
1165
1166 if (digest_ctx.active) {
1167 ++active_count;
1168 meta_out.hashsel = digest_ctx.current_src;
1169 meta_out.hashconf = digest_ctx.hash_conf;
1170 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1171 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1172 if (digest_ctx.done) {
1173 assert(digest_ctx.pad_descs == NULL);
1174 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1175 if (failed) {
1176 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1177 goto error_cleanup;
1178 }
1179 }
1180 } else {
1181 meta_out.hashsel = src_none;
1182 }
1183
1184 if (csum_ctx.active) {
1185 ++active_count;
1186 meta_out.csumsel = csum_ctx.current_src;
1187 if (csum_ctx.done) {
1188 assert(csum_ctx.pad_descs == NULL);
1189 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1190 if (failed) {
1191 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1192 goto error_cleanup;
1193 }
1194 }
1195 } else {
1196 meta_out.csumsel = src_none;
1197 }
1198 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1199 /* Setup DMA out descriptors for the indata. */
1200 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1201 if (failed) {
1202 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1203 goto error_cleanup;
1204 }
1205 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1206 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1207 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1208 */
1209 assert(active_count >= eop_needed_count);
1210 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1211 if (eop_needed_count) {
1212 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1213 if (active_count > 1) {
1214 /* Use zero length EOP descriptor. */
1215 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1216 struct strcop_meta_out ed_mo = {0};
1217 if (!ed) {
1218 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1219 failed = -ENOMEM;
1220 goto error_cleanup;
1221 }
1222
1223 assert(cipher_ctx.active && cipher_ctx.done);
1224
1225 if (cipher_ctx.unit_no == src_dma){
1226 /* mem2mem */
1227 ed_mo.ciphsel = src_none;
1228 } else {
1229 ed_mo.ciphsel = cipher_ctx.current_src;
1230 }
1231 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1232 ed_mo.cbcmode = cipher_ctx.cbcmode;
1233 ed_mo.decrypt = cipher_ctx.decrypt;
1234
1235 ed->free_buf = NULL;
1236 ed->dma_descr->wait = 1;
1237 ed->dma_descr->out_eop = 1;
1238
1239 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1240 ed->dma_descr->after = ed->dma_descr->buf;
1241 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1242 current_out_cdesc->next = ed;
1243 current_out_cdesc = ed;
1244 } else {
1245 /* Set EOP in the current out descriptor since the only active module is
1246 * the one needing the EOP. */
1247
1248 current_out_cdesc->dma_descr->out_eop = 1;
1249 }
1250 }
1251
1252 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1253 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1254 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1255 indata_ix += odsc->length;
1256 odsc = odsc->next;
1257 } /* while (odsc) */ /* Process descriptors. */
1258 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1259 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1260 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1261 failed = -EINVAL;
1262 goto error_cleanup;
1263 }
1264 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1265 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1266 failed = -EINVAL;
1267 goto error_cleanup;
1268 }
1269 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1270 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1271 failed = -EINVAL;
1272 goto error_cleanup;
1273 }
1274
1275 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1276 if (failed){
1277 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1278 goto error_cleanup;
1279 }
1280 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1281 if (failed){
1282 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1283 goto error_cleanup;
1284 }
1285 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1286 if (failed){
1287 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1288 goto error_cleanup;
1289 }
1290
1291 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1292 (*int_op)->cdesc_out = out_cdesc_head.next;
1293 (*int_op)->cdesc_in = in_cdesc_head.next;
1294 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1295
1296 setup_descr_chain(out_cdesc_head.next);
1297 setup_descr_chain(in_cdesc_head.next);
1298
1299 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1300 * last DMA out descriptor for EOL.
1301 */
1302 current_in_cdesc->dma_descr->intr = 1;
1303 current_in_cdesc->dma_descr->eol = 1;
1304 current_out_cdesc->dma_descr->eol = 1;
1305
1306 /* Setup DMA contexts. */
1307 (*int_op)->ctx_out.next = NULL;
1308 (*int_op)->ctx_out.eol = 1;
1309 (*int_op)->ctx_out.intr = 0;
1310 (*int_op)->ctx_out.store_mode = 0;
1311 (*int_op)->ctx_out.en = 0;
1312 (*int_op)->ctx_out.dis = 0;
1313 (*int_op)->ctx_out.md0 = 0;
1314 (*int_op)->ctx_out.md1 = 0;
1315 (*int_op)->ctx_out.md2 = 0;
1316 (*int_op)->ctx_out.md3 = 0;
1317 (*int_op)->ctx_out.md4 = 0;
1318 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1319 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1320
1321 (*int_op)->ctx_in.next = NULL;
1322 (*int_op)->ctx_in.eol = 1;
1323 (*int_op)->ctx_in.intr = 0;
1324 (*int_op)->ctx_in.store_mode = 0;
1325 (*int_op)->ctx_in.en = 0;
1326 (*int_op)->ctx_in.dis = 0;
1327 (*int_op)->ctx_in.md0 = 0;
1328 (*int_op)->ctx_in.md1 = 0;
1329 (*int_op)->ctx_in.md2 = 0;
1330 (*int_op)->ctx_in.md3 = 0;
1331 (*int_op)->ctx_in.md4 = 0;
1332
1333 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1334 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1335
1336 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1337 return 0;
1338
1339 error_cleanup:
1340 {
1341 /* Free all allocated resources. */
1342 struct cryptocop_dma_desc *tmp_cdesc;
1343 while (digest_ctx.pad_descs){
1344 tmp_cdesc = digest_ctx.pad_descs->next;
1345 free_cdesc(digest_ctx.pad_descs);
1346 digest_ctx.pad_descs = tmp_cdesc;
1347 }
1348 while (csum_ctx.pad_descs){
1349 tmp_cdesc = csum_ctx.pad_descs->next;
1350 free_cdesc(csum_ctx.pad_descs);
1351 csum_ctx.pad_descs = tmp_cdesc;
1352 }
1353 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1354
1355 if (*int_op != NULL) delete_internal_operation(*int_op);
1356 }
1357 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1358 return failed;
1359 }
1360
1361
1362 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1363 {
1364 void *ptr = iop->alloc_ptr;
1365 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1366 struct cryptocop_dma_desc *next;
1367
1368 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1369
1370 while (cd) {
1371 next = cd->next;
1372 free_cdesc(cd);
1373 cd = next;
1374 }
1375 cd = iop->cdesc_in;
1376 while (cd) {
1377 next = cd->next;
1378 free_cdesc(cd);
1379 cd = next;
1380 }
1381 kfree(ptr);
1382 }
1383
1384 #define MD5_MIN_PAD_LENGTH (9)
1385 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1386
1387 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1388 {
1389 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1390 unsigned char *p;
1391 int i;
1392 unsigned long long int bit_length = hashed_length << 3;
1393
1394 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1395
1396 p = kmalloc(padlen, alloc_flag);
1397 if (!pad) return -ENOMEM;
1398
1399 *p = 0x80;
1400 memset(p+1, 0, padlen - 1);
1401
1402 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405 while (bit_length != 0){
1406 p[i++] = bit_length % 0x100;
1407 bit_length >>= 8;
1408 }
1409
1410 *pad = (char*)p;
1411 *pad_length = padlen;
1412
1413 return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422 unsigned char *p;
1423 int i;
1424 unsigned long long int bit_length = hashed_length << 3;
1425
1426 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428 p = kmalloc(padlen, alloc_flag);
1429 if (!pad) return -ENOMEM;
1430
1431 *p = 0x80;
1432 memset(p+1, 0, padlen - 1);
1433
1434 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1435
1436 i = padlen - 1;
1437 while (bit_length != 0){
1438 p[i--] = bit_length % 0x100;
1439 bit_length >>= 8;
1440 }
1441
1442 *pad = (char*)p;
1443 *pad_length = padlen;
1444
1445 return 0;
1446 }
1447
1448
1449 static int transform_ok(struct cryptocop_transform_init *tinit)
1450 {
1451 switch (tinit->alg){
1452 case cryptocop_alg_csum:
1453 switch (tinit->csum_mode){
1454 case cryptocop_csum_le:
1455 case cryptocop_csum_be:
1456 break;
1457 default:
1458 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1459 return -EINVAL;
1460 }
1461 case cryptocop_alg_mem2mem:
1462 case cryptocop_alg_md5:
1463 case cryptocop_alg_sha1:
1464 if (tinit->keylen != 0) {
1465 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1466 return -EINVAL; /* This check is a bit strict. */
1467 }
1468 break;
1469 case cryptocop_alg_des:
1470 if (tinit->keylen != 64) {
1471 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1472 return -EINVAL;
1473 }
1474 break;
1475 case cryptocop_alg_3des:
1476 if (tinit->keylen != 192) {
1477 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1478 return -EINVAL;
1479 }
1480 break;
1481 case cryptocop_alg_aes:
1482 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1483 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1484 return -EINVAL;
1485 }
1486 break;
1487 case cryptocop_no_alg:
1488 default:
1489 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1490 return -EINVAL;
1491 }
1492
1493 switch (tinit->alg){
1494 case cryptocop_alg_des:
1495 case cryptocop_alg_3des:
1496 case cryptocop_alg_aes:
1497 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1498 default:
1499 break;
1500 }
1501 return 0;
1502 }
1503
1504
1505 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1506 {
1507 struct cryptocop_session *sess;
1508 struct cryptocop_transform_init *tfrm_in = tinit;
1509 struct cryptocop_transform_init *tmp_in;
1510 int no_tfrms = 0;
1511 int i;
1512 unsigned long int flags;
1513
1514 init_stream_coprocessor(); /* For safety if we are called early */
1515
1516 while (tfrm_in){
1517 int err;
1518 ++no_tfrms;
1519 if ((err = transform_ok(tfrm_in))) {
1520 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1521 return err;
1522 }
1523 tfrm_in = tfrm_in->next;
1524 }
1525 if (0 == no_tfrms) {
1526 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1527 return -EINVAL;
1528 }
1529
1530 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1531 if (!sess){
1532 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1533 return -ENOMEM;
1534 }
1535
1536 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1537 if (!sess->tfrm_ctx) {
1538 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1539 kfree(sess);
1540 return -ENOMEM;
1541 }
1542
1543 tfrm_in = tinit;
1544 for (i = 0; i < no_tfrms; i++){
1545 tmp_in = tfrm_in->next;
1546 while (tmp_in){
1547 if (tmp_in->tid == tfrm_in->tid) {
1548 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1549 kfree(sess->tfrm_ctx);
1550 kfree(sess);
1551 return -EINVAL;
1552 }
1553 tmp_in = tmp_in->next;
1554 }
1555 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1556 sess->tfrm_ctx[i].dec_key_set = 0;
1557 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1558
1559 tfrm_in = tfrm_in->next;
1560 }
1561 sess->tfrm_ctx[i-1].next = NULL;
1562
1563 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1564 sess->sid = next_sid;
1565 next_sid++;
1566 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1567 * OTOH 2^64 is a really large number of session. */
1568 if (next_sid == 0) next_sid = 1;
1569
1570 /* Prepend to session list. */
1571 sess->next = cryptocop_sessions;
1572 cryptocop_sessions = sess;
1573 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1574 *sid = sess->sid;
1575 return 0;
1576 }
1577
1578
1579 int cryptocop_free_session(cryptocop_session_id sid)
1580 {
1581 struct cryptocop_transform_ctx *tc;
1582 struct cryptocop_session *sess = NULL;
1583 struct cryptocop_session *psess = NULL;
1584 unsigned long int flags;
1585 int i;
1586 LIST_HEAD(remove_list);
1587 struct list_head *node, *tmp;
1588 struct cryptocop_prio_job *pj;
1589
1590 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1591
1592 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1593 sess = cryptocop_sessions;
1594 while (sess && sess->sid != sid){
1595 psess = sess;
1596 sess = sess->next;
1597 }
1598 if (sess){
1599 if (psess){
1600 psess->next = sess->next;
1601 } else {
1602 cryptocop_sessions = sess->next;
1603 }
1604 }
1605 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1606
1607 if (!sess) return -EINVAL;
1608
1609 /* Remove queued jobs. */
1610 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1611
1612 for (i = 0; i < cryptocop_prio_no_prios; i++){
1613 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1614 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1615 pj = list_entry(node, struct cryptocop_prio_job, node);
1616 if (pj->oper->sid == sid) {
1617 list_move_tail(node, &remove_list);
1618 }
1619 }
1620 }
1621 }
1622 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1623
1624 list_for_each_safe(node, tmp, &remove_list) {
1625 list_del(node);
1626 pj = list_entry(node, struct cryptocop_prio_job, node);
1627 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1628 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1629 pj->oper->cb(pj->oper, pj->oper->cb_data);
1630 delete_internal_operation(pj->iop);
1631 kfree(pj);
1632 }
1633
1634 tc = sess->tfrm_ctx;
1635 /* Erase keying data. */
1636 while (tc){
1637 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1638 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640 tc = tc->next;
1641 }
1642 kfree(sess->tfrm_ctx);
1643 kfree(sess);
1644
1645 return 0;
1646 }
1647
1648 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1649 {
1650 struct cryptocop_session *sess;
1651 unsigned long int flags;
1652
1653 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1654 sess = cryptocop_sessions;
1655 while (sess && (sess->sid != sid)){
1656 sess = sess->next;
1657 }
1658 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1659
1660 return sess;
1661 }
1662
1663 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1664 {
1665 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1666
1667 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1668 assert(sess != NULL);
1669 while (tc && tc->init.tid != tid){
1670 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1671 tc = tc->next;
1672 }
1673 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1674 return tc;
1675 }
1676
1677
1678
1679 /* The AES s-transform matrix (s-box). */
1680 static const u8 aes_sbox[256] = {
1681 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1682 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1683 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1684 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1685 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1686 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1687 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1688 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1689 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1690 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1691 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1692 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1693 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1694 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1695 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1696 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1697 };
1698
1699 /* AES has a 32 bit word round constants for each round in the
1700 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1701 */
1702 static u32 round_constant[11] = {
1703 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1704 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1705 0x1B000000, 0x36000000, 0x6C000000
1706 };
1707
1708 /* Apply the s-box to each of the four occtets in w. */
1709 static u32 aes_ks_subword(const u32 w)
1710 {
1711 u8 bytes[4];
1712
1713 *(u32*)(&bytes[0]) = w;
1714 bytes[0] = aes_sbox[bytes[0]];
1715 bytes[1] = aes_sbox[bytes[1]];
1716 bytes[2] = aes_sbox[bytes[2]];
1717 bytes[3] = aes_sbox[bytes[3]];
1718 return *(u32*)(&bytes[0]);
1719 }
1720
1721 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1722 * (Note that AES words are 32 bit long)
1723 *
1724 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1725 * word temp
1726 * i = 0
1727 * while (i < Nk) {
1728 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1729 * i = i + 1
1730 * }
1731 * i = Nk
1732 *
1733 * while (i < (Nb * (Nr + 1))) {
1734 * temp = w[i - 1]
1735 * if ((i mod Nk) == 0) {
1736 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1737 * }
1738 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1739 * temp = SubWord(temp)
1740 * }
1741 * w[i] = w[i - Nk] xor temp
1742 * }
1743 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1744 * SubWord(t) applies the AES s-box individually to each octet
1745 * in a 32 bit word.
1746 *
1747 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1748 * values for Nr of 10, 12, and 14). Nb is always 4.
1749 *
1750 * To construct w[i], w[i - 1] and w[i - Nk] must be
1751 * available. Consequently we must keep a state of the last Nk words
1752 * to be able to create the last round keys.
1753 */
1754 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1755 {
1756 u32 temp;
1757 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1758 u8 w_last_ix;
1759 int i;
1760 u8 nr, nk;
1761
1762 switch (keylength){
1763 case 128:
1764 nk = 4;
1765 nr = 10;
1766 break;
1767 case 192:
1768 nk = 6;
1769 nr = 12;
1770 break;
1771 case 256:
1772 nk = 8;
1773 nr = 14;
1774 break;
1775 default:
1776 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1777 };
1778
1779 /* Need to do host byte order correction here since key is byte oriented and the
1780 * kx algorithm is word (u32) oriented. */
1781 for (i = 0; i < nk; i+=1) {
1782 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1783 }
1784
1785 i = (int)nk;
1786 w_last_ix = i - 1;
1787 while (i < (4 * (nr + 2))) {
1788 temp = w_ring[w_last_ix];
1789 if (!(i % nk)) {
1790 /* RotWord(temp) */
1791 temp = (temp << 8) | (temp >> 24);
1792 temp = aes_ks_subword(temp);
1793 temp ^= round_constant[i/nk - 1];
1794 } else if ((nk > 6) && ((i % nk) == 4)) {
1795 temp = aes_ks_subword(temp);
1796 }
1797 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1798 temp ^= w_ring[w_last_ix];
1799 w_ring[w_last_ix] = temp;
1800
1801 /* We need the round keys for round Nr+1 and Nr+2 (round key
1802 * Nr+2 is the round key beyond the last one used when
1803 * encrypting). Rounds are numbered starting from 0, Nr=10
1804 * implies 11 rounds are used in encryption/decryption.
1805 */
1806 if (i >= (4 * nr)) {
1807 /* Need to do host byte order correction here, the key
1808 * is byte oriented. */
1809 *(u32*)dec_key = cpu_to_be32(temp);
1810 dec_key += 4;
1811 }
1812 ++i;
1813 }
1814 }
1815
1816
1817 /**** Job/operation management. ****/
1818
1819 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1820 {
1821 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1822 }
1823
1824 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1825 {
1826 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1827 }
1828
1829 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1830 {
1831 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1832 }
1833
1834 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1835 {
1836 int ret;
1837 struct cryptocop_prio_job *pj = NULL;
1838 unsigned long int flags;
1839
1840 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1841
1842 if (!operation || !operation->cb){
1843 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1844 return -EINVAL;
1845 }
1846
1847 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1848 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1849 return ret;
1850 }
1851 assert(pj != NULL);
1852
1853 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1854 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1855 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1856
1857 /* Make sure a job is running */
1858 cryptocop_start_job();
1859 return 0;
1860 }
1861
1862 static void cryptocop_do_tasklet(unsigned long unused);
1863 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1864
1865 static void cryptocop_do_tasklet(unsigned long unused)
1866 {
1867 struct list_head *node;
1868 struct cryptocop_prio_job *pj = NULL;
1869 unsigned long flags;
1870
1871 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1872
1873 do {
1874 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1875 if (!list_empty(&cryptocop_completed_jobs)){
1876 node = cryptocop_completed_jobs.next;
1877 list_del(node);
1878 pj = list_entry(node, struct cryptocop_prio_job, node);
1879 } else {
1880 pj = NULL;
1881 }
1882 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1883 if (pj) {
1884 assert(pj->oper != NULL);
1885
1886 /* Notify consumer of operation completeness. */
1887 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1888
1889 pj->oper->operation_status = 0; /* Job is completed. */
1890 pj->oper->cb(pj->oper, pj->oper->cb_data);
1891 delete_internal_operation(pj->iop);
1892 kfree(pj);
1893 }
1894 } while (pj != NULL);
1895
1896 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1897 }
1898
1899 static irqreturn_t
1900 dma_done_interrupt(int irq, void *dev_id)
1901 {
1902 struct cryptocop_prio_job *done_job;
1903 reg_dma_rw_ack_intr ack_intr = {
1904 .data = 1,
1905 };
1906
1907 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1908
1909 DEBUG(printk("cryptocop DMA done\n"));
1910
1911 spin_lock(&running_job_lock);
1912 if (cryptocop_running_job == NULL){
1913 printk("stream co-processor got interrupt when not busy\n");
1914 spin_unlock(&running_job_lock);
1915 return IRQ_HANDLED;
1916 }
1917 done_job = cryptocop_running_job;
1918 cryptocop_running_job = NULL;
1919 spin_unlock(&running_job_lock);
1920
1921 /* Start processing a job. */
1922 if (!spin_trylock(&cryptocop_process_lock)){
1923 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1924 } else {
1925 cryptocop_start_job();
1926 spin_unlock(&cryptocop_process_lock);
1927 }
1928
1929 done_job->oper->operation_status = 0; /* Job is completed. */
1930 if (done_job->oper->fast_callback){
1931 /* This operation wants callback from interrupt. */
1932 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1933 delete_internal_operation(done_job->iop);
1934 kfree(done_job);
1935 } else {
1936 spin_lock(&cryptocop_completed_jobs_lock);
1937 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1938 spin_unlock(&cryptocop_completed_jobs_lock);
1939 tasklet_schedule(&cryptocop_tasklet);
1940 }
1941
1942 DEBUG(printk("cryptocop leave irq handler\n"));
1943 return IRQ_HANDLED;
1944 }
1945
1946
1947 /* Setup interrupts and DMA channels. */
1948 static int init_cryptocop(void)
1949 {
1950 unsigned long flags;
1951 reg_dma_rw_cfg dma_cfg = {.en = 1};
1952 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1953 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1954 reg_strcop_rw_cfg strcop_cfg = {
1955 .ipend = regk_strcop_little,
1956 .td1 = regk_strcop_e,
1957 .td2 = regk_strcop_d,
1958 .td3 = regk_strcop_e,
1959 .ignore_sync = 0,
1960 .en = 1
1961 };
1962
1963 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1964 "stream co-processor DMA", NULL))
1965 panic("request_irq stream co-processor irq dma9");
1966
1967 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1968 0, dma_strp);
1969 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1970 0, dma_strp);
1971
1972 local_irq_save(flags);
1973
1974 /* Reset and enable the cryptocop. */
1975 strcop_cfg.en = 0;
1976 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1977 strcop_cfg.en = 1;
1978 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1979
1980 /* Enable DMAs. */
1981 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1982 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1983
1984 /* Set up wordsize = 4 for DMAs. */
1985 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1986 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1987
1988 /* Enable interrupts. */
1989 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1990
1991 /* Clear intr ack. */
1992 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1993
1994 local_irq_restore(flags);
1995
1996 return 0;
1997 }
1998
1999 /* Free used cryptocop hw resources (interrupt and DMA channels). */
2000 static void release_cryptocop(void)
2001 {
2002 unsigned long flags;
2003 reg_dma_rw_cfg dma_cfg = {.en = 0};
2004 reg_dma_rw_intr_mask intr_mask_in = {0};
2005 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2006
2007 local_irq_save(flags);
2008
2009 /* Clear intr ack. */
2010 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2011
2012 /* Disable DMAs. */
2013 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2014 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2015
2016 /* Disable interrupts. */
2017 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2018
2019 local_irq_restore(flags);
2020
2021 free_irq(DMA_IRQ, NULL);
2022
2023 (void)crisv32_free_dma(OUT_DMA);
2024 (void)crisv32_free_dma(IN_DMA);
2025 }
2026
2027
2028 /* Init job queue. */
2029 static int cryptocop_job_queue_init(void)
2030 {
2031 int i;
2032
2033 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2034
2035 for (i = 0; i < cryptocop_prio_no_prios; i++){
2036 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2037 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2038 }
2039 return 0;
2040 }
2041
2042
2043 static void cryptocop_job_queue_close(void)
2044 {
2045 struct list_head *node, *tmp;
2046 struct cryptocop_prio_job *pj = NULL;
2047 unsigned long int process_flags, flags;
2048 int i;
2049
2050 /* FIXME: This is as yet untested code. */
2051
2052 /* Stop strcop from getting an operation to process while we are closing the
2053 module. */
2054 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2055
2056 /* Empty the job queue. */
2057 for (i = 0; i < cryptocop_prio_no_prios; i++){
2058 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2059 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2060 pj = list_entry(node, struct cryptocop_prio_job, node);
2061 list_del(node);
2062
2063 /* Call callback to notify consumer of job removal. */
2064 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2065 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2066 pj->oper->cb(pj->oper, pj->oper->cb_data);
2067
2068 delete_internal_operation(pj->iop);
2069 kfree(pj);
2070 }
2071 }
2072 }
2073 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2074
2075 /* Remove the running job, if any. */
2076 spin_lock_irqsave(&running_job_lock, flags);
2077 if (cryptocop_running_job){
2078 reg_strcop_rw_cfg rw_cfg;
2079 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2080
2081 /* Stop DMA. */
2082 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2083 dma_out_cfg.en = regk_dma_no;
2084 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2085
2086 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2087 dma_in_cfg.en = regk_dma_no;
2088 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2089
2090 /* Disble the cryptocop. */
2091 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2092 rw_cfg.en = 0;
2093 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2094
2095 pj = cryptocop_running_job;
2096 cryptocop_running_job = NULL;
2097
2098 /* Call callback to notify consumer of job removal. */
2099 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2100 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2101 pj->oper->cb(pj->oper, pj->oper->cb_data);
2102
2103 delete_internal_operation(pj->iop);
2104 kfree(pj);
2105 }
2106 spin_unlock_irqrestore(&running_job_lock, flags);
2107
2108 /* Remove completed jobs, if any. */
2109 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2110
2111 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2112 pj = list_entry(node, struct cryptocop_prio_job, node);
2113 list_del(node);
2114 /* Call callback to notify consumer of job removal. */
2115 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2116 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2117 pj->oper->cb(pj->oper, pj->oper->cb_data);
2118
2119 delete_internal_operation(pj->iop);
2120 kfree(pj);
2121 }
2122 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2123 }
2124
2125
2126 static void cryptocop_start_job(void)
2127 {
2128 int i;
2129 struct cryptocop_prio_job *pj;
2130 unsigned long int flags;
2131 unsigned long int running_job_flags;
2132 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2133
2134 DEBUG(printk("cryptocop_start_job: entering\n"));
2135
2136 spin_lock_irqsave(&running_job_lock, running_job_flags);
2137 if (cryptocop_running_job != NULL){
2138 /* Already running. */
2139 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2140 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2141 return;
2142 }
2143 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2144
2145 /* Check the queues in priority order. */
2146 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2147 if (i == cryptocop_prio_no_prios) {
2148 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2149 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2150 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2151 return; /* No jobs to run */
2152 }
2153 DEBUG(printk("starting job for prio %d\n", i));
2154
2155 /* TODO: Do not starve lower priority jobs. Let in a lower
2156 * prio job for every N-th processed higher prio job or some
2157 * other scheduling policy. This could reasonably be
2158 * tweakable since the optimal balance would depend on the
2159 * type of load on the system. */
2160
2161 /* Pull the DMA lists from the job and start the DMA client. */
2162 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2163 list_del(&pj->node);
2164 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2165 cryptocop_running_job = pj;
2166
2167 /* Set config register (3DES and CSUM modes). */
2168 switch (pj->iop->tdes_mode){
2169 case cryptocop_3des_eee:
2170 rw_cfg.td1 = regk_strcop_e;
2171 rw_cfg.td2 = regk_strcop_e;
2172 rw_cfg.td3 = regk_strcop_e;
2173 break;
2174 case cryptocop_3des_eed:
2175 rw_cfg.td1 = regk_strcop_e;
2176 rw_cfg.td2 = regk_strcop_e;
2177 rw_cfg.td3 = regk_strcop_d;
2178 break;
2179 case cryptocop_3des_ede:
2180 rw_cfg.td1 = regk_strcop_e;
2181 rw_cfg.td2 = regk_strcop_d;
2182 rw_cfg.td3 = regk_strcop_e;
2183 break;
2184 case cryptocop_3des_edd:
2185 rw_cfg.td1 = regk_strcop_e;
2186 rw_cfg.td2 = regk_strcop_d;
2187 rw_cfg.td3 = regk_strcop_d;
2188 break;
2189 case cryptocop_3des_dee:
2190 rw_cfg.td1 = regk_strcop_d;
2191 rw_cfg.td2 = regk_strcop_e;
2192 rw_cfg.td3 = regk_strcop_e;
2193 break;
2194 case cryptocop_3des_ded:
2195 rw_cfg.td1 = regk_strcop_d;
2196 rw_cfg.td2 = regk_strcop_e;
2197 rw_cfg.td3 = regk_strcop_d;
2198 break;
2199 case cryptocop_3des_dde:
2200 rw_cfg.td1 = regk_strcop_d;
2201 rw_cfg.td2 = regk_strcop_d;
2202 rw_cfg.td3 = regk_strcop_e;
2203 break;
2204 case cryptocop_3des_ddd:
2205 rw_cfg.td1 = regk_strcop_d;
2206 rw_cfg.td2 = regk_strcop_d;
2207 rw_cfg.td3 = regk_strcop_d;
2208 break;
2209 default:
2210 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2211 }
2212 switch (pj->iop->csum_mode){
2213 case cryptocop_csum_le:
2214 rw_cfg.ipend = regk_strcop_little;
2215 break;
2216 case cryptocop_csum_be:
2217 rw_cfg.ipend = regk_strcop_big;
2218 break;
2219 default:
2220 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2221 }
2222 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2223
2224 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2225 "ctx_in: 0x%p, phys: 0x%p\n"
2226 "ctx_out: 0x%p, phys: 0x%p\n",
2227 pj,
2228 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2229 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2230
2231 /* Start input DMA. */
2232 flush_dma_context(&pj->iop->ctx_in);
2233 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2234
2235 /* Start output DMA. */
2236 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2237
2238 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2239 DEBUG(printk("cryptocop_start_job: exiting\n"));
2240 }
2241
2242
2243 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2244 {
2245 int err;
2246 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2247 void *iop_alloc_ptr = NULL;
2248
2249 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2250 if (!*pj) return -ENOMEM;
2251
2252 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2253
2254 (*pj)->oper = operation;
2255 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2256
2257 if (operation->use_dmalists) {
2258 DEBUG(print_user_dma_lists(&operation->list_op));
2259 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2260 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2261 kfree(*pj);
2262 return -EINVAL;
2263 }
2264 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2265 if (!iop_alloc_ptr) {
2266 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2267 kfree(*pj);
2268 return -ENOMEM;
2269 }
2270 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2271 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2272 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2273 (*pj)->iop->sid = operation->sid;
2274 (*pj)->iop->cdesc_out = NULL;
2275 (*pj)->iop->cdesc_in = NULL;
2276 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2277 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2278 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2279 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2280
2281 /* Setup DMA contexts. */
2282 (*pj)->iop->ctx_out.next = NULL;
2283 (*pj)->iop->ctx_out.eol = 1;
2284 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2285 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2286
2287 (*pj)->iop->ctx_in.next = NULL;
2288 (*pj)->iop->ctx_in.eol = 1;
2289 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2290 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2291 } else {
2292 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2293 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2294 kfree(*pj);
2295 return err;
2296 }
2297 }
2298 DEBUG(print_dma_descriptors((*pj)->iop));
2299
2300 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2301
2302 return 0;
2303 }
2304
2305
2306 static int cryptocop_open(struct inode *inode, struct file *filp)
2307 {
2308 int p = iminor(inode);
2309
2310 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2311
2312 filp->private_data = NULL;
2313 return 0;
2314 }
2315
2316
2317 static int cryptocop_release(struct inode *inode, struct file *filp)
2318 {
2319 struct cryptocop_private *dev = filp->private_data;
2320 struct cryptocop_private *dev_next;
2321
2322 while (dev){
2323 dev_next = dev->next;
2324 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2325 (void)cryptocop_free_session(dev->sid);
2326 }
2327 kfree(dev);
2328 dev = dev_next;
2329 }
2330
2331 return 0;
2332 }
2333
2334
2335 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2336 unsigned int cmd, unsigned long arg)
2337 {
2338 struct cryptocop_private *dev = filp->private_data;
2339 struct cryptocop_private *prev_dev = NULL;
2340 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2341 struct strcop_session_op sop;
2342 int err;
2343
2344 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2345
2346 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2347 return -EFAULT;
2348 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2349 if (err) return -EFAULT;
2350
2351 while (dev && (dev->sid != sop.ses_id)) {
2352 prev_dev = dev;
2353 dev = dev->next;
2354 }
2355 if (dev){
2356 if (prev_dev){
2357 prev_dev->next = dev->next;
2358 } else {
2359 filp->private_data = dev->next;
2360 }
2361 err = cryptocop_free_session(dev->sid);
2362 if (err) return -EFAULT;
2363 } else {
2364 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2365 return -EINVAL;
2366 }
2367 return 0;
2368 }
2369
2370
2371 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2372 {
2373 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2374
2375 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2376
2377 jc->processed = 1;
2378 wake_up(&cryptocop_ioc_process_wq);
2379 }
2380
2381
2382 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2383 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2384 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2385
2386 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2387 {
2388 size_t ch_ix = 0;
2389
2390 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2391 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2392 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2393
2394 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2395 return ch_ix;
2396 }
2397
2398
2399 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2400 {
2401 size_t ch_ix = INT_MAX;
2402 size_t tmp_ix = 0;
2403
2404 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2405 if (crp_op->cipher_start > ix) {
2406 ch_ix = crp_op->cipher_start;
2407 } else {
2408 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2409 }
2410 }
2411 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2412 if (crp_op->digest_start > ix) {
2413 tmp_ix = crp_op->digest_start;
2414 } else {
2415 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2416 }
2417 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2418 }
2419 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2420 if (crp_op->csum_start > ix) {
2421 tmp_ix = crp_op->csum_start;
2422 } else {
2423 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2424 }
2425 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2426 }
2427 if (ch_ix == INT_MAX) ch_ix = ix;
2428 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2429 return ch_ix;
2430 }
2431
2432
2433 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2434 * Return -1 for ok, 0 for fail. */
2435 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2436 {
2437 int tmplen;
2438
2439 assert(iov != NULL);
2440 assert(iovix != NULL);
2441 assert(pages != NULL);
2442 assert(pageix != NULL);
2443 assert(pageoffset != NULL);
2444
2445 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446
2447 while (map_length > 0){
2448 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2449 if (*iovix >= iovlen){
2450 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2451 return 0;
2452 }
2453 if (*pageix >= nopages){
2454 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2455 return 0;
2456 }
2457 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2458 tmplen = PAGE_SIZE - *pageoffset;
2459 if (tmplen < map_length){
2460 (*pageoffset) = 0;
2461 (*pageix)++;
2462 } else {
2463 tmplen = map_length;
2464 (*pageoffset) += map_length;
2465 }
2466 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2467 iov[*iovix].iov_len = tmplen;
2468 map_length -= tmplen;
2469 (*iovix)++;
2470 }
2471 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2472 return -1;
2473 }
2474
2475
2476
2477 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2478 {
2479 int i;
2480 struct cryptocop_private *dev = filp->private_data;
2481 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2482 struct strcop_crypto_op oper = {0};
2483 int err = 0;
2484 struct cryptocop_operation *cop = NULL;
2485
2486 struct ioctl_job_cb_ctx *jc = NULL;
2487
2488 struct page **inpages = NULL;
2489 struct page **outpages = NULL;
2490 int noinpages = 0;
2491 int nooutpages = 0;
2492
2493 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2494 * can get connected/disconnected on different places in the indata. */
2495 struct cryptocop_desc_cfg dcfgs[5*3];
2496 int desc_ix = 0;
2497 int dcfg_ix = 0;
2498 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2499 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2500 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2501
2502 unsigned char *digest_result = NULL;
2503 int digest_length = 0;
2504 int cblocklen = 0;
2505 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2506 struct cryptocop_session *sess;
2507
2508 int iovlen = 0;
2509 int iovix = 0;
2510 int pageix = 0;
2511 int pageoffset = 0;
2512
2513 size_t prev_ix = 0;
2514 size_t next_ix;
2515
2516 int cipher_active, digest_active, csum_active;
2517 int end_digest, end_csum;
2518 int digest_done = 0;
2519 int cipher_done = 0;
2520 int csum_done = 0;
2521
2522 DEBUG(printk("cryptocop_ioctl_process\n"));
2523
2524 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2525 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2526 return -EFAULT;
2527 }
2528 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2529 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2530 return -EFAULT;
2531 }
2532 DEBUG(print_strcop_crypto_op(&oper));
2533
2534 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2535 if (!dev){
2536 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2537 return -EINVAL;
2538 }
2539
2540 /* Check buffers. */
2541 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2542 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2543 return -EINVAL;
2544 }
2545
2546 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2547 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2548 return -EFAULT;
2549 }
2550 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2551 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2552 return -EFAULT;
2553 }
2554
2555 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2556 if (!cop) {
2557 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2558 return -ENOMEM;
2559 }
2560 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2561 if (!jc) {
2562 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2563 err = -ENOMEM;
2564 goto error_cleanup;
2565 }
2566 jc->processed = 0;
2567
2568 cop->cb_data = jc;
2569 cop->cb = ioctl_process_job_callback;
2570 cop->operation_status = 0;
2571 cop->use_dmalists = 0;
2572 cop->in_interrupt = 0;
2573 cop->fast_callback = 0;
2574 cop->tfrm_op.tfrm_cfg = NULL;
2575 cop->tfrm_op.desc = NULL;
2576 cop->tfrm_op.indata = NULL;
2577 cop->tfrm_op.incount = 0;
2578 cop->tfrm_op.inlen = 0;
2579 cop->tfrm_op.outdata = NULL;
2580 cop->tfrm_op.outcount = 0;
2581 cop->tfrm_op.outlen = 0;
2582
2583 sess = get_session(oper.ses_id);
2584 if (!sess){
2585 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2586 kfree(cop);
2587 kfree(jc);
2588 return -EINVAL;
2589 }
2590
2591 if (oper.do_cipher) {
2592 unsigned int cipher_outlen = 0;
2593 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2594 if (!tc) {
2595 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2596 err = -EINVAL;
2597 goto error_cleanup;
2598 }
2599 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2600 ciph_tcfg.inject_ix = 0;
2601 ciph_tcfg.flags = 0;
2602 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2603 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2604 kfree(cop);
2605 kfree(jc);
2606 return -EINVAL;
2607 }
2608 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2609 if (oper.cipher_len % cblocklen) {
2610 kfree(cop);
2611 kfree(jc);
2612 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2613 return -EINVAL;
2614 }
2615 cipher_outlen = oper.cipher_len;
2616 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2617 if (oper.cipher_explicit) {
2618 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2619 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2620 } else {
2621 cipher_outlen = oper.cipher_len - cblocklen;
2622 }
2623 } else {
2624 if (oper.cipher_explicit){
2625 kfree(cop);
2626 kfree(jc);
2627 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2628 return -EINVAL;
2629 }
2630 }
2631 if (oper.cipher_outlen != cipher_outlen) {
2632 kfree(cop);
2633 kfree(jc);
2634 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2635 return -EINVAL;
2636 }
2637
2638 if (oper.decrypt){
2639 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2640 } else {
2641 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2642 }
2643 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2644 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2645 }
2646 if (oper.do_digest){
2647 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2648 if (!tc) {
2649 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2650 err = -EINVAL;
2651 goto error_cleanup;
2652 }
2653 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2654 digest_result = kmalloc(digest_length, GFP_KERNEL);
2655 if (!digest_result) {
2656 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2657 err = -EINVAL;
2658 goto error_cleanup;
2659 }
2660 DEBUG(memset(digest_result, 0xff, digest_length));
2661
2662 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2663 digest_tcfg.inject_ix = 0;
2664 ciph_tcfg.inject_ix += digest_length;
2665 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2666 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2667 err = -EINVAL;
2668 goto error_cleanup;
2669 }
2670
2671 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2672 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2673 }
2674 if (oper.do_csum){
2675 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2676 csum_tcfg.inject_ix = digest_length;
2677 ciph_tcfg.inject_ix += 2;
2678
2679 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2680 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2681 kfree(cop);
2682 kfree(jc);
2683 return -EINVAL;
2684 }
2685
2686 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2687 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2688 }
2689
2690 prev_ix = first_cfg_change_ix(&oper);
2691 if (prev_ix > oper.inlen) {
2692 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2693 nooutpages = noinpages = 0;
2694 err = -EINVAL;
2695 goto error_cleanup;
2696 }
2697 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2698
2699 /* Map user pages for in and out data of the operation. */
2700 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2701 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2702 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2703 if (!inpages){
2704 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2705 nooutpages = noinpages = 0;
2706 err = -ENOMEM;
2707 goto error_cleanup;
2708 }
2709 if (oper.do_cipher){
2710 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2711 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2712 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2713 if (!outpages){
2714 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2715 nooutpages = noinpages = 0;
2716 err = -ENOMEM;
2717 goto error_cleanup;
2718 }
2719 }
2720
2721 /* Acquire the mm page semaphore. */
2722 down_read(&current->mm->mmap_sem);
2723
2724 err = get_user_pages(current,
2725 current->mm,
2726 (unsigned long int)(oper.indata + prev_ix),
2727 noinpages,
2728 0, /* read access only for in data */
2729 0, /* no force */
2730 inpages,
2731 NULL);
2732
2733 if (err < 0) {
2734 up_read(&current->mm->mmap_sem);
2735 nooutpages = noinpages = 0;
2736 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2737 goto error_cleanup;
2738 }
2739 noinpages = err;
2740 if (oper.do_cipher){
2741 err = get_user_pages(current,
2742 current->mm,
2743 (unsigned long int)oper.cipher_outdata,
2744 nooutpages,
2745 1, /* write access for out data */
2746 0, /* no force */
2747 outpages,
2748 NULL);
2749 up_read(&current->mm->mmap_sem);
2750 if (err < 0) {
2751 nooutpages = 0;
2752 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2753 goto error_cleanup;
2754 }
2755 nooutpages = err;
2756 } else {
2757 up_read(&current->mm->mmap_sem);
2758 }
2759
2760 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2761 * csum output and splits when units are (dis-)connected. */
2762 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2763 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2764 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2765 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2766 err = -ENOMEM;
2767 goto error_cleanup;
2768 }
2769
2770 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2771 cop->tfrm_op.outlen = 0;
2772 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2773 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2774 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2775
2776 /* Setup the in iovecs. */
2777 cop->tfrm_op.incount = noinpages;
2778 if (noinpages > 1){
2779 size_t tmplen = cop->tfrm_op.inlen;
2780
2781 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783 tmplen -= cop->tfrm_op.indata[0].iov_len;
2784 for (i = 1; i<noinpages; i++){
2785 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2786 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2787 tmplen -= PAGE_SIZE;
2788 }
2789 } else {
2790 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2791 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2792 }
2793
2794 iovlen = nooutpages + 6;
2795 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2796
2797 next_ix = next_cfg_change_ix(&oper, prev_ix);
2798 if (prev_ix == next_ix){
2799 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2800 err = -EINVAL; /* This should be impossible barring bugs. */
2801 goto error_cleanup;
2802 }
2803 while (prev_ix != next_ix){
2804 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2805 descs[desc_ix].cfg = NULL;
2806 descs[desc_ix].length = next_ix - prev_ix;
2807
2808 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2809 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2810 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2811 cipher_active = 1;
2812
2813 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2814 cipher_done = 1;
2815 dcfgs[dcfg_ix].last = 1;
2816 } else {
2817 dcfgs[dcfg_ix].last = 0;
2818 }
2819 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2820 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2821 ++dcfg_ix;
2822 }
2823 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2824 digest_active = 1;
2825 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2826 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2827 if (next_ix == (oper.digest_start + oper.digest_len)){
2828 assert(!digest_done);
2829 digest_done = 1;
2830 dcfgs[dcfg_ix].last = 1;
2831 } else {
2832 dcfgs[dcfg_ix].last = 0;
2833 }
2834 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2835 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2836 ++dcfg_ix;
2837 }
2838 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2839 csum_active = 1;
2840 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2841 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2842 if (next_ix == (oper.csum_start + oper.csum_len)){
2843 csum_done = 1;
2844 dcfgs[dcfg_ix].last = 1;
2845 } else {
2846 dcfgs[dcfg_ix].last = 0;
2847 }
2848 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2849 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2850 ++dcfg_ix;
2851 }
2852 if (!descs[desc_ix].cfg){
2853 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2854 err = -EINVAL;
2855 goto error_cleanup;
2856 }
2857 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2858 ++desc_ix;
2859 prev_ix = next_ix;
2860 next_ix = next_cfg_change_ix(&oper, prev_ix);
2861 }
2862 if (desc_ix > 0){
2863 descs[desc_ix-1].next = NULL;
2864 } else {
2865 descs[0].next = NULL;
2866 }
2867 if (oper.do_digest) {
2868 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2869 /* Add outdata iovec, length == <length of type of digest> */
2870 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2871 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2872 ++iovix;
2873 }
2874 if (oper.do_csum) {
2875 /* Add outdata iovec, length == 2, the length of csum. */
2876 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2877 /* Add outdata iovec, length == <length of type of digest> */
2878 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2879 cop->tfrm_op.outdata[iovix].iov_len = 2;
2880 ++iovix;
2881 }
2882 if (oper.do_cipher) {
2883 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2884 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2885 err = -ENOSYS; /* This should be impossible barring bugs. */
2886 goto error_cleanup;
2887 }
2888 }
2889 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2890 cop->tfrm_op.outcount = iovix;
2891 assert(iovix <= (nooutpages + 6));
2892
2893 cop->sid = oper.ses_id;
2894 cop->tfrm_op.desc = &descs[0];
2895
2896 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2897
2898 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2899 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2900 err = -EINVAL;
2901 goto error_cleanup;
2902 }
2903
2904 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2905
2906 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2907 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2908 if (!jc->processed){
2909 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2910 err = -EIO;
2911 goto error_cleanup;
2912 }
2913
2914 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2915 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2916 if (cop->operation_status == 0){
2917 if (oper.do_digest){
2918 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2919 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2920 if (0 != err){
2921 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2922 err = -EFAULT;
2923 goto error_cleanup;
2924 }
2925 }
2926 if (oper.do_csum){
2927 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2928 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2929 if (0 != err){
2930 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2931 err = -EFAULT;
2932 goto error_cleanup;
2933 }
2934 }
2935 err = 0;
2936 } else {
2937 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2938 err = cop->operation_status;
2939 }
2940
2941 error_cleanup:
2942 /* Release page caches. */
2943 for (i = 0; i < noinpages; i++){
2944 put_page(inpages[i]);
2945 }
2946 for (i = 0; i < nooutpages; i++){
2947 int spdl_err;
2948 /* Mark output pages dirty. */
2949 spdl_err = set_page_dirty_lock(outpages[i]);
2950 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2951 }
2952 for (i = 0; i < nooutpages; i++){
2953 put_page(outpages[i]);
2954 }
2955
2956 kfree(digest_result);
2957 kfree(inpages);
2958 kfree(outpages);
2959 if (cop){
2960 kfree(cop->tfrm_op.indata);
2961 kfree(cop->tfrm_op.outdata);
2962 kfree(cop);
2963 }
2964 kfree(jc);
2965
2966 DEBUG(print_lock_status());
2967
2968 return err;
2969 }
2970
2971
2972 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2973 {
2974 cryptocop_session_id sid;
2975 int err;
2976 struct cryptocop_private *dev;
2977 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2978 struct strcop_session_op sop;
2979 struct cryptocop_transform_init *tis = NULL;
2980 struct cryptocop_transform_init ti_cipher = {0};
2981 struct cryptocop_transform_init ti_digest = {0};
2982 struct cryptocop_transform_init ti_csum = {0};
2983
2984 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2985 return -EFAULT;
2986 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2987 if (err) return -EFAULT;
2988 if (sop.cipher != cryptocop_cipher_none) {
2989 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2990 }
2991 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2992
2993 DEBUG(printk("\tcipher:%d\n"
2994 "\tcipher_mode:%d\n"
2995 "\tdigest:%d\n"
2996 "\tcsum:%d\n",
2997 (int)sop.cipher,
2998 (int)sop.cmode,
2999 (int)sop.digest,
3000 (int)sop.csum));
3001
3002 if (sop.cipher != cryptocop_cipher_none){
3003 /* Init the cipher. */
3004 switch (sop.cipher){
3005 case cryptocop_cipher_des:
3006 ti_cipher.alg = cryptocop_alg_des;
3007 break;
3008 case cryptocop_cipher_3des:
3009 ti_cipher.alg = cryptocop_alg_3des;
3010 break;
3011 case cryptocop_cipher_aes:
3012 ti_cipher.alg = cryptocop_alg_aes;
3013 break;
3014 default:
3015 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3016 return -EINVAL;
3017 };
3018 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3019 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3020 ti_cipher.keylen = sop.keylen;
3021 switch (sop.cmode){
3022 case cryptocop_cipher_mode_cbc:
3023 case cryptocop_cipher_mode_ecb:
3024 ti_cipher.cipher_mode = sop.cmode;
3025 break;
3026 default:
3027 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3028 return -EINVAL;
3029 }
3030 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3031 switch (sop.des3_mode){
3032 case cryptocop_3des_eee:
3033 case cryptocop_3des_eed:
3034 case cryptocop_3des_ede:
3035 case cryptocop_3des_edd:
3036 case cryptocop_3des_dee:
3037 case cryptocop_3des_ded:
3038 case cryptocop_3des_dde:
3039 case cryptocop_3des_ddd:
3040 ti_cipher.tdes_mode = sop.des3_mode;
3041 break;
3042 default:
3043 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3044 return -EINVAL;
3045 }
3046 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3047 ti_cipher.next = tis;
3048 tis = &ti_cipher;
3049 } /* if (sop.cipher != cryptocop_cipher_none) */
3050 if (sop.digest != cryptocop_digest_none){
3051 DEBUG(printk("setting digest transform\n"));
3052 switch (sop.digest){
3053 case cryptocop_digest_md5:
3054 ti_digest.alg = cryptocop_alg_md5;
3055 break;
3056 case cryptocop_digest_sha1:
3057 ti_digest.alg = cryptocop_alg_sha1;
3058 break;
3059 default:
3060 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3061 return -EINVAL;
3062 }
3063 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3064 ti_digest.next = tis;
3065 tis = &ti_digest;
3066 } /* if (sop.digest != cryptocop_digest_none) */
3067 if (sop.csum != cryptocop_csum_none){
3068 DEBUG(printk("setting csum transform\n"));
3069 switch (sop.csum){
3070 case cryptocop_csum_le:
3071 case cryptocop_csum_be:
3072 ti_csum.csum_mode = sop.csum;
3073 break;
3074 default:
3075 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3076 return -EINVAL;
3077 }
3078 ti_csum.alg = cryptocop_alg_csum;
3079 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3080 ti_csum.next = tis;
3081 tis = &ti_csum;
3082 } /* (sop.csum != cryptocop_csum_none) */
3083 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3084 if (!dev){
3085 DEBUG_API(printk("create session, alloc dev\n"));
3086 return -ENOMEM;
3087 }
3088
3089 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3090 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3091
3092 if (err) {
3093 kfree(dev);
3094 return err;
3095 }
3096 sess_op->ses_id = sid;
3097 dev->sid = sid;
3098 dev->next = filp->private_data;
3099 filp->private_data = dev;
3100
3101 return 0;
3102 }
3103
3104 static int cryptocop_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
3105 {
3106 int err = 0;
3107 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3108 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3109 return -ENOTTY;
3110 }
3111 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3112 return -ENOTTY;
3113 }
3114 /* Access check of the argument. Some commands, e.g. create session and process op,
3115 needs additional checks. Those are handled in the command handling functions. */
3116 if (_IOC_DIR(cmd) & _IOC_READ)
3117 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3118 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3119 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3120 if (err) return -EFAULT;
3121
3122 switch (cmd) {
3123 case CRYPTOCOP_IO_CREATE_SESSION:
3124 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3125 case CRYPTOCOP_IO_CLOSE_SESSION:
3126 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3127 case CRYPTOCOP_IO_PROCESS_OP:
3128 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3129 default:
3130 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3131 return -ENOTTY;
3132 }
3133 return 0;
3134 }
3135
3136
3137 #ifdef LDEBUG
3138 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3139 {
3140 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3141 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3142 int i;
3143
3144 printk("print_dma_descriptors start\n");
3145
3146 printk("iop:\n");
3147 printk("\tsid: 0x%lld\n", iop->sid);
3148
3149 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3150 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3151 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3152 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3153
3154 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3155 printk("\tnext: 0x%p\n"
3156 "\tsaved_data: 0x%p\n"
3157 "\tsaved_data_buf: 0x%p\n",
3158 iop->ctx_out.next,
3159 iop->ctx_out.saved_data,
3160 iop->ctx_out.saved_data_buf);
3161
3162 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3163 printk("\tnext: 0x%p\n"
3164 "\tsaved_data: 0x%p\n"
3165 "\tsaved_data_buf: 0x%p\n",
3166 iop->ctx_in.next,
3167 iop->ctx_in.saved_data,
3168 iop->ctx_in.saved_data_buf);
3169
3170 i = 0;
3171 while (cdesc_out) {
3172 dma_descr_data *td;
3173 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3174 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3175 td = cdesc_out->dma_descr;
3176 printk("\n\tbuf: 0x%p\n"
3177 "\tafter: 0x%p\n"
3178 "\tmd: 0x%04x\n"
3179 "\tnext: 0x%p\n",
3180 td->buf,
3181 td->after,
3182 td->md,
3183 td->next);
3184 printk("flags:\n"
3185 "\twait:\t%d\n"
3186 "\teol:\t%d\n"
3187 "\touteop:\t%d\n"
3188 "\tineop:\t%d\n"
3189 "\tintr:\t%d\n",
3190 td->wait,
3191 td->eol,
3192 td->out_eop,
3193 td->in_eop,
3194 td->intr);
3195 cdesc_out = cdesc_out->next;
3196 i++;
3197 }
3198 i = 0;
3199 while (cdesc_in) {
3200 dma_descr_data *td;
3201 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3202 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3203 td = cdesc_in->dma_descr;
3204 printk("\n\tbuf: 0x%p\n"
3205 "\tafter: 0x%p\n"
3206 "\tmd: 0x%04x\n"
3207 "\tnext: 0x%p\n",
3208 td->buf,
3209 td->after,
3210 td->md,
3211 td->next);
3212 printk("flags:\n"
3213 "\twait:\t%d\n"
3214 "\teol:\t%d\n"
3215 "\touteop:\t%d\n"
3216 "\tineop:\t%d\n"
3217 "\tintr:\t%d\n",
3218 td->wait,
3219 td->eol,
3220 td->out_eop,
3221 td->in_eop,
3222 td->intr);
3223 cdesc_in = cdesc_in->next;
3224 i++;
3225 }
3226
3227 printk("print_dma_descriptors end\n");
3228 }
3229
3230
3231 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3232 {
3233 printk("print_strcop_crypto_op, 0x%p\n", cop);
3234
3235 /* Indata. */
3236 printk("indata=0x%p\n"
3237 "inlen=%d\n"
3238 "do_cipher=%d\n"
3239 "decrypt=%d\n"
3240 "cipher_explicit=%d\n"
3241 "cipher_start=%d\n"
3242 "cipher_len=%d\n"
3243 "outdata=0x%p\n"
3244 "outlen=%d\n",
3245 cop->indata,
3246 cop->inlen,
3247 cop->do_cipher,
3248 cop->decrypt,
3249 cop->cipher_explicit,
3250 cop->cipher_start,
3251 cop->cipher_len,
3252 cop->cipher_outdata,
3253 cop->cipher_outlen);
3254
3255 printk("do_digest=%d\n"
3256 "digest_start=%d\n"
3257 "digest_len=%d\n",
3258 cop->do_digest,
3259 cop->digest_start,
3260 cop->digest_len);
3261
3262 printk("do_csum=%d\n"
3263 "csum_start=%d\n"
3264 "csum_len=%d\n",
3265 cop->do_csum,
3266 cop->csum_start,
3267 cop->csum_len);
3268 }
3269
3270 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3271 {
3272 struct cryptocop_desc *d;
3273 struct cryptocop_tfrm_cfg *tc;
3274 struct cryptocop_desc_cfg *dc;
3275 int i;
3276
3277 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3278 printk("sid: %lld\n", cop->sid);
3279 printk("operation_status=%d\n"
3280 "use_dmalists=%d\n"
3281 "in_interrupt=%d\n"
3282 "fast_callback=%d\n",
3283 cop->operation_status,
3284 cop->use_dmalists,
3285 cop->in_interrupt,
3286 cop->fast_callback);
3287
3288 if (cop->use_dmalists){
3289 print_user_dma_lists(&cop->list_op);
3290 } else {
3291 printk("cop->tfrm_op\n"
3292 "tfrm_cfg=0x%p\n"
3293 "desc=0x%p\n"
3294 "indata=0x%p\n"
3295 "incount=%d\n"
3296 "inlen=%d\n"
3297 "outdata=0x%p\n"
3298 "outcount=%d\n"
3299 "outlen=%d\n\n",
3300 cop->tfrm_op.tfrm_cfg,
3301 cop->tfrm_op.desc,
3302 cop->tfrm_op.indata,
3303 cop->tfrm_op.incount,
3304 cop->tfrm_op.inlen,
3305 cop->tfrm_op.outdata,
3306 cop->tfrm_op.outcount,
3307 cop->tfrm_op.outlen);
3308
3309 tc = cop->tfrm_op.tfrm_cfg;
3310 while (tc){
3311 printk("tfrm_cfg, 0x%p\n"
3312 "tid=%d\n"
3313 "flags=%d\n"
3314 "inject_ix=%d\n"
3315 "next=0x%p\n",
3316 tc,
3317 tc->tid,
3318 tc->flags,
3319 tc->inject_ix,
3320 tc->next);
3321 tc = tc->next;
3322 }
3323 d = cop->tfrm_op.desc;
3324 while (d){
3325 printk("\n======================desc, 0x%p\n"
3326 "length=%d\n"
3327 "cfg=0x%p\n"
3328 "next=0x%p\n",
3329 d,
3330 d->length,
3331 d->cfg,
3332 d->next);
3333 dc = d->cfg;
3334 while (dc){
3335 printk("=========desc_cfg, 0x%p\n"
3336 "tid=%d\n"
3337 "src=%d\n"
3338 "last=%d\n"
3339 "next=0x%p\n",
3340 dc,
3341 dc->tid,
3342 dc->src,
3343 dc->last,
3344 dc->next);
3345 dc = dc->next;
3346 }
3347 d = d->next;
3348 }
3349 printk("\n====iniov\n");
3350 for (i = 0; i < cop->tfrm_op.incount; i++){
3351 printk("indata[%d]\n"
3352 "base=0x%p\n"
3353 "len=%d\n",
3354 i,
3355 cop->tfrm_op.indata[i].iov_base,
3356 cop->tfrm_op.indata[i].iov_len);
3357 }
3358 printk("\n====outiov\n");
3359 for (i = 0; i < cop->tfrm_op.outcount; i++){
3360 printk("outdata[%d]\n"
3361 "base=0x%p\n"
3362 "len=%d\n",
3363 i,
3364 cop->tfrm_op.outdata[i].iov_base,
3365 cop->tfrm_op.outdata[i].iov_len);
3366 }
3367 }
3368 printk("------------end print_cryptocop_operation\n");
3369 }
3370
3371
3372 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3373 {
3374 dma_descr_data *dd;
3375 int i;
3376
3377 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3378
3379 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3380 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3381
3382 printk("##############outlist\n");
3383 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3384 i = 0;
3385 while (dd != NULL) {
3386 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3387 printk("\n\tbuf: 0x%p\n"
3388 "\tafter: 0x%p\n"
3389 "\tmd: 0x%04x\n"
3390 "\tnext: 0x%p\n",
3391 dd->buf,
3392 dd->after,
3393 dd->md,
3394 dd->next);
3395 printk("flags:\n"
3396 "\twait:\t%d\n"
3397 "\teol:\t%d\n"
3398 "\touteop:\t%d\n"
3399 "\tineop:\t%d\n"
3400 "\tintr:\t%d\n",
3401 dd->wait,
3402 dd->eol,
3403 dd->out_eop,
3404 dd->in_eop,
3405 dd->intr);
3406 if (dd->eol)
3407 dd = NULL;
3408 else
3409 dd = phys_to_virt((unsigned long int)dd->next);
3410 ++i;
3411 }
3412
3413 printk("##############inlist\n");
3414 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3415 i = 0;
3416 while (dd != NULL) {
3417 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3418 printk("\n\tbuf: 0x%p\n"
3419 "\tafter: 0x%p\n"
3420 "\tmd: 0x%04x\n"
3421 "\tnext: 0x%p\n",
3422 dd->buf,
3423 dd->after,
3424 dd->md,
3425 dd->next);
3426 printk("flags:\n"
3427 "\twait:\t%d\n"
3428 "\teol:\t%d\n"
3429 "\touteop:\t%d\n"
3430 "\tineop:\t%d\n"
3431 "\tintr:\t%d\n",
3432 dd->wait,
3433 dd->eol,
3434 dd->out_eop,
3435 dd->in_eop,
3436 dd->intr);
3437 if (dd->eol)
3438 dd = NULL;
3439 else
3440 dd = phys_to_virt((unsigned long int)dd->next);
3441 ++i;
3442 }
3443 }
3444
3445
3446 static void print_lock_status(void)
3447 {
3448 printk("**********************print_lock_status\n");
3449 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3450 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3451 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3452 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3453 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3454 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3455 }
3456 #endif /* LDEBUG */
3457
3458
3459 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3460
3461 static int init_stream_coprocessor(void)
3462 {
3463 int err;
3464 int i;
3465 static int initialized = 0;
3466
3467 if (initialized)
3468 return 0;
3469
3470 initialized = 1;
3471
3472 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3473
3474 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3475 if (err < 0) {
3476 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3477 return err;
3478 }
3479
3480 err = init_cryptocop();
3481 if (err) {
3482 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3483 return err;
3484 }
3485 err = cryptocop_job_queue_init();
3486 if (err) {
3487 release_cryptocop();
3488 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3489 return err;
3490 }
3491 /* Init the descriptor pool. */
3492 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3493 descr_pool[i].from_pool = 1;
3494 descr_pool[i].next = &descr_pool[i + 1];
3495 }
3496 descr_pool[i].from_pool = 1;
3497 descr_pool[i].next = NULL;
3498 descr_pool_free_list = &descr_pool[0];
3499 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3500
3501 spin_lock_init(&cryptocop_completed_jobs_lock);
3502 spin_lock_init(&cryptocop_job_queue_lock);
3503 spin_lock_init(&descr_pool_lock);
3504 spin_lock_init(&cryptocop_sessions_lock);
3505 spin_lock_init(&running_job_lock);
3506 spin_lock_init(&cryptocop_process_lock);
3507
3508 cryptocop_sessions = NULL;
3509 next_sid = 1;
3510
3511 cryptocop_running_job = NULL;
3512
3513 printk("stream co-processor: init done.\n");
3514 return 0;
3515 }
3516
3517 static void __exit exit_stream_coprocessor(void)
3518 {
3519 release_cryptocop();
3520 cryptocop_job_queue_close();
3521 }
3522
3523 module_init(init_stream_coprocessor);
3524 module_exit(exit_stream_coprocessor);
3525
WRT350N Kernel Patches