?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
warning removal
[cor_2_6_31.git] / arch / cris / arch-v32 / drivers / cryptocop.c
blobfd529a0ec758aa3a018df065729616310a39cf12
1 /*
2 * Stream co-processor driver for the ETRAX FS
3 *
4 * Copyright (C) 2003-2007 Axis Communications AB
5 */
6
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/smp_lock.h>
15 #include <linux/spinlock.h>
16 #include <linux/stddef.h>
17
18 #include <asm/uaccess.h>
19 #include <asm/io.h>
20 #include <asm/atomic.h>
21
22 #include <linux/list.h>
23 #include <linux/interrupt.h>
24
25 #include <asm/signal.h>
26 #include <asm/irq.h>
27
28 #include <dma.h>
29 #include <hwregs/dma.h>
30 #include <hwregs/reg_map.h>
31 #include <hwregs/reg_rdwr.h>
32 #include <hwregs/intr_vect_defs.h>
33
34 #include <hwregs/strcop.h>
35 #include <hwregs/strcop_defs.h>
36 #include <cryptocop.h>
37
38 #ifdef CONFIG_ETRAXFS
39 #define IN_DMA 9
40 #define OUT_DMA 8
41 #define IN_DMA_INST regi_dma9
42 #define OUT_DMA_INST regi_dma8
43 #define DMA_IRQ DMA9_INTR_VECT
44 #else
45 #define IN_DMA 3
46 #define OUT_DMA 2
47 #define IN_DMA_INST regi_dma3
48 #define OUT_DMA_INST regi_dma2
49 #define DMA_IRQ DMA3_INTR_VECT
50 #endif
51
52 #define DESCR_ALLOC_PAD (31)
53
54 struct cryptocop_dma_desc {
55 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
56 dma_descr_data *dma_descr;
57
58 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
59
60 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
61 struct cryptocop_dma_desc *next;
62 };
63
64
65 struct cryptocop_int_operation{
66 void *alloc_ptr;
67 cryptocop_session_id sid;
68
69 dma_descr_context ctx_out;
70 dma_descr_context ctx_in;
71
72 /* DMA descriptors allocated by driver. */
73 struct cryptocop_dma_desc *cdesc_out;
74 struct cryptocop_dma_desc *cdesc_in;
75
76 /* Strcop config to use. */
77 cryptocop_3des_mode tdes_mode;
78 cryptocop_csum_type csum_mode;
79
80 /* DMA descrs provided by consumer. */
81 dma_descr_data *ddesc_out;
82 dma_descr_data *ddesc_in;
83 };
84
85
86 struct cryptocop_tfrm_ctx {
87 cryptocop_tfrm_id tid;
88 unsigned int blocklength;
89
90 unsigned int start_ix;
91
92 struct cryptocop_tfrm_cfg *tcfg;
93 struct cryptocop_transform_ctx *tctx;
94
95 unsigned char previous_src;
96 unsigned char current_src;
97
98 /* Values to use in metadata out. */
99 unsigned char hash_conf;
100 unsigned char hash_mode;
101 unsigned char ciph_conf;
102 unsigned char cbcmode;
103 unsigned char decrypt;
104
105 unsigned int requires_padding:1;
106 unsigned int strict_block_length:1;
107 unsigned int active:1;
108 unsigned int done:1;
109 size_t consumed;
110 size_t produced;
111
112 /* Pad (input) descriptors to put in the DMA out list when the transform
113 * output is put on the DMA in list. */
114 struct cryptocop_dma_desc *pad_descs;
115
116 struct cryptocop_tfrm_ctx *prev_src;
117 struct cryptocop_tfrm_ctx *curr_src;
118
119 /* Mapping to HW. */
120 unsigned char unit_no;
121 };
122
123
124 struct cryptocop_private{
125 cryptocop_session_id sid;
126 struct cryptocop_private *next;
127 };
128
129 /* Session list. */
130
131 struct cryptocop_transform_ctx{
132 struct cryptocop_transform_init init;
133 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
134 unsigned int dec_key_set:1;
135
136 struct cryptocop_transform_ctx *next;
137 };
138
139
140 struct cryptocop_session{
141 cryptocop_session_id sid;
142
143 struct cryptocop_transform_ctx *tfrm_ctx;
144
145 struct cryptocop_session *next;
146 };
147
148 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
149 kernel have highest priority since TCPIP stack processing must not
150 be a bottleneck. */
151 typedef enum {
152 cryptocop_prio_kernel_csum = 0,
153 cryptocop_prio_kernel = 1,
154 cryptocop_prio_user = 2,
155 cryptocop_prio_no_prios = 3
156 } cryptocop_queue_priority;
157
158 struct cryptocop_prio_queue{
159 struct list_head jobs;
160 cryptocop_queue_priority prio;
161 };
162
163 struct cryptocop_prio_job{
164 struct list_head node;
165 cryptocop_queue_priority prio;
166
167 struct cryptocop_operation *oper;
168 struct cryptocop_int_operation *iop;
169 };
170
171 struct ioctl_job_cb_ctx {
172 unsigned int processed:1;
173 };
174
175
176 static struct cryptocop_session *cryptocop_sessions = NULL;
177 spinlock_t cryptocop_sessions_lock;
178
179 /* Next Session ID to assign. */
180 static cryptocop_session_id next_sid = 1;
181
182 /* Pad for checksum. */
183 static const char csum_zero_pad[1] = {0x00};
184
185 /* Trash buffer for mem2mem operations. */
186 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
187 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
188
189 /* Descriptor pool. */
190 /* FIXME Tweak this value. */
191 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
192 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
193 static struct cryptocop_dma_desc *descr_pool_free_list;
194 static int descr_pool_no_free;
195 static spinlock_t descr_pool_lock;
196
197 /* Lock to stop cryptocop to start processing of a new operation. The holder
198 of this lock MUST call cryptocop_start_job() after it is unlocked. */
199 spinlock_t cryptocop_process_lock;
200
201 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
202 static spinlock_t cryptocop_job_queue_lock;
203 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
204 static spinlock_t running_job_lock;
205
206 /* The interrupt handler appends completed jobs to this list. The scehduled
207 * tasklet removes them upon sending the response to the crypto consumer. */
208 static struct list_head cryptocop_completed_jobs;
209 static spinlock_t cryptocop_completed_jobs_lock;
210
211 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
212
213
214 /** Local functions. **/
215
216 static int cryptocop_open(struct inode *, struct file *);
217
218 static int cryptocop_release(struct inode *, struct file *);
219
220 static int cryptocop_ioctl(struct inode *inode, struct file *file,
221 unsigned int cmd, unsigned long arg);
222
223 static void cryptocop_start_job(void);
224
225 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
226 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
227
228 static int cryptocop_job_queue_init(void);
229 static void cryptocop_job_queue_close(void);
230
231 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
232
233 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
234
235 static int transform_ok(struct cryptocop_transform_init *tinit);
236
237 static struct cryptocop_session *get_session(cryptocop_session_id sid);
238
239 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
240
241 static void delete_internal_operation(struct cryptocop_int_operation *iop);
242
243 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
244
245 static int init_stream_coprocessor(void);
246
247 static void __exit exit_stream_coprocessor(void);
248
249 /*#define LDEBUG*/
250 #ifdef LDEBUG
251 #define DEBUG(s) s
252 #define DEBUG_API(s) s
253 static void print_cryptocop_operation(struct cryptocop_operation *cop);
254 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
255 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
256 static void print_lock_status(void);
257 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
258 #define assert(s) do{if (!(s)) panic(#s);} while(0);
259 #else
260 #define DEBUG(s)
261 #define DEBUG_API(s)
262 #define assert(s)
263 #endif
264
265
266 /* Transform constants. */
267 #define DES_BLOCK_LENGTH (8)
268 #define AES_BLOCK_LENGTH (16)
269 #define MD5_BLOCK_LENGTH (64)
270 #define SHA1_BLOCK_LENGTH (64)
271 #define CSUM_BLOCK_LENGTH (2)
272 #define MD5_STATE_LENGTH (16)
273 #define SHA1_STATE_LENGTH (20)
274
275 /* The device number. */
276 #define CRYPTOCOP_MAJOR (254)
277 #define CRYPTOCOP_MINOR (0)
278
279
280
281 const struct file_operations cryptocop_fops = {
282 .owner = THIS_MODULE,
283 .open = cryptocop_open,
284 .release = cryptocop_release,
285 .ioctl = cryptocop_ioctl
286 };
287
288
289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290 {
291 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
292 kfree(cdesc->free_buf);
293
294 if (cdesc->from_pool) {
295 unsigned long int flags;
296 spin_lock_irqsave(&descr_pool_lock, flags);
297 cdesc->next = descr_pool_free_list;
298 descr_pool_free_list = cdesc;
299 ++descr_pool_no_free;
300 spin_unlock_irqrestore(&descr_pool_lock, flags);
301 } else {
302 kfree(cdesc);
303 }
304 }
305
306
307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308 {
309 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 struct cryptocop_dma_desc *cdesc;
311
312 if (use_pool) {
313 unsigned long int flags;
314 spin_lock_irqsave(&descr_pool_lock, flags);
315 if (!descr_pool_free_list) {
316 spin_unlock_irqrestore(&descr_pool_lock, flags);
317 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 return NULL;
319 }
320 cdesc = descr_pool_free_list;
321 descr_pool_free_list = descr_pool_free_list->next;
322 --descr_pool_no_free;
323 spin_unlock_irqrestore(&descr_pool_lock, flags);
324 cdesc->from_pool = 1;
325 } else {
326 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 if (!cdesc) {
328 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 return NULL;
330 }
331 cdesc->from_pool = 0;
332 }
333 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334
335 cdesc->next = NULL;
336
337 cdesc->free_buf = NULL;
338 cdesc->dma_descr->out_eop = 0;
339 cdesc->dma_descr->in_eop = 0;
340 cdesc->dma_descr->intr = 0;
341 cdesc->dma_descr->eol = 0;
342 cdesc->dma_descr->wait = 0;
343 cdesc->dma_descr->buf = NULL;
344 cdesc->dma_descr->after = NULL;
345
346 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 return cdesc;
348 }
349
350
351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352 {
353 DEBUG(printk("setup_descr_chain: entering\n"));
354 while (cd) {
355 if (cd->next) {
356 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 } else {
358 cd->dma_descr->next = NULL;
359 }
360 cd = cd->next;
361 }
362 DEBUG(printk("setup_descr_chain: exit\n"));
363 }
364
365
366 /* Create a pad descriptor for the transform.
367 * Return -1 for error, 0 if pad created. */
368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369 {
370 struct cryptocop_dma_desc *cdesc = NULL;
371 int error = 0;
372 struct strcop_meta_out mo = {
373 .ciphsel = src_none,
374 .hashsel = src_none,
375 .csumsel = src_none
376 };
377 char *pad;
378 size_t plen;
379
380 DEBUG(printk("create_pad_descriptor: start.\n"));
381 /* Setup pad descriptor. */
382
383 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 cdesc = alloc_cdesc(alloc_flag);
385 if (!cdesc){
386 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 goto error_cleanup;
388 }
389 switch (tc->unit_no) {
390 case src_md5:
391 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 if (error){
393 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 goto error_cleanup;
395 }
396 cdesc->free_buf = pad;
397 mo.hashsel = src_dma;
398 mo.hashconf = tc->hash_conf;
399 mo.hashmode = tc->hash_mode;
400 break;
401 case src_sha1:
402 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 if (error){
404 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 goto error_cleanup;
406 }
407 cdesc->free_buf = pad;
408 mo.hashsel = src_dma;
409 mo.hashconf = tc->hash_conf;
410 mo.hashmode = tc->hash_mode;
411 break;
412 case src_csum:
413 if (tc->consumed % tc->blocklength){
414 pad = (char*)csum_zero_pad;
415 plen = 1;
416 } else {
417 pad = (char*)cdesc; /* Use any pointer. */
418 plen = 0;
419 }
420 mo.csumsel = src_dma;
421 break;
422 }
423 cdesc->dma_descr->wait = 1;
424 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
425 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427
428 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 *pad_desc = cdesc;
430
431 return 0;
432
433 error_cleanup:
434 if (cdesc) free_cdesc(cdesc);
435 return -1;
436 }
437
438
439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440 {
441 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
442 struct strcop_meta_out mo = {0};
443
444 DEBUG(printk("setup_key_dl_desc\n"));
445
446 if (!key_desc) {
447 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 return -ENOMEM;
449 }
450
451 /* Download key. */
452 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 /* Precook the AES decrypt key. */
454 if (!tc->tctx->dec_key_set){
455 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 tc->tctx->dec_key_set = 1;
457 }
458 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 } else {
461 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 }
464 /* Setup metadata. */
465 mo.dlkey = 1;
466 switch (tc->tctx->init.keylen) {
467 case 64:
468 mo.decrypt = 0;
469 mo.hashmode = 0;
470 break;
471 case 128:
472 mo.decrypt = 0;
473 mo.hashmode = 1;
474 break;
475 case 192:
476 mo.decrypt = 1;
477 mo.hashmode = 0;
478 break;
479 case 256:
480 mo.decrypt = 1;
481 mo.hashmode = 1;
482 break;
483 default:
484 break;
485 }
486 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488
489 key_desc->dma_descr->out_eop = 1;
490 key_desc->dma_descr->wait = 1;
491 key_desc->dma_descr->intr = 0;
492
493 *kd = key_desc;
494 return 0;
495 }
496
497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498 {
499 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
500 struct strcop_meta_out mo = {0};
501
502 DEBUG(printk("setup_cipher_iv_desc\n"));
503
504 if (!iv_desc) {
505 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 return -ENOMEM;
507 }
508 /* Download IV. */
509 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511
512 /* Setup metadata. */
513 mo.hashsel = mo.csumsel = src_none;
514 mo.ciphsel = src_dma;
515 mo.ciphconf = tc->ciph_conf;
516 mo.cbcmode = tc->cbcmode;
517
518 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519
520 iv_desc->dma_descr->out_eop = 0;
521 iv_desc->dma_descr->wait = 1;
522 iv_desc->dma_descr->intr = 0;
523
524 *id = iv_desc;
525 return 0;
526 }
527
528 /* Map the ouput length of the transform to operation output starting on the inject index. */
529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530 {
531 int err = 0;
532 struct cryptocop_dma_desc head = {0};
533 struct cryptocop_dma_desc *outdesc = &head;
534 size_t iov_offset = 0;
535 size_t out_ix = 0;
536 int outiov_ix = 0;
537 struct strcop_meta_in mi = {0};
538
539 size_t out_length = tc->produced;
540 int rem_length;
541 int dlength;
542
543 assert(out_length != 0);
544 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 return -EINVAL;
547 }
548 /* Traverse the out iovec until the result inject index is reached. */
549 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 outiov_ix++;
552 }
553 if (outiov_ix >= operation->tfrm_op.outcount){
554 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 return -EINVAL;
556 }
557 iov_offset = tc->tcfg->inject_ix - out_ix;
558 mi.dmasel = tc->unit_no;
559
560 /* Setup the output descriptors. */
561 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 outdesc->next = alloc_cdesc(alloc_flag);
563 if (!outdesc->next) {
564 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 err = -ENOMEM;
566 goto error_cleanup;
567 }
568 outdesc = outdesc->next;
569 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 dlength = (out_length < rem_length) ? out_length : rem_length;
571
572 DEBUG(printk("create_input_descriptors:\n"
573 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 "outcount=%d, outiov_ix=%d\n",
576 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577
578 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581
582 out_length -= dlength;
583 iov_offset += dlength;
584 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 iov_offset = 0;
586 ++outiov_ix;
587 }
588 }
589 if (out_length > 0){
590 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 err = -EINVAL;
592 goto error_cleanup;
593 }
594 /* Set sync in last descriptor. */
595 mi.sync = 1;
596 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597
598 *id = head.next;
599 return 0;
600
601 error_cleanup:
602 while (head.next) {
603 outdesc = head.next->next;
604 free_cdesc(head.next);
605 head.next = outdesc;
606 }
607 return err;
608 }
609
610
611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612 {
613 while (desc_len != 0) {
614 struct cryptocop_dma_desc *cdesc;
615 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
617
618 cdesc = alloc_cdesc(alloc_flag);
619 if (!cdesc) {
620 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 return -ENOMEM;
622 }
623 (*current_out_cdesc)->next = cdesc;
624 (*current_out_cdesc) = cdesc;
625
626 cdesc->free_buf = NULL;
627
628 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630
631 desc_len -= dlength;
632 *iniov_offset += dlength;
633 assert(desc_len >= 0);
634 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 *iniov_offset = 0;
636 ++(*iniov_ix);
637 if (*iniov_ix > operation->tfrm_op.incount) {
638 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 return -EINVAL;
640 }
641 }
642 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 } /* while (desc_len != 0) */
644 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646
647 return 0;
648 }
649
650
651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652 {
653 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 if (tc->tcfg) {
655 int failed = 0;
656 struct cryptocop_dma_desc *idescs = NULL;
657 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 if (tc->pad_descs) {
659 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 while (tc->pad_descs) {
661 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 (*current_out_cdesc)->next = tc->pad_descs;
663 tc->pad_descs = tc->pad_descs->next;
664 (*current_out_cdesc) = (*current_out_cdesc)->next;
665 }
666 }
667
668 /* Setup and append output descriptors to DMA in list. */
669 if (tc->unit_no == src_dma){
670 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 unsigned int start_ix = tc->start_ix;
673 while (start_ix){
674 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 if (!(*current_in_cdesc)->next){
677 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 return -ENOMEM;
679 }
680 (*current_in_cdesc) = (*current_in_cdesc)->next;
681 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 start_ix -= desclen;
685 }
686 mi.sync = 1;
687 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 }
689
690 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 if (failed){
692 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 return failed;
694 }
695 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 while (idescs) {
697 DEBUG(printk("append descriptor 0x%p\n", idescs));
698 (*current_in_cdesc)->next = idescs;
699 idescs = idescs->next;
700 (*current_in_cdesc) = (*current_in_cdesc)->next;
701 }
702 }
703 return 0;
704 }
705
706
707
708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709 {
710 struct cryptocop_session *sess;
711 struct cryptocop_transform_ctx *tctx;
712
713 struct cryptocop_tfrm_ctx digest_ctx = {
714 .previous_src = src_none,
715 .current_src = src_none,
716 .start_ix = 0,
717 .requires_padding = 1,
718 .strict_block_length = 0,
719 .hash_conf = 0,
720 .hash_mode = 0,
721 .ciph_conf = 0,
722 .cbcmode = 0,
723 .decrypt = 0,
724 .consumed = 0,
725 .produced = 0,
726 .pad_descs = NULL,
727 .active = 0,
728 .done = 0,
729 .prev_src = NULL,
730 .curr_src = NULL,
731 .tcfg = NULL};
732 struct cryptocop_tfrm_ctx cipher_ctx = {
733 .previous_src = src_none,
734 .current_src = src_none,
735 .start_ix = 0,
736 .requires_padding = 0,
737 .strict_block_length = 1,
738 .hash_conf = 0,
739 .hash_mode = 0,
740 .ciph_conf = 0,
741 .cbcmode = 0,
742 .decrypt = 0,
743 .consumed = 0,
744 .produced = 0,
745 .pad_descs = NULL,
746 .active = 0,
747 .done = 0,
748 .prev_src = NULL,
749 .curr_src = NULL,
750 .tcfg = NULL};
751 struct cryptocop_tfrm_ctx csum_ctx = {
752 .previous_src = src_none,
753 .current_src = src_none,
754 .start_ix = 0,
755 .blocklength = 2,
756 .requires_padding = 1,
757 .strict_block_length = 0,
758 .hash_conf = 0,
759 .hash_mode = 0,
760 .ciph_conf = 0,
761 .cbcmode = 0,
762 .decrypt = 0,
763 .consumed = 0,
764 .produced = 0,
765 .pad_descs = NULL,
766 .active = 0,
767 .done = 0,
768 .tcfg = NULL,
769 .prev_src = NULL,
770 .curr_src = NULL,
771 .unit_no = src_csum};
772 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773
774 unsigned int indata_ix = 0;
775
776 /* iovec accounting. */
777 int iniov_ix = 0;
778 int iniov_offset = 0;
779
780 /* Operation descriptor cfg traversal pointer. */
781 struct cryptocop_desc *odsc;
782
783 int failed = 0;
784 /* List heads for allocated descriptors. */
785 struct cryptocop_dma_desc out_cdesc_head = {0};
786 struct cryptocop_dma_desc in_cdesc_head = {0};
787
788 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790
791 struct cryptocop_tfrm_ctx *output_tc = NULL;
792 void *iop_alloc_ptr;
793
794 assert(operation != NULL);
795 assert(int_op != NULL);
796
797 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 DEBUG(print_cryptocop_operation(operation));
799
800 sess = get_session(operation->sid);
801 if (!sess) {
802 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 failed = -EINVAL;
804 goto error_cleanup;
805 }
806 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 if (!iop_alloc_ptr) {
808 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
809 failed = -ENOMEM;
810 goto error_cleanup;
811 }
812 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 (*int_op)->alloc_ptr = iop_alloc_ptr;
815 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816
817 (*int_op)->sid = operation->sid;
818 (*int_op)->cdesc_out = NULL;
819 (*int_op)->cdesc_in = NULL;
820 (*int_op)->tdes_mode = cryptocop_3des_ede;
821 (*int_op)->csum_mode = cryptocop_csum_le;
822 (*int_op)->ddesc_out = NULL;
823 (*int_op)->ddesc_in = NULL;
824
825 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 if (!tcfg) {
827 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 failed = -EINVAL;
829 goto error_cleanup;
830 }
831 while (tcfg) {
832 tctx = get_transform_ctx(sess, tcfg->tid);
833 if (!tctx) {
834 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 failed = -EINVAL;
836 goto error_cleanup;
837 }
838 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 failed = -EINVAL;
841 goto error_cleanup;
842 }
843 switch (tctx->init.alg){
844 case cryptocop_alg_mem2mem:
845 if (cipher_ctx.tcfg != NULL){
846 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 failed = -EINVAL;
848 goto error_cleanup;
849 }
850 /* mem2mem is handled as a NULL cipher. */
851 cipher_ctx.cbcmode = 0;
852 cipher_ctx.decrypt = 0;
853 cipher_ctx.blocklength = 1;
854 cipher_ctx.ciph_conf = 0;
855 cipher_ctx.unit_no = src_dma;
856 cipher_ctx.tcfg = tcfg;
857 cipher_ctx.tctx = tctx;
858 break;
859 case cryptocop_alg_des:
860 case cryptocop_alg_3des:
861 case cryptocop_alg_aes:
862 /* cipher */
863 if (cipher_ctx.tcfg != NULL){
864 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 failed = -EINVAL;
866 goto error_cleanup;
867 }
868 cipher_ctx.tcfg = tcfg;
869 cipher_ctx.tctx = tctx;
870 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 cipher_ctx.decrypt = 1;
872 }
873 switch (tctx->init.cipher_mode) {
874 case cryptocop_cipher_mode_ecb:
875 cipher_ctx.cbcmode = 0;
876 break;
877 case cryptocop_cipher_mode_cbc:
878 cipher_ctx.cbcmode = 1;
879 break;
880 default:
881 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 failed = -EINVAL;
883 goto error_cleanup;
884 }
885 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 switch (tctx->init.alg){
887 case cryptocop_alg_des:
888 cipher_ctx.ciph_conf = 0;
889 cipher_ctx.unit_no = src_des;
890 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 break;
892 case cryptocop_alg_3des:
893 cipher_ctx.ciph_conf = 1;
894 cipher_ctx.unit_no = src_des;
895 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 break;
897 case cryptocop_alg_aes:
898 cipher_ctx.ciph_conf = 2;
899 cipher_ctx.unit_no = src_aes;
900 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 break;
902 default:
903 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 }
905 (*int_op)->tdes_mode = tctx->init.tdes_mode;
906 break;
907 case cryptocop_alg_md5:
908 case cryptocop_alg_sha1:
909 /* digest */
910 if (digest_ctx.tcfg != NULL){
911 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 failed = -EINVAL;
913 goto error_cleanup;
914 }
915 digest_ctx.tcfg = tcfg;
916 digest_ctx.tctx = tctx;
917 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 switch (tctx->init.alg){
919 case cryptocop_alg_md5:
920 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 digest_ctx.unit_no = src_md5;
922 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 break;
924 case cryptocop_alg_sha1:
925 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 digest_ctx.unit_no = src_sha1;
927 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 break;
929 default:
930 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 }
932 break;
933 case cryptocop_alg_csum:
934 /* digest */
935 if (csum_ctx.tcfg != NULL){
936 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 failed = -EINVAL;
938 goto error_cleanup;
939 }
940 (*int_op)->csum_mode = tctx->init.csum_mode;
941 csum_ctx.tcfg = tcfg;
942 csum_ctx.tctx = tctx;
943 break;
944 default:
945 /* no algorithm. */
946 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 failed = -EINVAL;
948 goto error_cleanup;
949 }
950 tcfg = tcfg->next;
951 }
952 /* Download key if a cipher is used. */
953 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 struct cryptocop_dma_desc *key_desc = NULL;
955
956 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 if (failed) {
958 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 goto error_cleanup;
960 }
961 current_out_cdesc->next = key_desc;
962 current_out_cdesc = key_desc;
963 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964
965 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 struct cryptocop_dma_desc *iv_desc = NULL;
968
969 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970
971 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 if (failed) {
973 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 goto error_cleanup;
975 }
976 current_out_cdesc->next = iv_desc;
977 current_out_cdesc = iv_desc;
978 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 }
980 }
981
982 /* Process descriptors. */
983 odsc = operation->tfrm_op.desc;
984 while (odsc) {
985 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
986 struct strcop_meta_out meta_out = {0};
987 size_t desc_len = odsc->length;
988 int active_count, eop_needed_count;
989
990 output_tc = NULL;
991
992 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993
994 while (dcfg) {
995 struct cryptocop_tfrm_ctx *tc = NULL;
996
997 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 /* Get the local context for the transform and mark it as the output unit if it produces output. */
999 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 tc = &digest_ctx;
1001 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 tc = &cipher_ctx;
1003 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 tc = &csum_ctx;
1005 }
1006 if (!tc) {
1007 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 failed = -EINVAL;
1009 goto error_cleanup;
1010 }
1011 if (tc->done) {
1012 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 failed = -EINVAL;
1014 goto error_cleanup;
1015 }
1016 if (!tc->active) {
1017 tc->start_ix = indata_ix;
1018 tc->active = 1;
1019 }
1020
1021 tc->previous_src = tc->current_src;
1022 tc->prev_src = tc->curr_src;
1023 /* Map source unit id to DMA source config. */
1024 switch (dcfg->src){
1025 case cryptocop_source_dma:
1026 tc->current_src = src_dma;
1027 break;
1028 case cryptocop_source_des:
1029 tc->current_src = src_des;
1030 break;
1031 case cryptocop_source_3des:
1032 tc->current_src = src_des;
1033 break;
1034 case cryptocop_source_aes:
1035 tc->current_src = src_aes;
1036 break;
1037 case cryptocop_source_md5:
1038 case cryptocop_source_sha1:
1039 case cryptocop_source_csum:
1040 case cryptocop_source_none:
1041 default:
1042 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 */
1044 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 failed = -EINVAL;
1046 goto error_cleanup;
1047 }
1048 if (tc->current_src != src_dma) {
1049 /* Find the unit we are sourcing from. */
1050 if (digest_ctx.unit_no == tc->current_src){
1051 tc->curr_src = &digest_ctx;
1052 } else if (cipher_ctx.unit_no == tc->current_src){
1053 tc->curr_src = &cipher_ctx;
1054 } else if (csum_ctx.unit_no == tc->current_src){
1055 tc->curr_src = &csum_ctx;
1056 }
1057 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 failed = -EINVAL;
1060 goto error_cleanup;
1061 }
1062 } else {
1063 tc->curr_src = NULL;
1064 }
1065
1066 /* Detect source switch. */
1067 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 if (tc->active && (tc->current_src != tc->previous_src)) {
1069 /* Only allow source switch when both the old source unit and the new one have
1070 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 * transform blocklength). */
1072 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 {
1076 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 failed = -EINVAL;
1078 goto error_cleanup;
1079 }
1080 }
1081 /* Detect unit deactivation. */
1082 if (dcfg->last) {
1083 /* Length check of this is handled below. */
1084 tc->done = 1;
1085 }
1086 dcfg = dcfg->next;
1087 } /* while (dcfg) */
1088 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 failed = -EINVAL;
1093 goto error_cleanup;
1094 }
1095 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 failed = -EINVAL;
1098 goto error_cleanup;
1099 }
1100 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 failed = -EINVAL;
1103 goto error_cleanup;
1104 }
1105
1106 /* Update consumed and produced lengths.
1107
1108 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1109 other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1111 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 unit has processed an exact multiple of its block length the end result will be correct.
1113 Beware that if the source change restriction change this code will need to be (much) reworked.
1114 */
1115 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117 if (csum_ctx.active) {
1118 csum_ctx.consumed += desc_len;
1119 if (csum_ctx.done) {
1120 csum_ctx.produced = 2;
1121 }
1122 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 }
1124 if (digest_ctx.active) {
1125 digest_ctx.consumed += desc_len;
1126 if (digest_ctx.done) {
1127 if (digest_ctx.unit_no == src_md5) {
1128 digest_ctx.produced = MD5_STATE_LENGTH;
1129 } else {
1130 digest_ctx.produced = SHA1_STATE_LENGTH;
1131 }
1132 }
1133 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 }
1135 if (cipher_ctx.active) {
1136 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1137 assert(cipher_ctx.current_src == src_dma);
1138 cipher_ctx.consumed += desc_len;
1139 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 }
1143 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 }
1145
1146 /* Setup the DMA out descriptors. */
1147 /* Configure the metadata. */
1148 active_count = 0;
1149 eop_needed_count = 0;
1150 if (cipher_ctx.active) {
1151 ++active_count;
1152 if (cipher_ctx.unit_no == src_dma){
1153 /* mem2mem */
1154 meta_out.ciphsel = src_none;
1155 } else {
1156 meta_out.ciphsel = cipher_ctx.current_src;
1157 }
1158 meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 meta_out.cbcmode = cipher_ctx.cbcmode;
1160 meta_out.decrypt = cipher_ctx.decrypt;
1161 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 if (cipher_ctx.done) ++eop_needed_count;
1163 } else {
1164 meta_out.ciphsel = src_none;
1165 }
1166
1167 if (digest_ctx.active) {
1168 ++active_count;
1169 meta_out.hashsel = digest_ctx.current_src;
1170 meta_out.hashconf = digest_ctx.hash_conf;
1171 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 if (digest_ctx.done) {
1174 assert(digest_ctx.pad_descs == NULL);
1175 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 if (failed) {
1177 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 goto error_cleanup;
1179 }
1180 }
1181 } else {
1182 meta_out.hashsel = src_none;
1183 }
1184
1185 if (csum_ctx.active) {
1186 ++active_count;
1187 meta_out.csumsel = csum_ctx.current_src;
1188 if (csum_ctx.done) {
1189 assert(csum_ctx.pad_descs == NULL);
1190 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 if (failed) {
1192 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 goto error_cleanup;
1194 }
1195 }
1196 } else {
1197 meta_out.csumsel = src_none;
1198 }
1199 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 /* Setup DMA out descriptors for the indata. */
1201 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 if (failed) {
1203 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 goto error_cleanup;
1205 }
1206 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1207 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 */
1210 assert(active_count >= eop_needed_count);
1211 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 if (eop_needed_count) {
1213 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214 if (active_count > 1) {
1215 /* Use zero length EOP descriptor. */
1216 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 struct strcop_meta_out ed_mo = {0};
1218 if (!ed) {
1219 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 failed = -ENOMEM;
1221 goto error_cleanup;
1222 }
1223
1224 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226 if (cipher_ctx.unit_no == src_dma){
1227 /* mem2mem */
1228 ed_mo.ciphsel = src_none;
1229 } else {
1230 ed_mo.ciphsel = cipher_ctx.current_src;
1231 }
1232 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236 ed->free_buf = NULL;
1237 ed->dma_descr->wait = 1;
1238 ed->dma_descr->out_eop = 1;
1239
1240 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 ed->dma_descr->after = ed->dma_descr->buf;
1242 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 current_out_cdesc->next = ed;
1244 current_out_cdesc = ed;
1245 } else {
1246 /* Set EOP in the current out descriptor since the only active module is
1247 * the one needing the EOP. */
1248
1249 current_out_cdesc->dma_descr->out_eop = 1;
1250 }
1251 }
1252
1253 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 indata_ix += odsc->length;
1257 odsc = odsc->next;
1258 } /* while (odsc) */ /* Process descriptors. */
1259 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 failed = -EINVAL;
1263 goto error_cleanup;
1264 }
1265 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 failed = -EINVAL;
1268 goto error_cleanup;
1269 }
1270 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 failed = -EINVAL;
1273 goto error_cleanup;
1274 }
1275
1276 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 if (failed){
1278 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 goto error_cleanup;
1280 }
1281 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 if (failed){
1283 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 goto error_cleanup;
1285 }
1286 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 if (failed){
1288 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 goto error_cleanup;
1290 }
1291
1292 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 (*int_op)->cdesc_out = out_cdesc_head.next;
1294 (*int_op)->cdesc_in = in_cdesc_head.next;
1295 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297 setup_descr_chain(out_cdesc_head.next);
1298 setup_descr_chain(in_cdesc_head.next);
1299
1300 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 * last DMA out descriptor for EOL.
1302 */
1303 current_in_cdesc->dma_descr->intr = 1;
1304 current_in_cdesc->dma_descr->eol = 1;
1305 current_out_cdesc->dma_descr->eol = 1;
1306
1307 /* Setup DMA contexts. */
1308 (*int_op)->ctx_out.next = NULL;
1309 (*int_op)->ctx_out.eol = 1;
1310 (*int_op)->ctx_out.intr = 0;
1311 (*int_op)->ctx_out.store_mode = 0;
1312 (*int_op)->ctx_out.en = 0;
1313 (*int_op)->ctx_out.dis = 0;
1314 (*int_op)->ctx_out.md0 = 0;
1315 (*int_op)->ctx_out.md1 = 0;
1316 (*int_op)->ctx_out.md2 = 0;
1317 (*int_op)->ctx_out.md3 = 0;
1318 (*int_op)->ctx_out.md4 = 0;
1319 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322 (*int_op)->ctx_in.next = NULL;
1323 (*int_op)->ctx_in.eol = 1;
1324 (*int_op)->ctx_in.intr = 0;
1325 (*int_op)->ctx_in.store_mode = 0;
1326 (*int_op)->ctx_in.en = 0;
1327 (*int_op)->ctx_in.dis = 0;
1328 (*int_op)->ctx_in.md0 = 0;
1329 (*int_op)->ctx_in.md1 = 0;
1330 (*int_op)->ctx_in.md2 = 0;
1331 (*int_op)->ctx_in.md3 = 0;
1332 (*int_op)->ctx_in.md4 = 0;
1333
1334 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 return 0;
1339
1340 error_cleanup:
1341 {
1342 /* Free all allocated resources. */
1343 struct cryptocop_dma_desc *tmp_cdesc;
1344 while (digest_ctx.pad_descs){
1345 tmp_cdesc = digest_ctx.pad_descs->next;
1346 free_cdesc(digest_ctx.pad_descs);
1347 digest_ctx.pad_descs = tmp_cdesc;
1348 }
1349 while (csum_ctx.pad_descs){
1350 tmp_cdesc = csum_ctx.pad_descs->next;
1351 free_cdesc(csum_ctx.pad_descs);
1352 csum_ctx.pad_descs = tmp_cdesc;
1353 }
1354 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356 if (*int_op != NULL) delete_internal_operation(*int_op);
1357 }
1358 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 return failed;
1360 }
1361
1362
1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365 void *ptr = iop->alloc_ptr;
1366 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 struct cryptocop_dma_desc *next;
1368
1369 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371 while (cd) {
1372 next = cd->next;
1373 free_cdesc(cd);
1374 cd = next;
1375 }
1376 cd = iop->cdesc_in;
1377 while (cd) {
1378 next = cd->next;
1379 free_cdesc(cd);
1380 cd = next;
1381 }
1382 kfree(ptr);
1383 }
1384
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 unsigned char *p;
1392 int i;
1393 unsigned long long int bit_length = hashed_length << 3;
1394
1395 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397 p = kmalloc(padlen, alloc_flag);
1398 if (!p) return -ENOMEM;
1399
1400 *p = 0x80;
1401 memset(p+1, 0, padlen - 1);
1402
1403 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1404
1405 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1406 while (bit_length != 0){
1407 p[i++] = bit_length % 0x100;
1408 bit_length >>= 8;
1409 }
1410
1411 *pad = (char*)p;
1412 *pad_length = padlen;
1413
1414 return 0;
1415 }
1416
1417 #define SHA1_MIN_PAD_LENGTH (9)
1418 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1419
1420 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1421 {
1422 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1423 unsigned char *p;
1424 int i;
1425 unsigned long long int bit_length = hashed_length << 3;
1426
1427 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1428
1429 p = kmalloc(padlen, alloc_flag);
1430 if (!p) return -ENOMEM;
1431
1432 *p = 0x80;
1433 memset(p+1, 0, padlen - 1);
1434
1435 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1436
1437 i = padlen - 1;
1438 while (bit_length != 0){
1439 p[i--] = bit_length % 0x100;
1440 bit_length >>= 8;
1441 }
1442
1443 *pad = (char*)p;
1444 *pad_length = padlen;
1445
1446 return 0;
1447 }
1448
1449
1450 static int transform_ok(struct cryptocop_transform_init *tinit)
1451 {
1452 switch (tinit->alg){
1453 case cryptocop_alg_csum:
1454 switch (tinit->csum_mode){
1455 case cryptocop_csum_le:
1456 case cryptocop_csum_be:
1457 break;
1458 default:
1459 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1460 return -EINVAL;
1461 }
1462 case cryptocop_alg_mem2mem:
1463 case cryptocop_alg_md5:
1464 case cryptocop_alg_sha1:
1465 if (tinit->keylen != 0) {
1466 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1467 return -EINVAL; /* This check is a bit strict. */
1468 }
1469 break;
1470 case cryptocop_alg_des:
1471 if (tinit->keylen != 64) {
1472 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1473 return -EINVAL;
1474 }
1475 break;
1476 case cryptocop_alg_3des:
1477 if (tinit->keylen != 192) {
1478 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1479 return -EINVAL;
1480 }
1481 break;
1482 case cryptocop_alg_aes:
1483 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1484 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1485 return -EINVAL;
1486 }
1487 break;
1488 case cryptocop_no_alg:
1489 default:
1490 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1491 return -EINVAL;
1492 }
1493
1494 switch (tinit->alg){
1495 case cryptocop_alg_des:
1496 case cryptocop_alg_3des:
1497 case cryptocop_alg_aes:
1498 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1499 default:
1500 break;
1501 }
1502 return 0;
1503 }
1504
1505
1506 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1507 {
1508 struct cryptocop_session *sess;
1509 struct cryptocop_transform_init *tfrm_in = tinit;
1510 struct cryptocop_transform_init *tmp_in;
1511 int no_tfrms = 0;
1512 int i;
1513 unsigned long int flags;
1514
1515 init_stream_coprocessor(); /* For safety if we are called early */
1516
1517 while (tfrm_in){
1518 int err;
1519 ++no_tfrms;
1520 if ((err = transform_ok(tfrm_in))) {
1521 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1522 return err;
1523 }
1524 tfrm_in = tfrm_in->next;
1525 }
1526 if (0 == no_tfrms) {
1527 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1528 return -EINVAL;
1529 }
1530
1531 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1532 if (!sess){
1533 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1534 return -ENOMEM;
1535 }
1536
1537 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1538 if (!sess->tfrm_ctx) {
1539 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1540 kfree(sess);
1541 return -ENOMEM;
1542 }
1543
1544 tfrm_in = tinit;
1545 for (i = 0; i < no_tfrms; i++){
1546 tmp_in = tfrm_in->next;
1547 while (tmp_in){
1548 if (tmp_in->tid == tfrm_in->tid) {
1549 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1550 kfree(sess->tfrm_ctx);
1551 kfree(sess);
1552 return -EINVAL;
1553 }
1554 tmp_in = tmp_in->next;
1555 }
1556 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1557 sess->tfrm_ctx[i].dec_key_set = 0;
1558 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1559
1560 tfrm_in = tfrm_in->next;
1561 }
1562 sess->tfrm_ctx[i-1].next = NULL;
1563
1564 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1565 sess->sid = next_sid;
1566 next_sid++;
1567 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1568 * OTOH 2^64 is a really large number of session. */
1569 if (next_sid == 0) next_sid = 1;
1570
1571 /* Prepend to session list. */
1572 sess->next = cryptocop_sessions;
1573 cryptocop_sessions = sess;
1574 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1575 *sid = sess->sid;
1576 return 0;
1577 }
1578
1579
1580 int cryptocop_free_session(cryptocop_session_id sid)
1581 {
1582 struct cryptocop_transform_ctx *tc;
1583 struct cryptocop_session *sess = NULL;
1584 struct cryptocop_session *psess = NULL;
1585 unsigned long int flags;
1586 int i;
1587 LIST_HEAD(remove_list);
1588 struct list_head *node, *tmp;
1589 struct cryptocop_prio_job *pj;
1590
1591 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1592
1593 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1594 sess = cryptocop_sessions;
1595 while (sess && sess->sid != sid){
1596 psess = sess;
1597 sess = sess->next;
1598 }
1599 if (sess){
1600 if (psess){
1601 psess->next = sess->next;
1602 } else {
1603 cryptocop_sessions = sess->next;
1604 }
1605 }
1606 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1607
1608 if (!sess) return -EINVAL;
1609
1610 /* Remove queued jobs. */
1611 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1612
1613 for (i = 0; i < cryptocop_prio_no_prios; i++){
1614 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1615 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1616 pj = list_entry(node, struct cryptocop_prio_job, node);
1617 if (pj->oper->sid == sid) {
1618 list_move_tail(node, &remove_list);
1619 }
1620 }
1621 }
1622 }
1623 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1624
1625 list_for_each_safe(node, tmp, &remove_list) {
1626 list_del(node);
1627 pj = list_entry(node, struct cryptocop_prio_job, node);
1628 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1629 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1630 pj->oper->cb(pj->oper, pj->oper->cb_data);
1631 delete_internal_operation(pj->iop);
1632 kfree(pj);
1633 }
1634
1635 tc = sess->tfrm_ctx;
1636 /* Erase keying data. */
1637 while (tc){
1638 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1639 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1641 tc = tc->next;
1642 }
1643 kfree(sess->tfrm_ctx);
1644 kfree(sess);
1645
1646 return 0;
1647 }
1648
1649 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1650 {
1651 struct cryptocop_session *sess;
1652 unsigned long int flags;
1653
1654 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1655 sess = cryptocop_sessions;
1656 while (sess && (sess->sid != sid)){
1657 sess = sess->next;
1658 }
1659 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1660
1661 return sess;
1662 }
1663
1664 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1665 {
1666 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1667
1668 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1669 assert(sess != NULL);
1670 while (tc && tc->init.tid != tid){
1671 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1672 tc = tc->next;
1673 }
1674 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1675 return tc;
1676 }
1677
1678
1679
1680 /* The AES s-transform matrix (s-box). */
1681 static const u8 aes_sbox[256] = {
1682 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1683 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1684 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1685 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1686 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1687 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1688 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1689 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1690 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1691 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1692 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1693 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1694 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1695 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1696 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1697 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1698 };
1699
1700 /* AES has a 32 bit word round constants for each round in the
1701 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1702 */
1703 static u32 round_constant[11] = {
1704 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1705 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1706 0x1B000000, 0x36000000, 0x6C000000
1707 };
1708
1709 /* Apply the s-box to each of the four occtets in w. */
1710 static u32 aes_ks_subword(const u32 w)
1711 {
1712 u8 bytes[4];
1713
1714 *(u32*)(&bytes[0]) = w;
1715 bytes[0] = aes_sbox[bytes[0]];
1716 bytes[1] = aes_sbox[bytes[1]];
1717 bytes[2] = aes_sbox[bytes[2]];
1718 bytes[3] = aes_sbox[bytes[3]];
1719 return *(u32*)(&bytes[0]);
1720 }
1721
1722 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1723 * (Note that AES words are 32 bit long)
1724 *
1725 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1726 * word temp
1727 * i = 0
1728 * while (i < Nk) {
1729 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1730 * i = i + 1
1731 * }
1732 * i = Nk
1733 *
1734 * while (i < (Nb * (Nr + 1))) {
1735 * temp = w[i - 1]
1736 * if ((i mod Nk) == 0) {
1737 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1738 * }
1739 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1740 * temp = SubWord(temp)
1741 * }
1742 * w[i] = w[i - Nk] xor temp
1743 * }
1744 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1745 * SubWord(t) applies the AES s-box individually to each octet
1746 * in a 32 bit word.
1747 *
1748 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1749 * values for Nr of 10, 12, and 14). Nb is always 4.
1750 *
1751 * To construct w[i], w[i - 1] and w[i - Nk] must be
1752 * available. Consequently we must keep a state of the last Nk words
1753 * to be able to create the last round keys.
1754 */
1755 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1756 {
1757 u32 temp;
1758 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1759 u8 w_last_ix;
1760 int i;
1761 u8 nr, nk;
1762
1763 switch (keylength){
1764 case 128:
1765 nk = 4;
1766 nr = 10;
1767 break;
1768 case 192:
1769 nk = 6;
1770 nr = 12;
1771 break;
1772 case 256:
1773 nk = 8;
1774 nr = 14;
1775 break;
1776 default:
1777 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1778 };
1779
1780 /* Need to do host byte order correction here since key is byte oriented and the
1781 * kx algorithm is word (u32) oriented. */
1782 for (i = 0; i < nk; i+=1) {
1783 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1784 }
1785
1786 i = (int)nk;
1787 w_last_ix = i - 1;
1788 while (i < (4 * (nr + 2))) {
1789 temp = w_ring[w_last_ix];
1790 if (!(i % nk)) {
1791 /* RotWord(temp) */
1792 temp = (temp << 8) | (temp >> 24);
1793 temp = aes_ks_subword(temp);
1794 temp ^= round_constant[i/nk - 1];
1795 } else if ((nk > 6) && ((i % nk) == 4)) {
1796 temp = aes_ks_subword(temp);
1797 }
1798 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1799 temp ^= w_ring[w_last_ix];
1800 w_ring[w_last_ix] = temp;
1801
1802 /* We need the round keys for round Nr+1 and Nr+2 (round key
1803 * Nr+2 is the round key beyond the last one used when
1804 * encrypting). Rounds are numbered starting from 0, Nr=10
1805 * implies 11 rounds are used in encryption/decryption.
1806 */
1807 if (i >= (4 * nr)) {
1808 /* Need to do host byte order correction here, the key
1809 * is byte oriented. */
1810 *(u32*)dec_key = cpu_to_be32(temp);
1811 dec_key += 4;
1812 }
1813 ++i;
1814 }
1815 }
1816
1817
1818 /**** Job/operation management. ****/
1819
1820 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1821 {
1822 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1823 }
1824
1825 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1826 {
1827 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1828 }
1829
1830 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1831 {
1832 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1833 }
1834
1835 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1836 {
1837 int ret;
1838 struct cryptocop_prio_job *pj = NULL;
1839 unsigned long int flags;
1840
1841 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1842
1843 if (!operation || !operation->cb){
1844 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1845 return -EINVAL;
1846 }
1847
1848 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1849 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1850 return ret;
1851 }
1852 assert(pj != NULL);
1853
1854 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1855 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1856 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1857
1858 /* Make sure a job is running */
1859 cryptocop_start_job();
1860 return 0;
1861 }
1862
1863 static void cryptocop_do_tasklet(unsigned long unused);
1864 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1865
1866 static void cryptocop_do_tasklet(unsigned long unused)
1867 {
1868 struct list_head *node;
1869 struct cryptocop_prio_job *pj = NULL;
1870 unsigned long flags;
1871
1872 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1873
1874 do {
1875 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1876 if (!list_empty(&cryptocop_completed_jobs)){
1877 node = cryptocop_completed_jobs.next;
1878 list_del(node);
1879 pj = list_entry(node, struct cryptocop_prio_job, node);
1880 } else {
1881 pj = NULL;
1882 }
1883 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1884 if (pj) {
1885 assert(pj->oper != NULL);
1886
1887 /* Notify consumer of operation completeness. */
1888 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1889
1890 pj->oper->operation_status = 0; /* Job is completed. */
1891 pj->oper->cb(pj->oper, pj->oper->cb_data);
1892 delete_internal_operation(pj->iop);
1893 kfree(pj);
1894 }
1895 } while (pj != NULL);
1896
1897 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1898 }
1899
1900 static irqreturn_t
1901 dma_done_interrupt(int irq, void *dev_id)
1902 {
1903 struct cryptocop_prio_job *done_job;
1904 reg_dma_rw_ack_intr ack_intr = {
1905 .data = 1,
1906 };
1907
1908 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1909
1910 DEBUG(printk("cryptocop DMA done\n"));
1911
1912 spin_lock(&running_job_lock);
1913 if (cryptocop_running_job == NULL){
1914 printk("stream co-processor got interrupt when not busy\n");
1915 spin_unlock(&running_job_lock);
1916 return IRQ_HANDLED;
1917 }
1918 done_job = cryptocop_running_job;
1919 cryptocop_running_job = NULL;
1920 spin_unlock(&running_job_lock);
1921
1922 /* Start processing a job. */
1923 if (!spin_trylock(&cryptocop_process_lock)){
1924 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1925 } else {
1926 cryptocop_start_job();
1927 spin_unlock(&cryptocop_process_lock);
1928 }
1929
1930 done_job->oper->operation_status = 0; /* Job is completed. */
1931 if (done_job->oper->fast_callback){
1932 /* This operation wants callback from interrupt. */
1933 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1934 delete_internal_operation(done_job->iop);
1935 kfree(done_job);
1936 } else {
1937 spin_lock(&cryptocop_completed_jobs_lock);
1938 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1939 spin_unlock(&cryptocop_completed_jobs_lock);
1940 tasklet_schedule(&cryptocop_tasklet);
1941 }
1942
1943 DEBUG(printk("cryptocop leave irq handler\n"));
1944 return IRQ_HANDLED;
1945 }
1946
1947
1948 /* Setup interrupts and DMA channels. */
1949 static int init_cryptocop(void)
1950 {
1951 unsigned long flags;
1952 reg_dma_rw_cfg dma_cfg = {.en = 1};
1953 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1954 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1955 reg_strcop_rw_cfg strcop_cfg = {
1956 .ipend = regk_strcop_little,
1957 .td1 = regk_strcop_e,
1958 .td2 = regk_strcop_d,
1959 .td3 = regk_strcop_e,
1960 .ignore_sync = 0,
1961 .en = 1
1962 };
1963
1964 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1965 "stream co-processor DMA", NULL))
1966 panic("request_irq stream co-processor irq dma9");
1967
1968 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 0, dma_strp);
1970 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1971 0, dma_strp);
1972
1973 local_irq_save(flags);
1974
1975 /* Reset and enable the cryptocop. */
1976 strcop_cfg.en = 0;
1977 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978 strcop_cfg.en = 1;
1979 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1980
1981 /* Enable DMAs. */
1982 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1983 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1984
1985 /* Set up wordsize = 4 for DMAs. */
1986 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1987 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1988
1989 /* Enable interrupts. */
1990 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1991
1992 /* Clear intr ack. */
1993 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1994
1995 local_irq_restore(flags);
1996
1997 return 0;
1998 }
1999
2000 /* Free used cryptocop hw resources (interrupt and DMA channels). */
2001 static void release_cryptocop(void)
2002 {
2003 unsigned long flags;
2004 reg_dma_rw_cfg dma_cfg = {.en = 0};
2005 reg_dma_rw_intr_mask intr_mask_in = {0};
2006 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2007
2008 local_irq_save(flags);
2009
2010 /* Clear intr ack. */
2011 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2012
2013 /* Disable DMAs. */
2014 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2015 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2016
2017 /* Disable interrupts. */
2018 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2019
2020 local_irq_restore(flags);
2021
2022 free_irq(DMA_IRQ, NULL);
2023
2024 (void)crisv32_free_dma(OUT_DMA);
2025 (void)crisv32_free_dma(IN_DMA);
2026 }
2027
2028
2029 /* Init job queue. */
2030 static int cryptocop_job_queue_init(void)
2031 {
2032 int i;
2033
2034 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2035
2036 for (i = 0; i < cryptocop_prio_no_prios; i++){
2037 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2038 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2039 }
2040 return 0;
2041 }
2042
2043
2044 static void cryptocop_job_queue_close(void)
2045 {
2046 struct list_head *node, *tmp;
2047 struct cryptocop_prio_job *pj = NULL;
2048 unsigned long int process_flags, flags;
2049 int i;
2050
2051 /* FIXME: This is as yet untested code. */
2052
2053 /* Stop strcop from getting an operation to process while we are closing the
2054 module. */
2055 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2056
2057 /* Empty the job queue. */
2058 for (i = 0; i < cryptocop_prio_no_prios; i++){
2059 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2060 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2061 pj = list_entry(node, struct cryptocop_prio_job, node);
2062 list_del(node);
2063
2064 /* Call callback to notify consumer of job removal. */
2065 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2066 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2067 pj->oper->cb(pj->oper, pj->oper->cb_data);
2068
2069 delete_internal_operation(pj->iop);
2070 kfree(pj);
2071 }
2072 }
2073 }
2074 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2075
2076 /* Remove the running job, if any. */
2077 spin_lock_irqsave(&running_job_lock, flags);
2078 if (cryptocop_running_job){
2079 reg_strcop_rw_cfg rw_cfg;
2080 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2081
2082 /* Stop DMA. */
2083 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2084 dma_out_cfg.en = regk_dma_no;
2085 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2086
2087 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2088 dma_in_cfg.en = regk_dma_no;
2089 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2090
2091 /* Disble the cryptocop. */
2092 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2093 rw_cfg.en = 0;
2094 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2095
2096 pj = cryptocop_running_job;
2097 cryptocop_running_job = NULL;
2098
2099 /* Call callback to notify consumer of job removal. */
2100 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2101 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2102 pj->oper->cb(pj->oper, pj->oper->cb_data);
2103
2104 delete_internal_operation(pj->iop);
2105 kfree(pj);
2106 }
2107 spin_unlock_irqrestore(&running_job_lock, flags);
2108
2109 /* Remove completed jobs, if any. */
2110 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2111
2112 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2113 pj = list_entry(node, struct cryptocop_prio_job, node);
2114 list_del(node);
2115 /* Call callback to notify consumer of job removal. */
2116 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2117 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2118 pj->oper->cb(pj->oper, pj->oper->cb_data);
2119
2120 delete_internal_operation(pj->iop);
2121 kfree(pj);
2122 }
2123 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2124 }
2125
2126
2127 static void cryptocop_start_job(void)
2128 {
2129 int i;
2130 struct cryptocop_prio_job *pj;
2131 unsigned long int flags;
2132 unsigned long int running_job_flags;
2133 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2134
2135 DEBUG(printk("cryptocop_start_job: entering\n"));
2136
2137 spin_lock_irqsave(&running_job_lock, running_job_flags);
2138 if (cryptocop_running_job != NULL){
2139 /* Already running. */
2140 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2141 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2142 return;
2143 }
2144 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2145
2146 /* Check the queues in priority order. */
2147 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2148 if (i == cryptocop_prio_no_prios) {
2149 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2150 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2151 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2152 return; /* No jobs to run */
2153 }
2154 DEBUG(printk("starting job for prio %d\n", i));
2155
2156 /* TODO: Do not starve lower priority jobs. Let in a lower
2157 * prio job for every N-th processed higher prio job or some
2158 * other scheduling policy. This could reasonably be
2159 * tweakable since the optimal balance would depend on the
2160 * type of load on the system. */
2161
2162 /* Pull the DMA lists from the job and start the DMA client. */
2163 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2164 list_del(&pj->node);
2165 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2166 cryptocop_running_job = pj;
2167
2168 /* Set config register (3DES and CSUM modes). */
2169 switch (pj->iop->tdes_mode){
2170 case cryptocop_3des_eee:
2171 rw_cfg.td1 = regk_strcop_e;
2172 rw_cfg.td2 = regk_strcop_e;
2173 rw_cfg.td3 = regk_strcop_e;
2174 break;
2175 case cryptocop_3des_eed:
2176 rw_cfg.td1 = regk_strcop_e;
2177 rw_cfg.td2 = regk_strcop_e;
2178 rw_cfg.td3 = regk_strcop_d;
2179 break;
2180 case cryptocop_3des_ede:
2181 rw_cfg.td1 = regk_strcop_e;
2182 rw_cfg.td2 = regk_strcop_d;
2183 rw_cfg.td3 = regk_strcop_e;
2184 break;
2185 case cryptocop_3des_edd:
2186 rw_cfg.td1 = regk_strcop_e;
2187 rw_cfg.td2 = regk_strcop_d;
2188 rw_cfg.td3 = regk_strcop_d;
2189 break;
2190 case cryptocop_3des_dee:
2191 rw_cfg.td1 = regk_strcop_d;
2192 rw_cfg.td2 = regk_strcop_e;
2193 rw_cfg.td3 = regk_strcop_e;
2194 break;
2195 case cryptocop_3des_ded:
2196 rw_cfg.td1 = regk_strcop_d;
2197 rw_cfg.td2 = regk_strcop_e;
2198 rw_cfg.td3 = regk_strcop_d;
2199 break;
2200 case cryptocop_3des_dde:
2201 rw_cfg.td1 = regk_strcop_d;
2202 rw_cfg.td2 = regk_strcop_d;
2203 rw_cfg.td3 = regk_strcop_e;
2204 break;
2205 case cryptocop_3des_ddd:
2206 rw_cfg.td1 = regk_strcop_d;
2207 rw_cfg.td2 = regk_strcop_d;
2208 rw_cfg.td3 = regk_strcop_d;
2209 break;
2210 default:
2211 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2212 }
2213 switch (pj->iop->csum_mode){
2214 case cryptocop_csum_le:
2215 rw_cfg.ipend = regk_strcop_little;
2216 break;
2217 case cryptocop_csum_be:
2218 rw_cfg.ipend = regk_strcop_big;
2219 break;
2220 default:
2221 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2222 }
2223 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2224
2225 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2226 "ctx_in: 0x%p, phys: 0x%p\n"
2227 "ctx_out: 0x%p, phys: 0x%p\n",
2228 pj,
2229 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2230 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2231
2232 /* Start input DMA. */
2233 flush_dma_context(&pj->iop->ctx_in);
2234 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2235
2236 /* Start output DMA. */
2237 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2238
2239 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2240 DEBUG(printk("cryptocop_start_job: exiting\n"));
2241 }
2242
2243
2244 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2245 {
2246 int err;
2247 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2248 void *iop_alloc_ptr = NULL;
2249
2250 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2251 if (!*pj) return -ENOMEM;
2252
2253 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2254
2255 (*pj)->oper = operation;
2256 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2257
2258 if (operation->use_dmalists) {
2259 DEBUG(print_user_dma_lists(&operation->list_op));
2260 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2261 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2262 kfree(*pj);
2263 return -EINVAL;
2264 }
2265 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2266 if (!iop_alloc_ptr) {
2267 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2268 kfree(*pj);
2269 return -ENOMEM;
2270 }
2271 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2272 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2273 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2274 (*pj)->iop->sid = operation->sid;
2275 (*pj)->iop->cdesc_out = NULL;
2276 (*pj)->iop->cdesc_in = NULL;
2277 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2278 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2279 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2280 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2281
2282 /* Setup DMA contexts. */
2283 (*pj)->iop->ctx_out.next = NULL;
2284 (*pj)->iop->ctx_out.eol = 1;
2285 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2286 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2287
2288 (*pj)->iop->ctx_in.next = NULL;
2289 (*pj)->iop->ctx_in.eol = 1;
2290 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2291 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2292 } else {
2293 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2294 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2295 kfree(*pj);
2296 return err;
2297 }
2298 }
2299 DEBUG(print_dma_descriptors((*pj)->iop));
2300
2301 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2302
2303 return 0;
2304 }
2305
2306 static int cryptocop_open(struct inode *inode, struct file *filp)
2307 {
2308 int p = iminor(inode);
2309
2310 cycle_kernel_lock();
2311 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2312
2313 filp->private_data = NULL;
2314 return 0;
2315 }
2316
2317
2318 static int cryptocop_release(struct inode *inode, struct file *filp)
2319 {
2320 struct cryptocop_private *dev = filp->private_data;
2321 struct cryptocop_private *dev_next;
2322
2323 while (dev){
2324 dev_next = dev->next;
2325 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2326 (void)cryptocop_free_session(dev->sid);
2327 }
2328 kfree(dev);
2329 dev = dev_next;
2330 }
2331
2332 return 0;
2333 }
2334
2335
2336 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2337 unsigned int cmd, unsigned long arg)
2338 {
2339 struct cryptocop_private *dev = filp->private_data;
2340 struct cryptocop_private *prev_dev = NULL;
2341 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2342 struct strcop_session_op sop;
2343 int err;
2344
2345 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2346
2347 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2348 return -EFAULT;
2349 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2350 if (err) return -EFAULT;
2351
2352 while (dev && (dev->sid != sop.ses_id)) {
2353 prev_dev = dev;
2354 dev = dev->next;
2355 }
2356 if (dev){
2357 if (prev_dev){
2358 prev_dev->next = dev->next;
2359 } else {
2360 filp->private_data = dev->next;
2361 }
2362 err = cryptocop_free_session(dev->sid);
2363 if (err) return -EFAULT;
2364 } else {
2365 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2366 return -EINVAL;
2367 }
2368 return 0;
2369 }
2370
2371
2372 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2373 {
2374 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2375
2376 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2377
2378 jc->processed = 1;
2379 wake_up(&cryptocop_ioc_process_wq);
2380 }
2381
2382
2383 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2384 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2385 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2386
2387 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2388 {
2389 size_t ch_ix = 0;
2390
2391 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2392 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2393 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2394
2395 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2396 return ch_ix;
2397 }
2398
2399
2400 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2401 {
2402 size_t ch_ix = INT_MAX;
2403 size_t tmp_ix = 0;
2404
2405 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2406 if (crp_op->cipher_start > ix) {
2407 ch_ix = crp_op->cipher_start;
2408 } else {
2409 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2410 }
2411 }
2412 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2413 if (crp_op->digest_start > ix) {
2414 tmp_ix = crp_op->digest_start;
2415 } else {
2416 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2417 }
2418 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2419 }
2420 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2421 if (crp_op->csum_start > ix) {
2422 tmp_ix = crp_op->csum_start;
2423 } else {
2424 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2425 }
2426 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2427 }
2428 if (ch_ix == INT_MAX) ch_ix = ix;
2429 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2430 return ch_ix;
2431 }
2432
2433
2434 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2435 * Return -1 for ok, 0 for fail. */
2436 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2437 {
2438 int tmplen;
2439
2440 assert(iov != NULL);
2441 assert(iovix != NULL);
2442 assert(pages != NULL);
2443 assert(pageix != NULL);
2444 assert(pageoffset != NULL);
2445
2446 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447
2448 while (map_length > 0){
2449 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2450 if (*iovix >= iovlen){
2451 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2452 return 0;
2453 }
2454 if (*pageix >= nopages){
2455 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2456 return 0;
2457 }
2458 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2459 tmplen = PAGE_SIZE - *pageoffset;
2460 if (tmplen < map_length){
2461 (*pageoffset) = 0;
2462 (*pageix)++;
2463 } else {
2464 tmplen = map_length;
2465 (*pageoffset) += map_length;
2466 }
2467 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2468 iov[*iovix].iov_len = tmplen;
2469 map_length -= tmplen;
2470 (*iovix)++;
2471 }
2472 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2473 return -1;
2474 }
2475
2476
2477
2478 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2479 {
2480 int i;
2481 struct cryptocop_private *dev = filp->private_data;
2482 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2483 struct strcop_crypto_op oper = {0};
2484 int err = 0;
2485 struct cryptocop_operation *cop = NULL;
2486
2487 struct ioctl_job_cb_ctx *jc = NULL;
2488
2489 struct page **inpages = NULL;
2490 struct page **outpages = NULL;
2491 int noinpages = 0;
2492 int nooutpages = 0;
2493
2494 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2495 * can get connected/disconnected on different places in the indata. */
2496 struct cryptocop_desc_cfg dcfgs[5*3];
2497 int desc_ix = 0;
2498 int dcfg_ix = 0;
2499 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2500 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2501 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2502
2503 unsigned char *digest_result = NULL;
2504 int digest_length = 0;
2505 int cblocklen = 0;
2506 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2507 struct cryptocop_session *sess;
2508
2509 int iovlen = 0;
2510 int iovix = 0;
2511 int pageix = 0;
2512 int pageoffset = 0;
2513
2514 size_t prev_ix = 0;
2515 size_t next_ix;
2516
2517 int cipher_active, digest_active, csum_active;
2518 int end_digest, end_csum;
2519 int digest_done = 0;
2520 int cipher_done = 0;
2521 int csum_done = 0;
2522
2523 DEBUG(printk("cryptocop_ioctl_process\n"));
2524
2525 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2526 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2527 return -EFAULT;
2528 }
2529 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2530 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2531 return -EFAULT;
2532 }
2533 DEBUG(print_strcop_crypto_op(&oper));
2534
2535 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2536 if (!dev){
2537 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2538 return -EINVAL;
2539 }
2540
2541 /* Check buffers. */
2542 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2543 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2544 return -EINVAL;
2545 }
2546
2547 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2548 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2549 return -EFAULT;
2550 }
2551 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2552 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2553 return -EFAULT;
2554 }
2555
2556 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2557 if (!cop) {
2558 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2559 return -ENOMEM;
2560 }
2561 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2562 if (!jc) {
2563 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2564 err = -ENOMEM;
2565 goto error_cleanup;
2566 }
2567 jc->processed = 0;
2568
2569 cop->cb_data = jc;
2570 cop->cb = ioctl_process_job_callback;
2571 cop->operation_status = 0;
2572 cop->use_dmalists = 0;
2573 cop->in_interrupt = 0;
2574 cop->fast_callback = 0;
2575 cop->tfrm_op.tfrm_cfg = NULL;
2576 cop->tfrm_op.desc = NULL;
2577 cop->tfrm_op.indata = NULL;
2578 cop->tfrm_op.incount = 0;
2579 cop->tfrm_op.inlen = 0;
2580 cop->tfrm_op.outdata = NULL;
2581 cop->tfrm_op.outcount = 0;
2582 cop->tfrm_op.outlen = 0;
2583
2584 sess = get_session(oper.ses_id);
2585 if (!sess){
2586 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2587 kfree(cop);
2588 kfree(jc);
2589 return -EINVAL;
2590 }
2591
2592 if (oper.do_cipher) {
2593 unsigned int cipher_outlen = 0;
2594 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2595 if (!tc) {
2596 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2597 err = -EINVAL;
2598 goto error_cleanup;
2599 }
2600 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2601 ciph_tcfg.inject_ix = 0;
2602 ciph_tcfg.flags = 0;
2603 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2604 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2605 kfree(cop);
2606 kfree(jc);
2607 return -EINVAL;
2608 }
2609 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2610 if (oper.cipher_len % cblocklen) {
2611 kfree(cop);
2612 kfree(jc);
2613 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2614 return -EINVAL;
2615 }
2616 cipher_outlen = oper.cipher_len;
2617 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2618 if (oper.cipher_explicit) {
2619 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2620 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2621 } else {
2622 cipher_outlen = oper.cipher_len - cblocklen;
2623 }
2624 } else {
2625 if (oper.cipher_explicit){
2626 kfree(cop);
2627 kfree(jc);
2628 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2629 return -EINVAL;
2630 }
2631 }
2632 if (oper.cipher_outlen != cipher_outlen) {
2633 kfree(cop);
2634 kfree(jc);
2635 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2636 return -EINVAL;
2637 }
2638
2639 if (oper.decrypt){
2640 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2641 } else {
2642 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2643 }
2644 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2645 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2646 }
2647 if (oper.do_digest){
2648 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2649 if (!tc) {
2650 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2651 err = -EINVAL;
2652 goto error_cleanup;
2653 }
2654 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2655 digest_result = kmalloc(digest_length, GFP_KERNEL);
2656 if (!digest_result) {
2657 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2658 err = -EINVAL;
2659 goto error_cleanup;
2660 }
2661 DEBUG(memset(digest_result, 0xff, digest_length));
2662
2663 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2664 digest_tcfg.inject_ix = 0;
2665 ciph_tcfg.inject_ix += digest_length;
2666 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2667 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2668 err = -EINVAL;
2669 goto error_cleanup;
2670 }
2671
2672 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2673 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2674 }
2675 if (oper.do_csum){
2676 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2677 csum_tcfg.inject_ix = digest_length;
2678 ciph_tcfg.inject_ix += 2;
2679
2680 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2681 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2682 kfree(cop);
2683 kfree(jc);
2684 return -EINVAL;
2685 }
2686
2687 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2688 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2689 }
2690
2691 prev_ix = first_cfg_change_ix(&oper);
2692 if (prev_ix > oper.inlen) {
2693 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2694 nooutpages = noinpages = 0;
2695 err = -EINVAL;
2696 goto error_cleanup;
2697 }
2698 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2699
2700 /* Map user pages for in and out data of the operation. */
2701 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2702 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2703 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2704 if (!inpages){
2705 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2706 nooutpages = noinpages = 0;
2707 err = -ENOMEM;
2708 goto error_cleanup;
2709 }
2710 if (oper.do_cipher){
2711 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2712 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2713 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2714 if (!outpages){
2715 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2716 nooutpages = noinpages = 0;
2717 err = -ENOMEM;
2718 goto error_cleanup;
2719 }
2720 }
2721
2722 /* Acquire the mm page semaphore. */
2723 down_read(&current->mm->mmap_sem);
2724
2725 err = get_user_pages(current,
2726 current->mm,
2727 (unsigned long int)(oper.indata + prev_ix),
2728 noinpages,
2729 0, /* read access only for in data */
2730 0, /* no force */
2731 inpages,
2732 NULL);
2733
2734 if (err < 0) {
2735 up_read(&current->mm->mmap_sem);
2736 nooutpages = noinpages = 0;
2737 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2738 goto error_cleanup;
2739 }
2740 noinpages = err;
2741 if (oper.do_cipher){
2742 err = get_user_pages(current,
2743 current->mm,
2744 (unsigned long int)oper.cipher_outdata,
2745 nooutpages,
2746 1, /* write access for out data */
2747 0, /* no force */
2748 outpages,
2749 NULL);
2750 up_read(&current->mm->mmap_sem);
2751 if (err < 0) {
2752 nooutpages = 0;
2753 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2754 goto error_cleanup;
2755 }
2756 nooutpages = err;
2757 } else {
2758 up_read(&current->mm->mmap_sem);
2759 }
2760
2761 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2762 * csum output and splits when units are (dis-)connected. */
2763 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2764 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2765 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2766 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2767 err = -ENOMEM;
2768 goto error_cleanup;
2769 }
2770
2771 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2772 cop->tfrm_op.outlen = 0;
2773 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2774 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2775 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2776
2777 /* Setup the in iovecs. */
2778 cop->tfrm_op.incount = noinpages;
2779 if (noinpages > 1){
2780 size_t tmplen = cop->tfrm_op.inlen;
2781
2782 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2784 tmplen -= cop->tfrm_op.indata[0].iov_len;
2785 for (i = 1; i<noinpages; i++){
2786 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2787 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2788 tmplen -= PAGE_SIZE;
2789 }
2790 } else {
2791 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2792 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2793 }
2794
2795 iovlen = nooutpages + 6;
2796 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2797
2798 next_ix = next_cfg_change_ix(&oper, prev_ix);
2799 if (prev_ix == next_ix){
2800 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2801 err = -EINVAL; /* This should be impossible barring bugs. */
2802 goto error_cleanup;
2803 }
2804 while (prev_ix != next_ix){
2805 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2806 descs[desc_ix].cfg = NULL;
2807 descs[desc_ix].length = next_ix - prev_ix;
2808
2809 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2810 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2811 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2812 cipher_active = 1;
2813
2814 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2815 cipher_done = 1;
2816 dcfgs[dcfg_ix].last = 1;
2817 } else {
2818 dcfgs[dcfg_ix].last = 0;
2819 }
2820 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2821 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2822 ++dcfg_ix;
2823 }
2824 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2825 digest_active = 1;
2826 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2827 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2828 if (next_ix == (oper.digest_start + oper.digest_len)){
2829 assert(!digest_done);
2830 digest_done = 1;
2831 dcfgs[dcfg_ix].last = 1;
2832 } else {
2833 dcfgs[dcfg_ix].last = 0;
2834 }
2835 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2836 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2837 ++dcfg_ix;
2838 }
2839 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2840 csum_active = 1;
2841 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2842 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2843 if (next_ix == (oper.csum_start + oper.csum_len)){
2844 csum_done = 1;
2845 dcfgs[dcfg_ix].last = 1;
2846 } else {
2847 dcfgs[dcfg_ix].last = 0;
2848 }
2849 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2850 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2851 ++dcfg_ix;
2852 }
2853 if (!descs[desc_ix].cfg){
2854 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2855 err = -EINVAL;
2856 goto error_cleanup;
2857 }
2858 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2859 ++desc_ix;
2860 prev_ix = next_ix;
2861 next_ix = next_cfg_change_ix(&oper, prev_ix);
2862 }
2863 if (desc_ix > 0){
2864 descs[desc_ix-1].next = NULL;
2865 } else {
2866 descs[0].next = NULL;
2867 }
2868 if (oper.do_digest) {
2869 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2870 /* Add outdata iovec, length == <length of type of digest> */
2871 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2872 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2873 ++iovix;
2874 }
2875 if (oper.do_csum) {
2876 /* Add outdata iovec, length == 2, the length of csum. */
2877 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2878 /* Add outdata iovec, length == <length of type of digest> */
2879 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2880 cop->tfrm_op.outdata[iovix].iov_len = 2;
2881 ++iovix;
2882 }
2883 if (oper.do_cipher) {
2884 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2885 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2886 err = -ENOSYS; /* This should be impossible barring bugs. */
2887 goto error_cleanup;
2888 }
2889 }
2890 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2891 cop->tfrm_op.outcount = iovix;
2892 assert(iovix <= (nooutpages + 6));
2893
2894 cop->sid = oper.ses_id;
2895 cop->tfrm_op.desc = &descs[0];
2896
2897 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2898
2899 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2900 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2901 err = -EINVAL;
2902 goto error_cleanup;
2903 }
2904
2905 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2906
2907 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2908 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2909 if (!jc->processed){
2910 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2911 err = -EIO;
2912 goto error_cleanup;
2913 }
2914
2915 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2916 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2917 if (cop->operation_status == 0){
2918 if (oper.do_digest){
2919 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2920 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2921 if (0 != err){
2922 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2923 err = -EFAULT;
2924 goto error_cleanup;
2925 }
2926 }
2927 if (oper.do_csum){
2928 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2929 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2930 if (0 != err){
2931 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2932 err = -EFAULT;
2933 goto error_cleanup;
2934 }
2935 }
2936 err = 0;
2937 } else {
2938 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2939 err = cop->operation_status;
2940 }
2941
2942 error_cleanup:
2943 /* Release page caches. */
2944 for (i = 0; i < noinpages; i++){
2945 put_page(inpages[i]);
2946 }
2947 for (i = 0; i < nooutpages; i++){
2948 int spdl_err;
2949 /* Mark output pages dirty. */
2950 spdl_err = set_page_dirty_lock(outpages[i]);
2951 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2952 }
2953 for (i = 0; i < nooutpages; i++){
2954 put_page(outpages[i]);
2955 }
2956
2957 kfree(digest_result);
2958 kfree(inpages);
2959 kfree(outpages);
2960 if (cop){
2961 kfree(cop->tfrm_op.indata);
2962 kfree(cop->tfrm_op.outdata);
2963 kfree(cop);
2964 }
2965 kfree(jc);
2966
2967 DEBUG(print_lock_status());
2968
2969 return err;
2970 }
2971
2972
2973 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2974 {
2975 cryptocop_session_id sid;
2976 int err;
2977 struct cryptocop_private *dev;
2978 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2979 struct strcop_session_op sop;
2980 struct cryptocop_transform_init *tis = NULL;
2981 struct cryptocop_transform_init ti_cipher = {0};
2982 struct cryptocop_transform_init ti_digest = {0};
2983 struct cryptocop_transform_init ti_csum = {0};
2984
2985 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2986 return -EFAULT;
2987 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2988 if (err) return -EFAULT;
2989 if (sop.cipher != cryptocop_cipher_none) {
2990 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2991 }
2992 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2993
2994 DEBUG(printk("\tcipher:%d\n"
2995 "\tcipher_mode:%d\n"
2996 "\tdigest:%d\n"
2997 "\tcsum:%d\n",
2998 (int)sop.cipher,
2999 (int)sop.cmode,
3000 (int)sop.digest,
3001 (int)sop.csum));
3002
3003 if (sop.cipher != cryptocop_cipher_none){
3004 /* Init the cipher. */
3005 switch (sop.cipher){
3006 case cryptocop_cipher_des:
3007 ti_cipher.alg = cryptocop_alg_des;
3008 break;
3009 case cryptocop_cipher_3des:
3010 ti_cipher.alg = cryptocop_alg_3des;
3011 break;
3012 case cryptocop_cipher_aes:
3013 ti_cipher.alg = cryptocop_alg_aes;
3014 break;
3015 default:
3016 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3017 return -EINVAL;
3018 };
3019 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3020 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3021 ti_cipher.keylen = sop.keylen;
3022 switch (sop.cmode){
3023 case cryptocop_cipher_mode_cbc:
3024 case cryptocop_cipher_mode_ecb:
3025 ti_cipher.cipher_mode = sop.cmode;
3026 break;
3027 default:
3028 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3029 return -EINVAL;
3030 }
3031 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3032 switch (sop.des3_mode){
3033 case cryptocop_3des_eee:
3034 case cryptocop_3des_eed:
3035 case cryptocop_3des_ede:
3036 case cryptocop_3des_edd:
3037 case cryptocop_3des_dee:
3038 case cryptocop_3des_ded:
3039 case cryptocop_3des_dde:
3040 case cryptocop_3des_ddd:
3041 ti_cipher.tdes_mode = sop.des3_mode;
3042 break;
3043 default:
3044 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3045 return -EINVAL;
3046 }
3047 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3048 ti_cipher.next = tis;
3049 tis = &ti_cipher;
3050 } /* if (sop.cipher != cryptocop_cipher_none) */
3051 if (sop.digest != cryptocop_digest_none){
3052 DEBUG(printk("setting digest transform\n"));
3053 switch (sop.digest){
3054 case cryptocop_digest_md5:
3055 ti_digest.alg = cryptocop_alg_md5;
3056 break;
3057 case cryptocop_digest_sha1:
3058 ti_digest.alg = cryptocop_alg_sha1;
3059 break;
3060 default:
3061 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3062 return -EINVAL;
3063 }
3064 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3065 ti_digest.next = tis;
3066 tis = &ti_digest;
3067 } /* if (sop.digest != cryptocop_digest_none) */
3068 if (sop.csum != cryptocop_csum_none){
3069 DEBUG(printk("setting csum transform\n"));
3070 switch (sop.csum){
3071 case cryptocop_csum_le:
3072 case cryptocop_csum_be:
3073 ti_csum.csum_mode = sop.csum;
3074 break;
3075 default:
3076 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3077 return -EINVAL;
3078 }
3079 ti_csum.alg = cryptocop_alg_csum;
3080 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3081 ti_csum.next = tis;
3082 tis = &ti_csum;
3083 } /* (sop.csum != cryptocop_csum_none) */
3084 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3085 if (!dev){
3086 DEBUG_API(printk("create session, alloc dev\n"));
3087 return -ENOMEM;
3088 }
3089
3090 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3091 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3092
3093 if (err) {
3094 kfree(dev);
3095 return err;
3096 }
3097 sess_op->ses_id = sid;
3098 dev->sid = sid;
3099 dev->next = filp->private_data;
3100 filp->private_data = dev;
3101
3102 return 0;
3103 }
3104
3105 static int cryptocop_ioctl(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
3106 {
3107 int err = 0;
3108 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3109 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3110 return -ENOTTY;
3111 }
3112 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3113 return -ENOTTY;
3114 }
3115 /* Access check of the argument. Some commands, e.g. create session and process op,
3116 needs additional checks. Those are handled in the command handling functions. */
3117 if (_IOC_DIR(cmd) & _IOC_READ)
3118 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3119 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3120 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3121 if (err) return -EFAULT;
3122
3123 switch (cmd) {
3124 case CRYPTOCOP_IO_CREATE_SESSION:
3125 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3126 case CRYPTOCOP_IO_CLOSE_SESSION:
3127 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3128 case CRYPTOCOP_IO_PROCESS_OP:
3129 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3130 default:
3131 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3132 return -ENOTTY;
3133 }
3134 return 0;
3135 }
3136
3137
3138 #ifdef LDEBUG
3139 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3140 {
3141 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3142 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3143 int i;
3144
3145 printk("print_dma_descriptors start\n");
3146
3147 printk("iop:\n");
3148 printk("\tsid: 0x%lld\n", iop->sid);
3149
3150 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3151 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3152 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3153 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3154
3155 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3156 printk("\tnext: 0x%p\n"
3157 "\tsaved_data: 0x%p\n"
3158 "\tsaved_data_buf: 0x%p\n",
3159 iop->ctx_out.next,
3160 iop->ctx_out.saved_data,
3161 iop->ctx_out.saved_data_buf);
3162
3163 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3164 printk("\tnext: 0x%p\n"
3165 "\tsaved_data: 0x%p\n"
3166 "\tsaved_data_buf: 0x%p\n",
3167 iop->ctx_in.next,
3168 iop->ctx_in.saved_data,
3169 iop->ctx_in.saved_data_buf);
3170
3171 i = 0;
3172 while (cdesc_out) {
3173 dma_descr_data *td;
3174 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3175 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3176 td = cdesc_out->dma_descr;
3177 printk("\n\tbuf: 0x%p\n"
3178 "\tafter: 0x%p\n"
3179 "\tmd: 0x%04x\n"
3180 "\tnext: 0x%p\n",
3181 td->buf,
3182 td->after,
3183 td->md,
3184 td->next);
3185 printk("flags:\n"
3186 "\twait:\t%d\n"
3187 "\teol:\t%d\n"
3188 "\touteop:\t%d\n"
3189 "\tineop:\t%d\n"
3190 "\tintr:\t%d\n",
3191 td->wait,
3192 td->eol,
3193 td->out_eop,
3194 td->in_eop,
3195 td->intr);
3196 cdesc_out = cdesc_out->next;
3197 i++;
3198 }
3199 i = 0;
3200 while (cdesc_in) {
3201 dma_descr_data *td;
3202 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3203 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3204 td = cdesc_in->dma_descr;
3205 printk("\n\tbuf: 0x%p\n"
3206 "\tafter: 0x%p\n"
3207 "\tmd: 0x%04x\n"
3208 "\tnext: 0x%p\n",
3209 td->buf,
3210 td->after,
3211 td->md,
3212 td->next);
3213 printk("flags:\n"
3214 "\twait:\t%d\n"
3215 "\teol:\t%d\n"
3216 "\touteop:\t%d\n"
3217 "\tineop:\t%d\n"
3218 "\tintr:\t%d\n",
3219 td->wait,
3220 td->eol,
3221 td->out_eop,
3222 td->in_eop,
3223 td->intr);
3224 cdesc_in = cdesc_in->next;
3225 i++;
3226 }
3227
3228 printk("print_dma_descriptors end\n");
3229 }
3230
3231
3232 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3233 {
3234 printk("print_strcop_crypto_op, 0x%p\n", cop);
3235
3236 /* Indata. */
3237 printk("indata=0x%p\n"
3238 "inlen=%d\n"
3239 "do_cipher=%d\n"
3240 "decrypt=%d\n"
3241 "cipher_explicit=%d\n"
3242 "cipher_start=%d\n"
3243 "cipher_len=%d\n"
3244 "outdata=0x%p\n"
3245 "outlen=%d\n",
3246 cop->indata,
3247 cop->inlen,
3248 cop->do_cipher,
3249 cop->decrypt,
3250 cop->cipher_explicit,
3251 cop->cipher_start,
3252 cop->cipher_len,
3253 cop->cipher_outdata,
3254 cop->cipher_outlen);
3255
3256 printk("do_digest=%d\n"
3257 "digest_start=%d\n"
3258 "digest_len=%d\n",
3259 cop->do_digest,
3260 cop->digest_start,
3261 cop->digest_len);
3262
3263 printk("do_csum=%d\n"
3264 "csum_start=%d\n"
3265 "csum_len=%d\n",
3266 cop->do_csum,
3267 cop->csum_start,
3268 cop->csum_len);
3269 }
3270
3271 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3272 {
3273 struct cryptocop_desc *d;
3274 struct cryptocop_tfrm_cfg *tc;
3275 struct cryptocop_desc_cfg *dc;
3276 int i;
3277
3278 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3279 printk("sid: %lld\n", cop->sid);
3280 printk("operation_status=%d\n"
3281 "use_dmalists=%d\n"
3282 "in_interrupt=%d\n"
3283 "fast_callback=%d\n",
3284 cop->operation_status,
3285 cop->use_dmalists,
3286 cop->in_interrupt,
3287 cop->fast_callback);
3288
3289 if (cop->use_dmalists){
3290 print_user_dma_lists(&cop->list_op);
3291 } else {
3292 printk("cop->tfrm_op\n"
3293 "tfrm_cfg=0x%p\n"
3294 "desc=0x%p\n"
3295 "indata=0x%p\n"
3296 "incount=%d\n"
3297 "inlen=%d\n"
3298 "outdata=0x%p\n"
3299 "outcount=%d\n"
3300 "outlen=%d\n\n",
3301 cop->tfrm_op.tfrm_cfg,
3302 cop->tfrm_op.desc,
3303 cop->tfrm_op.indata,
3304 cop->tfrm_op.incount,
3305 cop->tfrm_op.inlen,
3306 cop->tfrm_op.outdata,
3307 cop->tfrm_op.outcount,
3308 cop->tfrm_op.outlen);
3309
3310 tc = cop->tfrm_op.tfrm_cfg;
3311 while (tc){
3312 printk("tfrm_cfg, 0x%p\n"
3313 "tid=%d\n"
3314 "flags=%d\n"
3315 "inject_ix=%d\n"
3316 "next=0x%p\n",
3317 tc,
3318 tc->tid,
3319 tc->flags,
3320 tc->inject_ix,
3321 tc->next);
3322 tc = tc->next;
3323 }
3324 d = cop->tfrm_op.desc;
3325 while (d){
3326 printk("\n======================desc, 0x%p\n"
3327 "length=%d\n"
3328 "cfg=0x%p\n"
3329 "next=0x%p\n",
3330 d,
3331 d->length,
3332 d->cfg,
3333 d->next);
3334 dc = d->cfg;
3335 while (dc){
3336 printk("=========desc_cfg, 0x%p\n"
3337 "tid=%d\n"
3338 "src=%d\n"
3339 "last=%d\n"
3340 "next=0x%p\n",
3341 dc,
3342 dc->tid,
3343 dc->src,
3344 dc->last,
3345 dc->next);
3346 dc = dc->next;
3347 }
3348 d = d->next;
3349 }
3350 printk("\n====iniov\n");
3351 for (i = 0; i < cop->tfrm_op.incount; i++){
3352 printk("indata[%d]\n"
3353 "base=0x%p\n"
3354 "len=%d\n",
3355 i,
3356 cop->tfrm_op.indata[i].iov_base,
3357 cop->tfrm_op.indata[i].iov_len);
3358 }
3359 printk("\n====outiov\n");
3360 for (i = 0; i < cop->tfrm_op.outcount; i++){
3361 printk("outdata[%d]\n"
3362 "base=0x%p\n"
3363 "len=%d\n",
3364 i,
3365 cop->tfrm_op.outdata[i].iov_base,
3366 cop->tfrm_op.outdata[i].iov_len);
3367 }
3368 }
3369 printk("------------end print_cryptocop_operation\n");
3370 }
3371
3372
3373 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3374 {
3375 dma_descr_data *dd;
3376 int i;
3377
3378 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3379
3380 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3381 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3382
3383 printk("##############outlist\n");
3384 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3385 i = 0;
3386 while (dd != NULL) {
3387 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3388 printk("\n\tbuf: 0x%p\n"
3389 "\tafter: 0x%p\n"
3390 "\tmd: 0x%04x\n"
3391 "\tnext: 0x%p\n",
3392 dd->buf,
3393 dd->after,
3394 dd->md,
3395 dd->next);
3396 printk("flags:\n"
3397 "\twait:\t%d\n"
3398 "\teol:\t%d\n"
3399 "\touteop:\t%d\n"
3400 "\tineop:\t%d\n"
3401 "\tintr:\t%d\n",
3402 dd->wait,
3403 dd->eol,
3404 dd->out_eop,
3405 dd->in_eop,
3406 dd->intr);
3407 if (dd->eol)
3408 dd = NULL;
3409 else
3410 dd = phys_to_virt((unsigned long int)dd->next);
3411 ++i;
3412 }
3413
3414 printk("##############inlist\n");
3415 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3416 i = 0;
3417 while (dd != NULL) {
3418 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3419 printk("\n\tbuf: 0x%p\n"
3420 "\tafter: 0x%p\n"
3421 "\tmd: 0x%04x\n"
3422 "\tnext: 0x%p\n",
3423 dd->buf,
3424 dd->after,
3425 dd->md,
3426 dd->next);
3427 printk("flags:\n"
3428 "\twait:\t%d\n"
3429 "\teol:\t%d\n"
3430 "\touteop:\t%d\n"
3431 "\tineop:\t%d\n"
3432 "\tintr:\t%d\n",
3433 dd->wait,
3434 dd->eol,
3435 dd->out_eop,
3436 dd->in_eop,
3437 dd->intr);
3438 if (dd->eol)
3439 dd = NULL;
3440 else
3441 dd = phys_to_virt((unsigned long int)dd->next);
3442 ++i;
3443 }
3444 }
3445
3446
3447 static void print_lock_status(void)
3448 {
3449 printk("**********************print_lock_status\n");
3450 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3451 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3452 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3453 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3454 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3455 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3456 }
3457 #endif /* LDEBUG */
3458
3459
3460 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3461
3462 static int init_stream_coprocessor(void)
3463 {
3464 int err;
3465 int i;
3466 static int initialized = 0;
3467
3468 if (initialized)
3469 return 0;
3470
3471 initialized = 1;
3472
3473 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3474
3475 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3476 if (err < 0) {
3477 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3478 return err;
3479 }
3480
3481 err = init_cryptocop();
3482 if (err) {
3483 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3484 return err;
3485 }
3486 err = cryptocop_job_queue_init();
3487 if (err) {
3488 release_cryptocop();
3489 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3490 return err;
3491 }
3492 /* Init the descriptor pool. */
3493 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3494 descr_pool[i].from_pool = 1;
3495 descr_pool[i].next = &descr_pool[i + 1];
3496 }
3497 descr_pool[i].from_pool = 1;
3498 descr_pool[i].next = NULL;
3499 descr_pool_free_list = &descr_pool[0];
3500 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3501
3502 spin_lock_init(&cryptocop_completed_jobs_lock);
3503 spin_lock_init(&cryptocop_job_queue_lock);
3504 spin_lock_init(&descr_pool_lock);
3505 spin_lock_init(&cryptocop_sessions_lock);
3506 spin_lock_init(&running_job_lock);
3507 spin_lock_init(&cryptocop_process_lock);
3508
3509 cryptocop_sessions = NULL;
3510 next_sid = 1;
3511
3512 cryptocop_running_job = NULL;
3513
3514 printk("stream co-processor: init done.\n");
3515 return 0;
3516 }
3517
3518 static void __exit exit_stream_coprocessor(void)
3519 {
3520 release_cryptocop();
3521 cryptocop_job_queue_close();
3522 }
3523
3524 module_init(init_stream_coprocessor);
3525 module_exit(exit_stream_coprocessor);
3526
connection oriented routing - ARCHIVE NO UPDATES HERE