?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Staging: Panel: panel: Fixed checkpatch line length warnings
[linux/fpc-iii.git] / arch / cris / arch-v32 / drivers / cryptocop.c
blob877da1908234b7b027fa92c8476903c9b4832dae
1 /*
2 * Stream co-processor driver for the ETRAX FS
3 *
4 * Copyright (C) 2003-2007 Axis Communications AB
5 */
6
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <linux/atomic.h>
20
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50
51 #define DESCR_ALLOC_PAD (31)
52
53 struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61 };
62
63
64 struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82 };
83
84
85 struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120 };
121
122
123 struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126 };
127
128 /* Session list. */
129
130 struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136 };
137
138
139 struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145 };
146
147 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150 typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156
157 struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160 };
161
162 struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168 };
169
170 struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172 };
173
174
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197 of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206 * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211
212
213 /** Local functions. **/
214
215 static int cryptocop_open(struct inode *, struct file *);
216
217 static int cryptocop_release(struct inode *, struct file *);
218
219 static long cryptocop_ioctl(struct file *file,
220 unsigned int cmd, unsigned long arg);
221
222 static void cryptocop_start_job(void);
223
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
243
244 static int init_stream_coprocessor(void);
245
246 static void __exit exit_stream_coprocessor(void);
247
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263
264
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH (8)
267 #define AES_BLOCK_LENGTH (16)
268 #define MD5_BLOCK_LENGTH (64)
269 #define SHA1_BLOCK_LENGTH (64)
270 #define CSUM_BLOCK_LENGTH (2)
271 #define MD5_STATE_LENGTH (16)
272 #define SHA1_STATE_LENGTH (20)
273
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR (254)
276 #define CRYPTOCOP_MINOR (0)
277
278
279
280 const struct file_operations cryptocop_fops = {
281 .owner = THIS_MODULE,
282 .open = cryptocop_open,
283 .release = cryptocop_release,
284 .unlocked_ioctl = cryptocop_ioctl,
285 .llseek = noop_llseek,
286 };
287
288
289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290 {
291 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
292 kfree(cdesc->free_buf);
293
294 if (cdesc->from_pool) {
295 unsigned long int flags;
296 spin_lock_irqsave(&descr_pool_lock, flags);
297 cdesc->next = descr_pool_free_list;
298 descr_pool_free_list = cdesc;
299 ++descr_pool_no_free;
300 spin_unlock_irqrestore(&descr_pool_lock, flags);
301 } else {
302 kfree(cdesc);
303 }
304 }
305
306
307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308 {
309 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 struct cryptocop_dma_desc *cdesc;
311
312 if (use_pool) {
313 unsigned long int flags;
314 spin_lock_irqsave(&descr_pool_lock, flags);
315 if (!descr_pool_free_list) {
316 spin_unlock_irqrestore(&descr_pool_lock, flags);
317 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 return NULL;
319 }
320 cdesc = descr_pool_free_list;
321 descr_pool_free_list = descr_pool_free_list->next;
322 --descr_pool_no_free;
323 spin_unlock_irqrestore(&descr_pool_lock, flags);
324 cdesc->from_pool = 1;
325 } else {
326 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 if (!cdesc) {
328 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 return NULL;
330 }
331 cdesc->from_pool = 0;
332 }
333 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334
335 cdesc->next = NULL;
336
337 cdesc->free_buf = NULL;
338 cdesc->dma_descr->out_eop = 0;
339 cdesc->dma_descr->in_eop = 0;
340 cdesc->dma_descr->intr = 0;
341 cdesc->dma_descr->eol = 0;
342 cdesc->dma_descr->wait = 0;
343 cdesc->dma_descr->buf = NULL;
344 cdesc->dma_descr->after = NULL;
345
346 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 return cdesc;
348 }
349
350
351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352 {
353 DEBUG(printk("setup_descr_chain: entering\n"));
354 while (cd) {
355 if (cd->next) {
356 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 } else {
358 cd->dma_descr->next = NULL;
359 }
360 cd = cd->next;
361 }
362 DEBUG(printk("setup_descr_chain: exit\n"));
363 }
364
365
366 /* Create a pad descriptor for the transform.
367 * Return -1 for error, 0 if pad created. */
368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369 {
370 struct cryptocop_dma_desc *cdesc = NULL;
371 int error = 0;
372 struct strcop_meta_out mo = {
373 .ciphsel = src_none,
374 .hashsel = src_none,
375 .csumsel = src_none
376 };
377 char *pad;
378 size_t plen;
379
380 DEBUG(printk("create_pad_descriptor: start.\n"));
381 /* Setup pad descriptor. */
382
383 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 cdesc = alloc_cdesc(alloc_flag);
385 if (!cdesc){
386 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 goto error_cleanup;
388 }
389 switch (tc->unit_no) {
390 case src_md5:
391 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 if (error){
393 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 goto error_cleanup;
395 }
396 cdesc->free_buf = pad;
397 mo.hashsel = src_dma;
398 mo.hashconf = tc->hash_conf;
399 mo.hashmode = tc->hash_mode;
400 break;
401 case src_sha1:
402 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 if (error){
404 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 goto error_cleanup;
406 }
407 cdesc->free_buf = pad;
408 mo.hashsel = src_dma;
409 mo.hashconf = tc->hash_conf;
410 mo.hashmode = tc->hash_mode;
411 break;
412 case src_csum:
413 if (tc->consumed % tc->blocklength){
414 pad = (char*)csum_zero_pad;
415 plen = 1;
416 } else {
417 pad = (char*)cdesc; /* Use any pointer. */
418 plen = 0;
419 }
420 mo.csumsel = src_dma;
421 break;
422 }
423 cdesc->dma_descr->wait = 1;
424 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
425 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427
428 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 *pad_desc = cdesc;
430
431 return 0;
432
433 error_cleanup:
434 if (cdesc) free_cdesc(cdesc);
435 return -1;
436 }
437
438
439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440 {
441 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
442 struct strcop_meta_out mo = {0};
443
444 DEBUG(printk("setup_key_dl_desc\n"));
445
446 if (!key_desc) {
447 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 return -ENOMEM;
449 }
450
451 /* Download key. */
452 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 /* Precook the AES decrypt key. */
454 if (!tc->tctx->dec_key_set){
455 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 tc->tctx->dec_key_set = 1;
457 }
458 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 } else {
461 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 }
464 /* Setup metadata. */
465 mo.dlkey = 1;
466 switch (tc->tctx->init.keylen) {
467 case 64:
468 mo.decrypt = 0;
469 mo.hashmode = 0;
470 break;
471 case 128:
472 mo.decrypt = 0;
473 mo.hashmode = 1;
474 break;
475 case 192:
476 mo.decrypt = 1;
477 mo.hashmode = 0;
478 break;
479 case 256:
480 mo.decrypt = 1;
481 mo.hashmode = 1;
482 break;
483 default:
484 break;
485 }
486 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488
489 key_desc->dma_descr->out_eop = 1;
490 key_desc->dma_descr->wait = 1;
491 key_desc->dma_descr->intr = 0;
492
493 *kd = key_desc;
494 return 0;
495 }
496
497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498 {
499 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
500 struct strcop_meta_out mo = {0};
501
502 DEBUG(printk("setup_cipher_iv_desc\n"));
503
504 if (!iv_desc) {
505 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 return -ENOMEM;
507 }
508 /* Download IV. */
509 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511
512 /* Setup metadata. */
513 mo.hashsel = mo.csumsel = src_none;
514 mo.ciphsel = src_dma;
515 mo.ciphconf = tc->ciph_conf;
516 mo.cbcmode = tc->cbcmode;
517
518 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519
520 iv_desc->dma_descr->out_eop = 0;
521 iv_desc->dma_descr->wait = 1;
522 iv_desc->dma_descr->intr = 0;
523
524 *id = iv_desc;
525 return 0;
526 }
527
528 /* Map the ouput length of the transform to operation output starting on the inject index. */
529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530 {
531 int err = 0;
532 struct cryptocop_dma_desc head = {0};
533 struct cryptocop_dma_desc *outdesc = &head;
534 size_t iov_offset = 0;
535 size_t out_ix = 0;
536 int outiov_ix = 0;
537 struct strcop_meta_in mi = {0};
538
539 size_t out_length = tc->produced;
540 int rem_length;
541 int dlength;
542
543 assert(out_length != 0);
544 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 return -EINVAL;
547 }
548 /* Traverse the out iovec until the result inject index is reached. */
549 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 outiov_ix++;
552 }
553 if (outiov_ix >= operation->tfrm_op.outcount){
554 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 return -EINVAL;
556 }
557 iov_offset = tc->tcfg->inject_ix - out_ix;
558 mi.dmasel = tc->unit_no;
559
560 /* Setup the output descriptors. */
561 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 outdesc->next = alloc_cdesc(alloc_flag);
563 if (!outdesc->next) {
564 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 err = -ENOMEM;
566 goto error_cleanup;
567 }
568 outdesc = outdesc->next;
569 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 dlength = (out_length < rem_length) ? out_length : rem_length;
571
572 DEBUG(printk("create_input_descriptors:\n"
573 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 "outcount=%d, outiov_ix=%d\n",
576 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577
578 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581
582 out_length -= dlength;
583 iov_offset += dlength;
584 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 iov_offset = 0;
586 ++outiov_ix;
587 }
588 }
589 if (out_length > 0){
590 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 err = -EINVAL;
592 goto error_cleanup;
593 }
594 /* Set sync in last descriptor. */
595 mi.sync = 1;
596 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597
598 *id = head.next;
599 return 0;
600
601 error_cleanup:
602 while (head.next) {
603 outdesc = head.next->next;
604 free_cdesc(head.next);
605 head.next = outdesc;
606 }
607 return err;
608 }
609
610
611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612 {
613 while (desc_len != 0) {
614 struct cryptocop_dma_desc *cdesc;
615 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
617
618 cdesc = alloc_cdesc(alloc_flag);
619 if (!cdesc) {
620 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 return -ENOMEM;
622 }
623 (*current_out_cdesc)->next = cdesc;
624 (*current_out_cdesc) = cdesc;
625
626 cdesc->free_buf = NULL;
627
628 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630
631 assert(desc_len >= dlength);
632 desc_len -= dlength;
633 *iniov_offset += dlength;
634 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 *iniov_offset = 0;
636 ++(*iniov_ix);
637 if (*iniov_ix > operation->tfrm_op.incount) {
638 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 return -EINVAL;
640 }
641 }
642 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 } /* while (desc_len != 0) */
644 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646
647 return 0;
648 }
649
650
651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652 {
653 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 if (tc->tcfg) {
655 int failed = 0;
656 struct cryptocop_dma_desc *idescs = NULL;
657 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 if (tc->pad_descs) {
659 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 while (tc->pad_descs) {
661 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 (*current_out_cdesc)->next = tc->pad_descs;
663 tc->pad_descs = tc->pad_descs->next;
664 (*current_out_cdesc) = (*current_out_cdesc)->next;
665 }
666 }
667
668 /* Setup and append output descriptors to DMA in list. */
669 if (tc->unit_no == src_dma){
670 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 unsigned int start_ix = tc->start_ix;
673 while (start_ix){
674 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 if (!(*current_in_cdesc)->next){
677 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 return -ENOMEM;
679 }
680 (*current_in_cdesc) = (*current_in_cdesc)->next;
681 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 start_ix -= desclen;
685 }
686 mi.sync = 1;
687 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 }
689
690 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 if (failed){
692 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 return failed;
694 }
695 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 while (idescs) {
697 DEBUG(printk("append descriptor 0x%p\n", idescs));
698 (*current_in_cdesc)->next = idescs;
699 idescs = idescs->next;
700 (*current_in_cdesc) = (*current_in_cdesc)->next;
701 }
702 }
703 return 0;
704 }
705
706
707
708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709 {
710 struct cryptocop_session *sess;
711 struct cryptocop_transform_ctx *tctx;
712
713 struct cryptocop_tfrm_ctx digest_ctx = {
714 .previous_src = src_none,
715 .current_src = src_none,
716 .start_ix = 0,
717 .requires_padding = 1,
718 .strict_block_length = 0,
719 .hash_conf = 0,
720 .hash_mode = 0,
721 .ciph_conf = 0,
722 .cbcmode = 0,
723 .decrypt = 0,
724 .consumed = 0,
725 .produced = 0,
726 .pad_descs = NULL,
727 .active = 0,
728 .done = 0,
729 .prev_src = NULL,
730 .curr_src = NULL,
731 .tcfg = NULL};
732 struct cryptocop_tfrm_ctx cipher_ctx = {
733 .previous_src = src_none,
734 .current_src = src_none,
735 .start_ix = 0,
736 .requires_padding = 0,
737 .strict_block_length = 1,
738 .hash_conf = 0,
739 .hash_mode = 0,
740 .ciph_conf = 0,
741 .cbcmode = 0,
742 .decrypt = 0,
743 .consumed = 0,
744 .produced = 0,
745 .pad_descs = NULL,
746 .active = 0,
747 .done = 0,
748 .prev_src = NULL,
749 .curr_src = NULL,
750 .tcfg = NULL};
751 struct cryptocop_tfrm_ctx csum_ctx = {
752 .previous_src = src_none,
753 .current_src = src_none,
754 .start_ix = 0,
755 .blocklength = 2,
756 .requires_padding = 1,
757 .strict_block_length = 0,
758 .hash_conf = 0,
759 .hash_mode = 0,
760 .ciph_conf = 0,
761 .cbcmode = 0,
762 .decrypt = 0,
763 .consumed = 0,
764 .produced = 0,
765 .pad_descs = NULL,
766 .active = 0,
767 .done = 0,
768 .tcfg = NULL,
769 .prev_src = NULL,
770 .curr_src = NULL,
771 .unit_no = src_csum};
772 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773
774 unsigned int indata_ix = 0;
775
776 /* iovec accounting. */
777 int iniov_ix = 0;
778 int iniov_offset = 0;
779
780 /* Operation descriptor cfg traversal pointer. */
781 struct cryptocop_desc *odsc;
782
783 int failed = 0;
784 /* List heads for allocated descriptors. */
785 struct cryptocop_dma_desc out_cdesc_head = {0};
786 struct cryptocop_dma_desc in_cdesc_head = {0};
787
788 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790
791 struct cryptocop_tfrm_ctx *output_tc = NULL;
792 void *iop_alloc_ptr;
793
794 assert(operation != NULL);
795 assert(int_op != NULL);
796
797 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 DEBUG(print_cryptocop_operation(operation));
799
800 sess = get_session(operation->sid);
801 if (!sess) {
802 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 failed = -EINVAL;
804 goto error_cleanup;
805 }
806 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 if (!iop_alloc_ptr) {
808 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
809 failed = -ENOMEM;
810 goto error_cleanup;
811 }
812 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 (*int_op)->alloc_ptr = iop_alloc_ptr;
815 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816
817 (*int_op)->sid = operation->sid;
818 (*int_op)->cdesc_out = NULL;
819 (*int_op)->cdesc_in = NULL;
820 (*int_op)->tdes_mode = cryptocop_3des_ede;
821 (*int_op)->csum_mode = cryptocop_csum_le;
822 (*int_op)->ddesc_out = NULL;
823 (*int_op)->ddesc_in = NULL;
824
825 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 if (!tcfg) {
827 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 failed = -EINVAL;
829 goto error_cleanup;
830 }
831 while (tcfg) {
832 tctx = get_transform_ctx(sess, tcfg->tid);
833 if (!tctx) {
834 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 failed = -EINVAL;
836 goto error_cleanup;
837 }
838 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 failed = -EINVAL;
841 goto error_cleanup;
842 }
843 switch (tctx->init.alg){
844 case cryptocop_alg_mem2mem:
845 if (cipher_ctx.tcfg != NULL){
846 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 failed = -EINVAL;
848 goto error_cleanup;
849 }
850 /* mem2mem is handled as a NULL cipher. */
851 cipher_ctx.cbcmode = 0;
852 cipher_ctx.decrypt = 0;
853 cipher_ctx.blocklength = 1;
854 cipher_ctx.ciph_conf = 0;
855 cipher_ctx.unit_no = src_dma;
856 cipher_ctx.tcfg = tcfg;
857 cipher_ctx.tctx = tctx;
858 break;
859 case cryptocop_alg_des:
860 case cryptocop_alg_3des:
861 case cryptocop_alg_aes:
862 /* cipher */
863 if (cipher_ctx.tcfg != NULL){
864 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 failed = -EINVAL;
866 goto error_cleanup;
867 }
868 cipher_ctx.tcfg = tcfg;
869 cipher_ctx.tctx = tctx;
870 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 cipher_ctx.decrypt = 1;
872 }
873 switch (tctx->init.cipher_mode) {
874 case cryptocop_cipher_mode_ecb:
875 cipher_ctx.cbcmode = 0;
876 break;
877 case cryptocop_cipher_mode_cbc:
878 cipher_ctx.cbcmode = 1;
879 break;
880 default:
881 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 failed = -EINVAL;
883 goto error_cleanup;
884 }
885 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 switch (tctx->init.alg){
887 case cryptocop_alg_des:
888 cipher_ctx.ciph_conf = 0;
889 cipher_ctx.unit_no = src_des;
890 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 break;
892 case cryptocop_alg_3des:
893 cipher_ctx.ciph_conf = 1;
894 cipher_ctx.unit_no = src_des;
895 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 break;
897 case cryptocop_alg_aes:
898 cipher_ctx.ciph_conf = 2;
899 cipher_ctx.unit_no = src_aes;
900 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 break;
902 default:
903 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 }
905 (*int_op)->tdes_mode = tctx->init.tdes_mode;
906 break;
907 case cryptocop_alg_md5:
908 case cryptocop_alg_sha1:
909 /* digest */
910 if (digest_ctx.tcfg != NULL){
911 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 failed = -EINVAL;
913 goto error_cleanup;
914 }
915 digest_ctx.tcfg = tcfg;
916 digest_ctx.tctx = tctx;
917 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 switch (tctx->init.alg){
919 case cryptocop_alg_md5:
920 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 digest_ctx.unit_no = src_md5;
922 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 break;
924 case cryptocop_alg_sha1:
925 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 digest_ctx.unit_no = src_sha1;
927 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 break;
929 default:
930 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 }
932 break;
933 case cryptocop_alg_csum:
934 /* digest */
935 if (csum_ctx.tcfg != NULL){
936 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 failed = -EINVAL;
938 goto error_cleanup;
939 }
940 (*int_op)->csum_mode = tctx->init.csum_mode;
941 csum_ctx.tcfg = tcfg;
942 csum_ctx.tctx = tctx;
943 break;
944 default:
945 /* no algorithm. */
946 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 failed = -EINVAL;
948 goto error_cleanup;
949 }
950 tcfg = tcfg->next;
951 }
952 /* Download key if a cipher is used. */
953 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 struct cryptocop_dma_desc *key_desc = NULL;
955
956 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 if (failed) {
958 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 goto error_cleanup;
960 }
961 current_out_cdesc->next = key_desc;
962 current_out_cdesc = key_desc;
963 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964
965 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 struct cryptocop_dma_desc *iv_desc = NULL;
968
969 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970
971 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 if (failed) {
973 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 goto error_cleanup;
975 }
976 current_out_cdesc->next = iv_desc;
977 current_out_cdesc = iv_desc;
978 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 }
980 }
981
982 /* Process descriptors. */
983 odsc = operation->tfrm_op.desc;
984 while (odsc) {
985 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
986 struct strcop_meta_out meta_out = {0};
987 size_t desc_len = odsc->length;
988 int active_count, eop_needed_count;
989
990 output_tc = NULL;
991
992 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993
994 while (dcfg) {
995 struct cryptocop_tfrm_ctx *tc = NULL;
996
997 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 /* Get the local context for the transform and mark it as the output unit if it produces output. */
999 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 tc = &digest_ctx;
1001 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 tc = &cipher_ctx;
1003 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 tc = &csum_ctx;
1005 }
1006 if (!tc) {
1007 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 failed = -EINVAL;
1009 goto error_cleanup;
1010 }
1011 if (tc->done) {
1012 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 failed = -EINVAL;
1014 goto error_cleanup;
1015 }
1016 if (!tc->active) {
1017 tc->start_ix = indata_ix;
1018 tc->active = 1;
1019 }
1020
1021 tc->previous_src = tc->current_src;
1022 tc->prev_src = tc->curr_src;
1023 /* Map source unit id to DMA source config. */
1024 switch (dcfg->src){
1025 case cryptocop_source_dma:
1026 tc->current_src = src_dma;
1027 break;
1028 case cryptocop_source_des:
1029 tc->current_src = src_des;
1030 break;
1031 case cryptocop_source_3des:
1032 tc->current_src = src_des;
1033 break;
1034 case cryptocop_source_aes:
1035 tc->current_src = src_aes;
1036 break;
1037 case cryptocop_source_md5:
1038 case cryptocop_source_sha1:
1039 case cryptocop_source_csum:
1040 case cryptocop_source_none:
1041 default:
1042 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 */
1044 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 failed = -EINVAL;
1046 goto error_cleanup;
1047 }
1048 if (tc->current_src != src_dma) {
1049 /* Find the unit we are sourcing from. */
1050 if (digest_ctx.unit_no == tc->current_src){
1051 tc->curr_src = &digest_ctx;
1052 } else if (cipher_ctx.unit_no == tc->current_src){
1053 tc->curr_src = &cipher_ctx;
1054 } else if (csum_ctx.unit_no == tc->current_src){
1055 tc->curr_src = &csum_ctx;
1056 }
1057 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 failed = -EINVAL;
1060 goto error_cleanup;
1061 }
1062 } else {
1063 tc->curr_src = NULL;
1064 }
1065
1066 /* Detect source switch. */
1067 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 if (tc->active && (tc->current_src != tc->previous_src)) {
1069 /* Only allow source switch when both the old source unit and the new one have
1070 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 * transform blocklength). */
1072 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 {
1076 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 failed = -EINVAL;
1078 goto error_cleanup;
1079 }
1080 }
1081 /* Detect unit deactivation. */
1082 if (dcfg->last) {
1083 /* Length check of this is handled below. */
1084 tc->done = 1;
1085 }
1086 dcfg = dcfg->next;
1087 } /* while (dcfg) */
1088 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 failed = -EINVAL;
1093 goto error_cleanup;
1094 }
1095 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 failed = -EINVAL;
1098 goto error_cleanup;
1099 }
1100 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 failed = -EINVAL;
1103 goto error_cleanup;
1104 }
1105
1106 /* Update consumed and produced lengths.
1107
1108 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1109 other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1111 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 unit has processed an exact multiple of its block length the end result will be correct.
1113 Beware that if the source change restriction change this code will need to be (much) reworked.
1114 */
1115 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117 if (csum_ctx.active) {
1118 csum_ctx.consumed += desc_len;
1119 if (csum_ctx.done) {
1120 csum_ctx.produced = 2;
1121 }
1122 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 }
1124 if (digest_ctx.active) {
1125 digest_ctx.consumed += desc_len;
1126 if (digest_ctx.done) {
1127 if (digest_ctx.unit_no == src_md5) {
1128 digest_ctx.produced = MD5_STATE_LENGTH;
1129 } else {
1130 digest_ctx.produced = SHA1_STATE_LENGTH;
1131 }
1132 }
1133 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 }
1135 if (cipher_ctx.active) {
1136 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1137 assert(cipher_ctx.current_src == src_dma);
1138 cipher_ctx.consumed += desc_len;
1139 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 }
1143 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 }
1145
1146 /* Setup the DMA out descriptors. */
1147 /* Configure the metadata. */
1148 active_count = 0;
1149 eop_needed_count = 0;
1150 if (cipher_ctx.active) {
1151 ++active_count;
1152 if (cipher_ctx.unit_no == src_dma){
1153 /* mem2mem */
1154 meta_out.ciphsel = src_none;
1155 } else {
1156 meta_out.ciphsel = cipher_ctx.current_src;
1157 }
1158 meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 meta_out.cbcmode = cipher_ctx.cbcmode;
1160 meta_out.decrypt = cipher_ctx.decrypt;
1161 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 if (cipher_ctx.done) ++eop_needed_count;
1163 } else {
1164 meta_out.ciphsel = src_none;
1165 }
1166
1167 if (digest_ctx.active) {
1168 ++active_count;
1169 meta_out.hashsel = digest_ctx.current_src;
1170 meta_out.hashconf = digest_ctx.hash_conf;
1171 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 if (digest_ctx.done) {
1174 assert(digest_ctx.pad_descs == NULL);
1175 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 if (failed) {
1177 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 goto error_cleanup;
1179 }
1180 }
1181 } else {
1182 meta_out.hashsel = src_none;
1183 }
1184
1185 if (csum_ctx.active) {
1186 ++active_count;
1187 meta_out.csumsel = csum_ctx.current_src;
1188 if (csum_ctx.done) {
1189 assert(csum_ctx.pad_descs == NULL);
1190 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 if (failed) {
1192 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 goto error_cleanup;
1194 }
1195 }
1196 } else {
1197 meta_out.csumsel = src_none;
1198 }
1199 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 /* Setup DMA out descriptors for the indata. */
1201 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 if (failed) {
1203 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 goto error_cleanup;
1205 }
1206 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1207 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 */
1210 assert(active_count >= eop_needed_count);
1211 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 if (eop_needed_count) {
1213 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214 if (active_count > 1) {
1215 /* Use zero length EOP descriptor. */
1216 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 struct strcop_meta_out ed_mo = {0};
1218 if (!ed) {
1219 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 failed = -ENOMEM;
1221 goto error_cleanup;
1222 }
1223
1224 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226 if (cipher_ctx.unit_no == src_dma){
1227 /* mem2mem */
1228 ed_mo.ciphsel = src_none;
1229 } else {
1230 ed_mo.ciphsel = cipher_ctx.current_src;
1231 }
1232 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236 ed->free_buf = NULL;
1237 ed->dma_descr->wait = 1;
1238 ed->dma_descr->out_eop = 1;
1239
1240 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 ed->dma_descr->after = ed->dma_descr->buf;
1242 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 current_out_cdesc->next = ed;
1244 current_out_cdesc = ed;
1245 } else {
1246 /* Set EOP in the current out descriptor since the only active module is
1247 * the one needing the EOP. */
1248
1249 current_out_cdesc->dma_descr->out_eop = 1;
1250 }
1251 }
1252
1253 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 indata_ix += odsc->length;
1257 odsc = odsc->next;
1258 } /* while (odsc) */ /* Process descriptors. */
1259 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 failed = -EINVAL;
1263 goto error_cleanup;
1264 }
1265 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 failed = -EINVAL;
1268 goto error_cleanup;
1269 }
1270 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 failed = -EINVAL;
1273 goto error_cleanup;
1274 }
1275
1276 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 if (failed){
1278 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 goto error_cleanup;
1280 }
1281 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 if (failed){
1283 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 goto error_cleanup;
1285 }
1286 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 if (failed){
1288 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 goto error_cleanup;
1290 }
1291
1292 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 (*int_op)->cdesc_out = out_cdesc_head.next;
1294 (*int_op)->cdesc_in = in_cdesc_head.next;
1295 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297 setup_descr_chain(out_cdesc_head.next);
1298 setup_descr_chain(in_cdesc_head.next);
1299
1300 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 * last DMA out descriptor for EOL.
1302 */
1303 current_in_cdesc->dma_descr->intr = 1;
1304 current_in_cdesc->dma_descr->eol = 1;
1305 current_out_cdesc->dma_descr->eol = 1;
1306
1307 /* Setup DMA contexts. */
1308 (*int_op)->ctx_out.next = NULL;
1309 (*int_op)->ctx_out.eol = 1;
1310 (*int_op)->ctx_out.intr = 0;
1311 (*int_op)->ctx_out.store_mode = 0;
1312 (*int_op)->ctx_out.en = 0;
1313 (*int_op)->ctx_out.dis = 0;
1314 (*int_op)->ctx_out.md0 = 0;
1315 (*int_op)->ctx_out.md1 = 0;
1316 (*int_op)->ctx_out.md2 = 0;
1317 (*int_op)->ctx_out.md3 = 0;
1318 (*int_op)->ctx_out.md4 = 0;
1319 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322 (*int_op)->ctx_in.next = NULL;
1323 (*int_op)->ctx_in.eol = 1;
1324 (*int_op)->ctx_in.intr = 0;
1325 (*int_op)->ctx_in.store_mode = 0;
1326 (*int_op)->ctx_in.en = 0;
1327 (*int_op)->ctx_in.dis = 0;
1328 (*int_op)->ctx_in.md0 = 0;
1329 (*int_op)->ctx_in.md1 = 0;
1330 (*int_op)->ctx_in.md2 = 0;
1331 (*int_op)->ctx_in.md3 = 0;
1332 (*int_op)->ctx_in.md4 = 0;
1333
1334 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 return 0;
1339
1340 error_cleanup:
1341 {
1342 /* Free all allocated resources. */
1343 struct cryptocop_dma_desc *tmp_cdesc;
1344 while (digest_ctx.pad_descs){
1345 tmp_cdesc = digest_ctx.pad_descs->next;
1346 free_cdesc(digest_ctx.pad_descs);
1347 digest_ctx.pad_descs = tmp_cdesc;
1348 }
1349 while (csum_ctx.pad_descs){
1350 tmp_cdesc = csum_ctx.pad_descs->next;
1351 free_cdesc(csum_ctx.pad_descs);
1352 csum_ctx.pad_descs = tmp_cdesc;
1353 }
1354 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356 if (*int_op != NULL) delete_internal_operation(*int_op);
1357 }
1358 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 return failed;
1360 }
1361
1362
1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365 void *ptr = iop->alloc_ptr;
1366 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 struct cryptocop_dma_desc *next;
1368
1369 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371 while (cd) {
1372 next = cd->next;
1373 free_cdesc(cd);
1374 cd = next;
1375 }
1376 cd = iop->cdesc_in;
1377 while (cd) {
1378 next = cd->next;
1379 free_cdesc(cd);
1380 cd = next;
1381 }
1382 kfree(ptr);
1383 }
1384
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 unsigned char *p;
1392 int i;
1393 unsigned long long int bit_length = hashed_length << 3;
1394
1395 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397 p = kzalloc(padlen, alloc_flag);
1398 if (!p) return -ENOMEM;
1399
1400 *p = 0x80;
1401
1402 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405 while (bit_length != 0){
1406 p[i++] = bit_length % 0x100;
1407 bit_length >>= 8;
1408 }
1409
1410 *pad = (char*)p;
1411 *pad_length = padlen;
1412
1413 return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422 unsigned char *p;
1423 int i;
1424 unsigned long long int bit_length = hashed_length << 3;
1425
1426 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428 p = kzalloc(padlen, alloc_flag);
1429 if (!p) return -ENOMEM;
1430
1431 *p = 0x80;
1432
1433 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434
1435 i = padlen - 1;
1436 while (bit_length != 0){
1437 p[i--] = bit_length % 0x100;
1438 bit_length >>= 8;
1439 }
1440
1441 *pad = (char*)p;
1442 *pad_length = padlen;
1443
1444 return 0;
1445 }
1446
1447
1448 static int transform_ok(struct cryptocop_transform_init *tinit)
1449 {
1450 switch (tinit->alg){
1451 case cryptocop_alg_csum:
1452 switch (tinit->csum_mode){
1453 case cryptocop_csum_le:
1454 case cryptocop_csum_be:
1455 break;
1456 default:
1457 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458 return -EINVAL;
1459 }
1460 case cryptocop_alg_mem2mem:
1461 case cryptocop_alg_md5:
1462 case cryptocop_alg_sha1:
1463 if (tinit->keylen != 0) {
1464 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465 return -EINVAL; /* This check is a bit strict. */
1466 }
1467 break;
1468 case cryptocop_alg_des:
1469 if (tinit->keylen != 64) {
1470 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471 return -EINVAL;
1472 }
1473 break;
1474 case cryptocop_alg_3des:
1475 if (tinit->keylen != 192) {
1476 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477 return -EINVAL;
1478 }
1479 break;
1480 case cryptocop_alg_aes:
1481 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483 return -EINVAL;
1484 }
1485 break;
1486 case cryptocop_no_alg:
1487 default:
1488 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489 return -EINVAL;
1490 }
1491
1492 switch (tinit->alg){
1493 case cryptocop_alg_des:
1494 case cryptocop_alg_3des:
1495 case cryptocop_alg_aes:
1496 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497 default:
1498 break;
1499 }
1500 return 0;
1501 }
1502
1503
1504 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505 {
1506 struct cryptocop_session *sess;
1507 struct cryptocop_transform_init *tfrm_in = tinit;
1508 struct cryptocop_transform_init *tmp_in;
1509 int no_tfrms = 0;
1510 int i;
1511 unsigned long int flags;
1512
1513 init_stream_coprocessor(); /* For safety if we are called early */
1514
1515 while (tfrm_in){
1516 int err;
1517 ++no_tfrms;
1518 if ((err = transform_ok(tfrm_in))) {
1519 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520 return err;
1521 }
1522 tfrm_in = tfrm_in->next;
1523 }
1524 if (0 == no_tfrms) {
1525 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526 return -EINVAL;
1527 }
1528
1529 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530 if (!sess){
1531 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532 return -ENOMEM;
1533 }
1534
1535 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536 if (!sess->tfrm_ctx) {
1537 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538 kfree(sess);
1539 return -ENOMEM;
1540 }
1541
1542 tfrm_in = tinit;
1543 for (i = 0; i < no_tfrms; i++){
1544 tmp_in = tfrm_in->next;
1545 while (tmp_in){
1546 if (tmp_in->tid == tfrm_in->tid) {
1547 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548 kfree(sess->tfrm_ctx);
1549 kfree(sess);
1550 return -EINVAL;
1551 }
1552 tmp_in = tmp_in->next;
1553 }
1554 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555 sess->tfrm_ctx[i].dec_key_set = 0;
1556 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557
1558 tfrm_in = tfrm_in->next;
1559 }
1560 sess->tfrm_ctx[i-1].next = NULL;
1561
1562 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563 sess->sid = next_sid;
1564 next_sid++;
1565 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566 * OTOH 2^64 is a really large number of session. */
1567 if (next_sid == 0) next_sid = 1;
1568
1569 /* Prepend to session list. */
1570 sess->next = cryptocop_sessions;
1571 cryptocop_sessions = sess;
1572 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573 *sid = sess->sid;
1574 return 0;
1575 }
1576
1577
1578 int cryptocop_free_session(cryptocop_session_id sid)
1579 {
1580 struct cryptocop_transform_ctx *tc;
1581 struct cryptocop_session *sess = NULL;
1582 struct cryptocop_session *psess = NULL;
1583 unsigned long int flags;
1584 int i;
1585 LIST_HEAD(remove_list);
1586 struct list_head *node, *tmp;
1587 struct cryptocop_prio_job *pj;
1588
1589 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590
1591 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592 sess = cryptocop_sessions;
1593 while (sess && sess->sid != sid){
1594 psess = sess;
1595 sess = sess->next;
1596 }
1597 if (sess){
1598 if (psess){
1599 psess->next = sess->next;
1600 } else {
1601 cryptocop_sessions = sess->next;
1602 }
1603 }
1604 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605
1606 if (!sess) return -EINVAL;
1607
1608 /* Remove queued jobs. */
1609 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610
1611 for (i = 0; i < cryptocop_prio_no_prios; i++){
1612 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614 pj = list_entry(node, struct cryptocop_prio_job, node);
1615 if (pj->oper->sid == sid) {
1616 list_move_tail(node, &remove_list);
1617 }
1618 }
1619 }
1620 }
1621 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622
1623 list_for_each_safe(node, tmp, &remove_list) {
1624 list_del(node);
1625 pj = list_entry(node, struct cryptocop_prio_job, node);
1626 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628 pj->oper->cb(pj->oper, pj->oper->cb_data);
1629 delete_internal_operation(pj->iop);
1630 kfree(pj);
1631 }
1632
1633 tc = sess->tfrm_ctx;
1634 /* Erase keying data. */
1635 while (tc){
1636 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 tc = tc->next;
1640 }
1641 kfree(sess->tfrm_ctx);
1642 kfree(sess);
1643
1644 return 0;
1645 }
1646
1647 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648 {
1649 struct cryptocop_session *sess;
1650 unsigned long int flags;
1651
1652 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653 sess = cryptocop_sessions;
1654 while (sess && (sess->sid != sid)){
1655 sess = sess->next;
1656 }
1657 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658
1659 return sess;
1660 }
1661
1662 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663 {
1664 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665
1666 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667 assert(sess != NULL);
1668 while (tc && tc->init.tid != tid){
1669 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670 tc = tc->next;
1671 }
1672 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673 return tc;
1674 }
1675
1676
1677
1678 /* The AES s-transform matrix (s-box). */
1679 static const u8 aes_sbox[256] = {
1680 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1681 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1682 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1683 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1684 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1685 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1686 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1687 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1688 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1689 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1690 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1691 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1692 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1693 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1694 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1695 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1696 };
1697
1698 /* AES has a 32 bit word round constants for each round in the
1699 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1700 */
1701 static u32 round_constant[11] = {
1702 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704 0x1B000000, 0x36000000, 0x6C000000
1705 };
1706
1707 /* Apply the s-box to each of the four occtets in w. */
1708 static u32 aes_ks_subword(const u32 w)
1709 {
1710 u8 bytes[4];
1711
1712 *(u32*)(&bytes[0]) = w;
1713 bytes[0] = aes_sbox[bytes[0]];
1714 bytes[1] = aes_sbox[bytes[1]];
1715 bytes[2] = aes_sbox[bytes[2]];
1716 bytes[3] = aes_sbox[bytes[3]];
1717 return *(u32*)(&bytes[0]);
1718 }
1719
1720 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721 * (Note that AES words are 32 bit long)
1722 *
1723 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724 * word temp
1725 * i = 0
1726 * while (i < Nk) {
1727 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728 * i = i + 1
1729 * }
1730 * i = Nk
1731 *
1732 * while (i < (Nb * (Nr + 1))) {
1733 * temp = w[i - 1]
1734 * if ((i mod Nk) == 0) {
1735 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736 * }
1737 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738 * temp = SubWord(temp)
1739 * }
1740 * w[i] = w[i - Nk] xor temp
1741 * }
1742 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743 * SubWord(t) applies the AES s-box individually to each octet
1744 * in a 32 bit word.
1745 *
1746 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747 * values for Nr of 10, 12, and 14). Nb is always 4.
1748 *
1749 * To construct w[i], w[i - 1] and w[i - Nk] must be
1750 * available. Consequently we must keep a state of the last Nk words
1751 * to be able to create the last round keys.
1752 */
1753 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1754 {
1755 u32 temp;
1756 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757 u8 w_last_ix;
1758 int i;
1759 u8 nr, nk;
1760
1761 switch (keylength){
1762 case 128:
1763 nk = 4;
1764 nr = 10;
1765 break;
1766 case 192:
1767 nk = 6;
1768 nr = 12;
1769 break;
1770 case 256:
1771 nk = 8;
1772 nr = 14;
1773 break;
1774 default:
1775 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776 };
1777
1778 /* Need to do host byte order correction here since key is byte oriented and the
1779 * kx algorithm is word (u32) oriented. */
1780 for (i = 0; i < nk; i+=1) {
1781 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782 }
1783
1784 i = (int)nk;
1785 w_last_ix = i - 1;
1786 while (i < (4 * (nr + 2))) {
1787 temp = w_ring[w_last_ix];
1788 if (!(i % nk)) {
1789 /* RotWord(temp) */
1790 temp = (temp << 8) | (temp >> 24);
1791 temp = aes_ks_subword(temp);
1792 temp ^= round_constant[i/nk - 1];
1793 } else if ((nk > 6) && ((i % nk) == 4)) {
1794 temp = aes_ks_subword(temp);
1795 }
1796 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797 temp ^= w_ring[w_last_ix];
1798 w_ring[w_last_ix] = temp;
1799
1800 /* We need the round keys for round Nr+1 and Nr+2 (round key
1801 * Nr+2 is the round key beyond the last one used when
1802 * encrypting). Rounds are numbered starting from 0, Nr=10
1803 * implies 11 rounds are used in encryption/decryption.
1804 */
1805 if (i >= (4 * nr)) {
1806 /* Need to do host byte order correction here, the key
1807 * is byte oriented. */
1808 *(u32*)dec_key = cpu_to_be32(temp);
1809 dec_key += 4;
1810 }
1811 ++i;
1812 }
1813 }
1814
1815
1816 /**** Job/operation management. ****/
1817
1818 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819 {
1820 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821 }
1822
1823 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824 {
1825 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826 }
1827
1828 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829 {
1830 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831 }
1832
1833 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834 {
1835 int ret;
1836 struct cryptocop_prio_job *pj = NULL;
1837 unsigned long int flags;
1838
1839 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840
1841 if (!operation || !operation->cb){
1842 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843 return -EINVAL;
1844 }
1845
1846 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848 return ret;
1849 }
1850 assert(pj != NULL);
1851
1852 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855
1856 /* Make sure a job is running */
1857 cryptocop_start_job();
1858 return 0;
1859 }
1860
1861 static void cryptocop_do_tasklet(unsigned long unused);
1862 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863
1864 static void cryptocop_do_tasklet(unsigned long unused)
1865 {
1866 struct list_head *node;
1867 struct cryptocop_prio_job *pj = NULL;
1868 unsigned long flags;
1869
1870 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871
1872 do {
1873 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874 if (!list_empty(&cryptocop_completed_jobs)){
1875 node = cryptocop_completed_jobs.next;
1876 list_del(node);
1877 pj = list_entry(node, struct cryptocop_prio_job, node);
1878 } else {
1879 pj = NULL;
1880 }
1881 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882 if (pj) {
1883 assert(pj->oper != NULL);
1884
1885 /* Notify consumer of operation completeness. */
1886 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887
1888 pj->oper->operation_status = 0; /* Job is completed. */
1889 pj->oper->cb(pj->oper, pj->oper->cb_data);
1890 delete_internal_operation(pj->iop);
1891 kfree(pj);
1892 }
1893 } while (pj != NULL);
1894
1895 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896 }
1897
1898 static irqreturn_t
1899 dma_done_interrupt(int irq, void *dev_id)
1900 {
1901 struct cryptocop_prio_job *done_job;
1902 reg_dma_rw_ack_intr ack_intr = {
1903 .data = 1,
1904 };
1905
1906 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907
1908 DEBUG(printk("cryptocop DMA done\n"));
1909
1910 spin_lock(&running_job_lock);
1911 if (cryptocop_running_job == NULL){
1912 printk("stream co-processor got interrupt when not busy\n");
1913 spin_unlock(&running_job_lock);
1914 return IRQ_HANDLED;
1915 }
1916 done_job = cryptocop_running_job;
1917 cryptocop_running_job = NULL;
1918 spin_unlock(&running_job_lock);
1919
1920 /* Start processing a job. */
1921 if (!spin_trylock(&cryptocop_process_lock)){
1922 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923 } else {
1924 cryptocop_start_job();
1925 spin_unlock(&cryptocop_process_lock);
1926 }
1927
1928 done_job->oper->operation_status = 0; /* Job is completed. */
1929 if (done_job->oper->fast_callback){
1930 /* This operation wants callback from interrupt. */
1931 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932 delete_internal_operation(done_job->iop);
1933 kfree(done_job);
1934 } else {
1935 spin_lock(&cryptocop_completed_jobs_lock);
1936 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937 spin_unlock(&cryptocop_completed_jobs_lock);
1938 tasklet_schedule(&cryptocop_tasklet);
1939 }
1940
1941 DEBUG(printk("cryptocop leave irq handler\n"));
1942 return IRQ_HANDLED;
1943 }
1944
1945
1946 /* Setup interrupts and DMA channels. */
1947 static int init_cryptocop(void)
1948 {
1949 unsigned long flags;
1950 reg_dma_rw_cfg dma_cfg = {.en = 1};
1951 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1953 reg_strcop_rw_cfg strcop_cfg = {
1954 .ipend = regk_strcop_little,
1955 .td1 = regk_strcop_e,
1956 .td2 = regk_strcop_d,
1957 .td3 = regk_strcop_e,
1958 .ignore_sync = 0,
1959 .en = 1
1960 };
1961
1962 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963 "stream co-processor DMA", NULL))
1964 panic("request_irq stream co-processor irq dma9");
1965
1966 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967 0, dma_strp);
1968 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 0, dma_strp);
1970
1971 local_irq_save(flags);
1972
1973 /* Reset and enable the cryptocop. */
1974 strcop_cfg.en = 0;
1975 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976 strcop_cfg.en = 1;
1977 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978
1979 /* Enable DMAs. */
1980 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982
1983 /* Set up wordsize = 4 for DMAs. */
1984 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986
1987 /* Enable interrupts. */
1988 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989
1990 /* Clear intr ack. */
1991 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992
1993 local_irq_restore(flags);
1994
1995 return 0;
1996 }
1997
1998 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1999 static void release_cryptocop(void)
2000 {
2001 unsigned long flags;
2002 reg_dma_rw_cfg dma_cfg = {.en = 0};
2003 reg_dma_rw_intr_mask intr_mask_in = {0};
2004 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2005
2006 local_irq_save(flags);
2007
2008 /* Clear intr ack. */
2009 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010
2011 /* Disable DMAs. */
2012 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014
2015 /* Disable interrupts. */
2016 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017
2018 local_irq_restore(flags);
2019
2020 free_irq(DMA_IRQ, NULL);
2021
2022 (void)crisv32_free_dma(OUT_DMA);
2023 (void)crisv32_free_dma(IN_DMA);
2024 }
2025
2026
2027 /* Init job queue. */
2028 static int cryptocop_job_queue_init(void)
2029 {
2030 int i;
2031
2032 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033
2034 for (i = 0; i < cryptocop_prio_no_prios; i++){
2035 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037 }
2038 return 0;
2039 }
2040
2041
2042 static void cryptocop_job_queue_close(void)
2043 {
2044 struct list_head *node, *tmp;
2045 struct cryptocop_prio_job *pj = NULL;
2046 unsigned long int process_flags, flags;
2047 int i;
2048
2049 /* FIXME: This is as yet untested code. */
2050
2051 /* Stop strcop from getting an operation to process while we are closing the
2052 module. */
2053 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054
2055 /* Empty the job queue. */
2056 for (i = 0; i < cryptocop_prio_no_prios; i++){
2057 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059 pj = list_entry(node, struct cryptocop_prio_job, node);
2060 list_del(node);
2061
2062 /* Call callback to notify consumer of job removal. */
2063 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065 pj->oper->cb(pj->oper, pj->oper->cb_data);
2066
2067 delete_internal_operation(pj->iop);
2068 kfree(pj);
2069 }
2070 }
2071 }
2072 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073
2074 /* Remove the running job, if any. */
2075 spin_lock_irqsave(&running_job_lock, flags);
2076 if (cryptocop_running_job){
2077 reg_strcop_rw_cfg rw_cfg;
2078 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2079
2080 /* Stop DMA. */
2081 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2082 dma_out_cfg.en = regk_dma_no;
2083 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2084
2085 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2086 dma_in_cfg.en = regk_dma_no;
2087 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2088
2089 /* Disble the cryptocop. */
2090 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091 rw_cfg.en = 0;
2092 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093
2094 pj = cryptocop_running_job;
2095 cryptocop_running_job = NULL;
2096
2097 /* Call callback to notify consumer of job removal. */
2098 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100 pj->oper->cb(pj->oper, pj->oper->cb_data);
2101
2102 delete_internal_operation(pj->iop);
2103 kfree(pj);
2104 }
2105 spin_unlock_irqrestore(&running_job_lock, flags);
2106
2107 /* Remove completed jobs, if any. */
2108 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109
2110 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111 pj = list_entry(node, struct cryptocop_prio_job, node);
2112 list_del(node);
2113 /* Call callback to notify consumer of job removal. */
2114 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116 pj->oper->cb(pj->oper, pj->oper->cb_data);
2117
2118 delete_internal_operation(pj->iop);
2119 kfree(pj);
2120 }
2121 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122 }
2123
2124
2125 static void cryptocop_start_job(void)
2126 {
2127 int i;
2128 struct cryptocop_prio_job *pj;
2129 unsigned long int flags;
2130 unsigned long int running_job_flags;
2131 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2132
2133 DEBUG(printk("cryptocop_start_job: entering\n"));
2134
2135 spin_lock_irqsave(&running_job_lock, running_job_flags);
2136 if (cryptocop_running_job != NULL){
2137 /* Already running. */
2138 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140 return;
2141 }
2142 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143
2144 /* Check the queues in priority order. */
2145 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146 if (i == cryptocop_prio_no_prios) {
2147 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150 return; /* No jobs to run */
2151 }
2152 DEBUG(printk("starting job for prio %d\n", i));
2153
2154 /* TODO: Do not starve lower priority jobs. Let in a lower
2155 * prio job for every N-th processed higher prio job or some
2156 * other scheduling policy. This could reasonably be
2157 * tweakable since the optimal balance would depend on the
2158 * type of load on the system. */
2159
2160 /* Pull the DMA lists from the job and start the DMA client. */
2161 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162 list_del(&pj->node);
2163 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164 cryptocop_running_job = pj;
2165
2166 /* Set config register (3DES and CSUM modes). */
2167 switch (pj->iop->tdes_mode){
2168 case cryptocop_3des_eee:
2169 rw_cfg.td1 = regk_strcop_e;
2170 rw_cfg.td2 = regk_strcop_e;
2171 rw_cfg.td3 = regk_strcop_e;
2172 break;
2173 case cryptocop_3des_eed:
2174 rw_cfg.td1 = regk_strcop_e;
2175 rw_cfg.td2 = regk_strcop_e;
2176 rw_cfg.td3 = regk_strcop_d;
2177 break;
2178 case cryptocop_3des_ede:
2179 rw_cfg.td1 = regk_strcop_e;
2180 rw_cfg.td2 = regk_strcop_d;
2181 rw_cfg.td3 = regk_strcop_e;
2182 break;
2183 case cryptocop_3des_edd:
2184 rw_cfg.td1 = regk_strcop_e;
2185 rw_cfg.td2 = regk_strcop_d;
2186 rw_cfg.td3 = regk_strcop_d;
2187 break;
2188 case cryptocop_3des_dee:
2189 rw_cfg.td1 = regk_strcop_d;
2190 rw_cfg.td2 = regk_strcop_e;
2191 rw_cfg.td3 = regk_strcop_e;
2192 break;
2193 case cryptocop_3des_ded:
2194 rw_cfg.td1 = regk_strcop_d;
2195 rw_cfg.td2 = regk_strcop_e;
2196 rw_cfg.td3 = regk_strcop_d;
2197 break;
2198 case cryptocop_3des_dde:
2199 rw_cfg.td1 = regk_strcop_d;
2200 rw_cfg.td2 = regk_strcop_d;
2201 rw_cfg.td3 = regk_strcop_e;
2202 break;
2203 case cryptocop_3des_ddd:
2204 rw_cfg.td1 = regk_strcop_d;
2205 rw_cfg.td2 = regk_strcop_d;
2206 rw_cfg.td3 = regk_strcop_d;
2207 break;
2208 default:
2209 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210 }
2211 switch (pj->iop->csum_mode){
2212 case cryptocop_csum_le:
2213 rw_cfg.ipend = regk_strcop_little;
2214 break;
2215 case cryptocop_csum_be:
2216 rw_cfg.ipend = regk_strcop_big;
2217 break;
2218 default:
2219 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220 }
2221 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222
2223 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224 "ctx_in: 0x%p, phys: 0x%p\n"
2225 "ctx_out: 0x%p, phys: 0x%p\n",
2226 pj,
2227 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229
2230 /* Start input DMA. */
2231 flush_dma_context(&pj->iop->ctx_in);
2232 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2233
2234 /* Start output DMA. */
2235 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2236
2237 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238 DEBUG(printk("cryptocop_start_job: exiting\n"));
2239 }
2240
2241
2242 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243 {
2244 int err;
2245 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246 void *iop_alloc_ptr = NULL;
2247
2248 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249 if (!*pj) return -ENOMEM;
2250
2251 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252
2253 (*pj)->oper = operation;
2254 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2255
2256 if (operation->use_dmalists) {
2257 DEBUG(print_user_dma_lists(&operation->list_op));
2258 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260 kfree(*pj);
2261 return -EINVAL;
2262 }
2263 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264 if (!iop_alloc_ptr) {
2265 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266 kfree(*pj);
2267 return -ENOMEM;
2268 }
2269 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272 (*pj)->iop->sid = operation->sid;
2273 (*pj)->iop->cdesc_out = NULL;
2274 (*pj)->iop->cdesc_in = NULL;
2275 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2279
2280 /* Setup DMA contexts. */
2281 (*pj)->iop->ctx_out.next = NULL;
2282 (*pj)->iop->ctx_out.eol = 1;
2283 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285
2286 (*pj)->iop->ctx_in.next = NULL;
2287 (*pj)->iop->ctx_in.eol = 1;
2288 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290 } else {
2291 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293 kfree(*pj);
2294 return err;
2295 }
2296 }
2297 DEBUG(print_dma_descriptors((*pj)->iop));
2298
2299 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300
2301 return 0;
2302 }
2303
2304 static int cryptocop_open(struct inode *inode, struct file *filp)
2305 {
2306 int p = iminor(inode);
2307
2308 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309
2310 filp->private_data = NULL;
2311 return 0;
2312 }
2313
2314
2315 static int cryptocop_release(struct inode *inode, struct file *filp)
2316 {
2317 struct cryptocop_private *dev = filp->private_data;
2318 struct cryptocop_private *dev_next;
2319
2320 while (dev){
2321 dev_next = dev->next;
2322 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323 (void)cryptocop_free_session(dev->sid);
2324 }
2325 kfree(dev);
2326 dev = dev_next;
2327 }
2328
2329 return 0;
2330 }
2331
2332
2333 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334 unsigned int cmd, unsigned long arg)
2335 {
2336 struct cryptocop_private *dev = filp->private_data;
2337 struct cryptocop_private *prev_dev = NULL;
2338 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2339 struct strcop_session_op sop;
2340 int err;
2341
2342 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343
2344 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345 return -EFAULT;
2346 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347 if (err) return -EFAULT;
2348
2349 while (dev && (dev->sid != sop.ses_id)) {
2350 prev_dev = dev;
2351 dev = dev->next;
2352 }
2353 if (dev){
2354 if (prev_dev){
2355 prev_dev->next = dev->next;
2356 } else {
2357 filp->private_data = dev->next;
2358 }
2359 err = cryptocop_free_session(dev->sid);
2360 if (err) return -EFAULT;
2361 } else {
2362 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363 return -EINVAL;
2364 }
2365 return 0;
2366 }
2367
2368
2369 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370 {
2371 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372
2373 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374
2375 jc->processed = 1;
2376 wake_up(&cryptocop_ioc_process_wq);
2377 }
2378
2379
2380 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2381 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2382 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2383
2384 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385 {
2386 size_t ch_ix = 0;
2387
2388 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391
2392 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393 return ch_ix;
2394 }
2395
2396
2397 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398 {
2399 size_t ch_ix = INT_MAX;
2400 size_t tmp_ix = 0;
2401
2402 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403 if (crp_op->cipher_start > ix) {
2404 ch_ix = crp_op->cipher_start;
2405 } else {
2406 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407 }
2408 }
2409 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410 if (crp_op->digest_start > ix) {
2411 tmp_ix = crp_op->digest_start;
2412 } else {
2413 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414 }
2415 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416 }
2417 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418 if (crp_op->csum_start > ix) {
2419 tmp_ix = crp_op->csum_start;
2420 } else {
2421 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422 }
2423 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424 }
2425 if (ch_ix == INT_MAX) ch_ix = ix;
2426 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427 return ch_ix;
2428 }
2429
2430
2431 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432 * Return -1 for ok, 0 for fail. */
2433 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434 {
2435 int tmplen;
2436
2437 assert(iov != NULL);
2438 assert(iovix != NULL);
2439 assert(pages != NULL);
2440 assert(pageix != NULL);
2441 assert(pageoffset != NULL);
2442
2443 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444
2445 while (map_length > 0){
2446 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447 if (*iovix >= iovlen){
2448 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449 return 0;
2450 }
2451 if (*pageix >= nopages){
2452 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453 return 0;
2454 }
2455 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456 tmplen = PAGE_SIZE - *pageoffset;
2457 if (tmplen < map_length){
2458 (*pageoffset) = 0;
2459 (*pageix)++;
2460 } else {
2461 tmplen = map_length;
2462 (*pageoffset) += map_length;
2463 }
2464 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465 iov[*iovix].iov_len = tmplen;
2466 map_length -= tmplen;
2467 (*iovix)++;
2468 }
2469 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470 return -1;
2471 }
2472
2473
2474
2475 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476 {
2477 int i;
2478 struct cryptocop_private *dev = filp->private_data;
2479 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2480 struct strcop_crypto_op oper = {0};
2481 int err = 0;
2482 struct cryptocop_operation *cop = NULL;
2483
2484 struct ioctl_job_cb_ctx *jc = NULL;
2485
2486 struct page **inpages = NULL;
2487 struct page **outpages = NULL;
2488 int noinpages = 0;
2489 int nooutpages = 0;
2490
2491 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492 * can get connected/disconnected on different places in the indata. */
2493 struct cryptocop_desc_cfg dcfgs[5*3];
2494 int desc_ix = 0;
2495 int dcfg_ix = 0;
2496 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2497 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2498 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2499
2500 unsigned char *digest_result = NULL;
2501 int digest_length = 0;
2502 int cblocklen = 0;
2503 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2504 struct cryptocop_session *sess;
2505
2506 int iovlen = 0;
2507 int iovix = 0;
2508 int pageix = 0;
2509 int pageoffset = 0;
2510
2511 size_t prev_ix = 0;
2512 size_t next_ix;
2513
2514 int cipher_active, digest_active, csum_active;
2515 int end_digest, end_csum;
2516 int digest_done = 0;
2517 int cipher_done = 0;
2518 int csum_done = 0;
2519
2520 DEBUG(printk("cryptocop_ioctl_process\n"));
2521
2522 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524 return -EFAULT;
2525 }
2526 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528 return -EFAULT;
2529 }
2530 DEBUG(print_strcop_crypto_op(&oper));
2531
2532 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533 if (!dev){
2534 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535 return -EINVAL;
2536 }
2537
2538 /* Check buffers. */
2539 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541 return -EINVAL;
2542 }
2543
2544 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546 return -EFAULT;
2547 }
2548 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550 return -EFAULT;
2551 }
2552
2553 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554 if (!cop) {
2555 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556 return -ENOMEM;
2557 }
2558 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559 if (!jc) {
2560 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561 err = -ENOMEM;
2562 goto error_cleanup;
2563 }
2564 jc->processed = 0;
2565
2566 cop->cb_data = jc;
2567 cop->cb = ioctl_process_job_callback;
2568 cop->operation_status = 0;
2569 cop->use_dmalists = 0;
2570 cop->in_interrupt = 0;
2571 cop->fast_callback = 0;
2572 cop->tfrm_op.tfrm_cfg = NULL;
2573 cop->tfrm_op.desc = NULL;
2574 cop->tfrm_op.indata = NULL;
2575 cop->tfrm_op.incount = 0;
2576 cop->tfrm_op.inlen = 0;
2577 cop->tfrm_op.outdata = NULL;
2578 cop->tfrm_op.outcount = 0;
2579 cop->tfrm_op.outlen = 0;
2580
2581 sess = get_session(oper.ses_id);
2582 if (!sess){
2583 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584 kfree(cop);
2585 kfree(jc);
2586 return -EINVAL;
2587 }
2588
2589 if (oper.do_cipher) {
2590 unsigned int cipher_outlen = 0;
2591 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592 if (!tc) {
2593 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594 err = -EINVAL;
2595 goto error_cleanup;
2596 }
2597 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598 ciph_tcfg.inject_ix = 0;
2599 ciph_tcfg.flags = 0;
2600 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602 kfree(cop);
2603 kfree(jc);
2604 return -EINVAL;
2605 }
2606 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607 if (oper.cipher_len % cblocklen) {
2608 kfree(cop);
2609 kfree(jc);
2610 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611 return -EINVAL;
2612 }
2613 cipher_outlen = oper.cipher_len;
2614 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615 if (oper.cipher_explicit) {
2616 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618 } else {
2619 cipher_outlen = oper.cipher_len - cblocklen;
2620 }
2621 } else {
2622 if (oper.cipher_explicit){
2623 kfree(cop);
2624 kfree(jc);
2625 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626 return -EINVAL;
2627 }
2628 }
2629 if (oper.cipher_outlen != cipher_outlen) {
2630 kfree(cop);
2631 kfree(jc);
2632 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633 return -EINVAL;
2634 }
2635
2636 if (oper.decrypt){
2637 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638 } else {
2639 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640 }
2641 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643 }
2644 if (oper.do_digest){
2645 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646 if (!tc) {
2647 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648 err = -EINVAL;
2649 goto error_cleanup;
2650 }
2651 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652 digest_result = kmalloc(digest_length, GFP_KERNEL);
2653 if (!digest_result) {
2654 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655 err = -EINVAL;
2656 goto error_cleanup;
2657 }
2658 DEBUG(memset(digest_result, 0xff, digest_length));
2659
2660 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661 digest_tcfg.inject_ix = 0;
2662 ciph_tcfg.inject_ix += digest_length;
2663 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665 err = -EINVAL;
2666 goto error_cleanup;
2667 }
2668
2669 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671 }
2672 if (oper.do_csum){
2673 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674 csum_tcfg.inject_ix = digest_length;
2675 ciph_tcfg.inject_ix += 2;
2676
2677 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679 kfree(cop);
2680 kfree(jc);
2681 return -EINVAL;
2682 }
2683
2684 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686 }
2687
2688 prev_ix = first_cfg_change_ix(&oper);
2689 if (prev_ix > oper.inlen) {
2690 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691 nooutpages = noinpages = 0;
2692 err = -EINVAL;
2693 goto error_cleanup;
2694 }
2695 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696
2697 /* Map user pages for in and out data of the operation. */
2698 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701 if (!inpages){
2702 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703 nooutpages = noinpages = 0;
2704 err = -ENOMEM;
2705 goto error_cleanup;
2706 }
2707 if (oper.do_cipher){
2708 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711 if (!outpages){
2712 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713 nooutpages = noinpages = 0;
2714 err = -ENOMEM;
2715 goto error_cleanup;
2716 }
2717 }
2718
2719 /* Acquire the mm page semaphore. */
2720 down_read(&current->mm->mmap_sem);
2721
2722 err = get_user_pages(current,
2723 current->mm,
2724 (unsigned long int)(oper.indata + prev_ix),
2725 noinpages,
2726 0, /* read access only for in data */
2727 0, /* no force */
2728 inpages,
2729 NULL);
2730
2731 if (err < 0) {
2732 up_read(&current->mm->mmap_sem);
2733 nooutpages = noinpages = 0;
2734 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2735 goto error_cleanup;
2736 }
2737 noinpages = err;
2738 if (oper.do_cipher){
2739 err = get_user_pages(current,
2740 current->mm,
2741 (unsigned long int)oper.cipher_outdata,
2742 nooutpages,
2743 1, /* write access for out data */
2744 0, /* no force */
2745 outpages,
2746 NULL);
2747 up_read(&current->mm->mmap_sem);
2748 if (err < 0) {
2749 nooutpages = 0;
2750 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2751 goto error_cleanup;
2752 }
2753 nooutpages = err;
2754 } else {
2755 up_read(&current->mm->mmap_sem);
2756 }
2757
2758 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2759 * csum output and splits when units are (dis-)connected. */
2760 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2761 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2762 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2763 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2764 err = -ENOMEM;
2765 goto error_cleanup;
2766 }
2767
2768 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2769 cop->tfrm_op.outlen = 0;
2770 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2771 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2772 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2773
2774 /* Setup the in iovecs. */
2775 cop->tfrm_op.incount = noinpages;
2776 if (noinpages > 1){
2777 size_t tmplen = cop->tfrm_op.inlen;
2778
2779 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2780 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2781 tmplen -= cop->tfrm_op.indata[0].iov_len;
2782 for (i = 1; i<noinpages; i++){
2783 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2784 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2785 tmplen -= PAGE_SIZE;
2786 }
2787 } else {
2788 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2789 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2790 }
2791
2792 iovlen = nooutpages + 6;
2793 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2794
2795 next_ix = next_cfg_change_ix(&oper, prev_ix);
2796 if (prev_ix == next_ix){
2797 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2798 err = -EINVAL; /* This should be impossible barring bugs. */
2799 goto error_cleanup;
2800 }
2801 while (prev_ix != next_ix){
2802 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2803 descs[desc_ix].cfg = NULL;
2804 descs[desc_ix].length = next_ix - prev_ix;
2805
2806 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2807 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2808 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2809 cipher_active = 1;
2810
2811 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2812 cipher_done = 1;
2813 dcfgs[dcfg_ix].last = 1;
2814 } else {
2815 dcfgs[dcfg_ix].last = 0;
2816 }
2817 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2818 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2819 ++dcfg_ix;
2820 }
2821 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2822 digest_active = 1;
2823 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2824 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2825 if (next_ix == (oper.digest_start + oper.digest_len)){
2826 assert(!digest_done);
2827 digest_done = 1;
2828 dcfgs[dcfg_ix].last = 1;
2829 } else {
2830 dcfgs[dcfg_ix].last = 0;
2831 }
2832 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2833 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2834 ++dcfg_ix;
2835 }
2836 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2837 csum_active = 1;
2838 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2839 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2840 if (next_ix == (oper.csum_start + oper.csum_len)){
2841 csum_done = 1;
2842 dcfgs[dcfg_ix].last = 1;
2843 } else {
2844 dcfgs[dcfg_ix].last = 0;
2845 }
2846 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2847 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2848 ++dcfg_ix;
2849 }
2850 if (!descs[desc_ix].cfg){
2851 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2852 err = -EINVAL;
2853 goto error_cleanup;
2854 }
2855 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2856 ++desc_ix;
2857 prev_ix = next_ix;
2858 next_ix = next_cfg_change_ix(&oper, prev_ix);
2859 }
2860 if (desc_ix > 0){
2861 descs[desc_ix-1].next = NULL;
2862 } else {
2863 descs[0].next = NULL;
2864 }
2865 if (oper.do_digest) {
2866 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2867 /* Add outdata iovec, length == <length of type of digest> */
2868 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2869 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2870 ++iovix;
2871 }
2872 if (oper.do_csum) {
2873 /* Add outdata iovec, length == 2, the length of csum. */
2874 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2875 /* Add outdata iovec, length == <length of type of digest> */
2876 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2877 cop->tfrm_op.outdata[iovix].iov_len = 2;
2878 ++iovix;
2879 }
2880 if (oper.do_cipher) {
2881 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2882 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2883 err = -ENOSYS; /* This should be impossible barring bugs. */
2884 goto error_cleanup;
2885 }
2886 }
2887 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2888 cop->tfrm_op.outcount = iovix;
2889 assert(iovix <= (nooutpages + 6));
2890
2891 cop->sid = oper.ses_id;
2892 cop->tfrm_op.desc = &descs[0];
2893
2894 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2895
2896 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2897 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2898 err = -EINVAL;
2899 goto error_cleanup;
2900 }
2901
2902 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2903
2904 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2905 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2906 if (!jc->processed){
2907 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2908 err = -EIO;
2909 goto error_cleanup;
2910 }
2911
2912 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2913 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2914 if (cop->operation_status == 0){
2915 if (oper.do_digest){
2916 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2917 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2918 if (0 != err){
2919 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2920 err = -EFAULT;
2921 goto error_cleanup;
2922 }
2923 }
2924 if (oper.do_csum){
2925 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2926 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2927 if (0 != err){
2928 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2929 err = -EFAULT;
2930 goto error_cleanup;
2931 }
2932 }
2933 err = 0;
2934 } else {
2935 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2936 err = cop->operation_status;
2937 }
2938
2939 error_cleanup:
2940 /* Release page caches. */
2941 for (i = 0; i < noinpages; i++){
2942 put_page(inpages[i]);
2943 }
2944 for (i = 0; i < nooutpages; i++){
2945 int spdl_err;
2946 /* Mark output pages dirty. */
2947 spdl_err = set_page_dirty_lock(outpages[i]);
2948 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2949 }
2950 for (i = 0; i < nooutpages; i++){
2951 put_page(outpages[i]);
2952 }
2953
2954 kfree(digest_result);
2955 kfree(inpages);
2956 kfree(outpages);
2957 if (cop){
2958 kfree(cop->tfrm_op.indata);
2959 kfree(cop->tfrm_op.outdata);
2960 kfree(cop);
2961 }
2962 kfree(jc);
2963
2964 DEBUG(print_lock_status());
2965
2966 return err;
2967 }
2968
2969
2970 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2971 {
2972 cryptocop_session_id sid;
2973 int err;
2974 struct cryptocop_private *dev;
2975 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2976 struct strcop_session_op sop;
2977 struct cryptocop_transform_init *tis = NULL;
2978 struct cryptocop_transform_init ti_cipher = {0};
2979 struct cryptocop_transform_init ti_digest = {0};
2980 struct cryptocop_transform_init ti_csum = {0};
2981
2982 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2983 return -EFAULT;
2984 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2985 if (err) return -EFAULT;
2986 if (sop.cipher != cryptocop_cipher_none) {
2987 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2988 }
2989 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2990
2991 DEBUG(printk("\tcipher:%d\n"
2992 "\tcipher_mode:%d\n"
2993 "\tdigest:%d\n"
2994 "\tcsum:%d\n",
2995 (int)sop.cipher,
2996 (int)sop.cmode,
2997 (int)sop.digest,
2998 (int)sop.csum));
2999
3000 if (sop.cipher != cryptocop_cipher_none){
3001 /* Init the cipher. */
3002 switch (sop.cipher){
3003 case cryptocop_cipher_des:
3004 ti_cipher.alg = cryptocop_alg_des;
3005 break;
3006 case cryptocop_cipher_3des:
3007 ti_cipher.alg = cryptocop_alg_3des;
3008 break;
3009 case cryptocop_cipher_aes:
3010 ti_cipher.alg = cryptocop_alg_aes;
3011 break;
3012 default:
3013 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3014 return -EINVAL;
3015 };
3016 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3017 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3018 ti_cipher.keylen = sop.keylen;
3019 switch (sop.cmode){
3020 case cryptocop_cipher_mode_cbc:
3021 case cryptocop_cipher_mode_ecb:
3022 ti_cipher.cipher_mode = sop.cmode;
3023 break;
3024 default:
3025 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3026 return -EINVAL;
3027 }
3028 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3029 switch (sop.des3_mode){
3030 case cryptocop_3des_eee:
3031 case cryptocop_3des_eed:
3032 case cryptocop_3des_ede:
3033 case cryptocop_3des_edd:
3034 case cryptocop_3des_dee:
3035 case cryptocop_3des_ded:
3036 case cryptocop_3des_dde:
3037 case cryptocop_3des_ddd:
3038 ti_cipher.tdes_mode = sop.des3_mode;
3039 break;
3040 default:
3041 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3042 return -EINVAL;
3043 }
3044 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3045 ti_cipher.next = tis;
3046 tis = &ti_cipher;
3047 } /* if (sop.cipher != cryptocop_cipher_none) */
3048 if (sop.digest != cryptocop_digest_none){
3049 DEBUG(printk("setting digest transform\n"));
3050 switch (sop.digest){
3051 case cryptocop_digest_md5:
3052 ti_digest.alg = cryptocop_alg_md5;
3053 break;
3054 case cryptocop_digest_sha1:
3055 ti_digest.alg = cryptocop_alg_sha1;
3056 break;
3057 default:
3058 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3059 return -EINVAL;
3060 }
3061 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3062 ti_digest.next = tis;
3063 tis = &ti_digest;
3064 } /* if (sop.digest != cryptocop_digest_none) */
3065 if (sop.csum != cryptocop_csum_none){
3066 DEBUG(printk("setting csum transform\n"));
3067 switch (sop.csum){
3068 case cryptocop_csum_le:
3069 case cryptocop_csum_be:
3070 ti_csum.csum_mode = sop.csum;
3071 break;
3072 default:
3073 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3074 return -EINVAL;
3075 }
3076 ti_csum.alg = cryptocop_alg_csum;
3077 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3078 ti_csum.next = tis;
3079 tis = &ti_csum;
3080 } /* (sop.csum != cryptocop_csum_none) */
3081 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3082 if (!dev){
3083 DEBUG_API(printk("create session, alloc dev\n"));
3084 return -ENOMEM;
3085 }
3086
3087 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3088 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3089
3090 if (err) {
3091 kfree(dev);
3092 return err;
3093 }
3094 sess_op->ses_id = sid;
3095 dev->sid = sid;
3096 dev->next = filp->private_data;
3097 filp->private_data = dev;
3098
3099 return 0;
3100 }
3101
3102 static long cryptocop_ioctl_unlocked(struct inode *inode,
3103 struct file *filp, unsigned int cmd, unsigned long arg)
3104 {
3105 int err = 0;
3106 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3107 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3108 return -ENOTTY;
3109 }
3110 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3111 return -ENOTTY;
3112 }
3113 /* Access check of the argument. Some commands, e.g. create session and process op,
3114 needs additional checks. Those are handled in the command handling functions. */
3115 if (_IOC_DIR(cmd) & _IOC_READ)
3116 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3117 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3118 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3119 if (err) return -EFAULT;
3120
3121 switch (cmd) {
3122 case CRYPTOCOP_IO_CREATE_SESSION:
3123 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3124 case CRYPTOCOP_IO_CLOSE_SESSION:
3125 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3126 case CRYPTOCOP_IO_PROCESS_OP:
3127 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3128 default:
3129 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3130 return -ENOTTY;
3131 }
3132 return 0;
3133 }
3134
3135 static long
3136 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3137 {
3138 long ret;
3139
3140 mutex_lock(&cryptocop_mutex);
3141 ret = cryptocop_ioctl_unlocked(file_inode(filp), filp, cmd, arg);
3142 mutex_unlock(&cryptocop_mutex);
3143
3144 return ret;
3145 }
3146
3147
3148 #ifdef LDEBUG
3149 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3150 {
3151 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3152 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3153 int i;
3154
3155 printk("print_dma_descriptors start\n");
3156
3157 printk("iop:\n");
3158 printk("\tsid: 0x%lld\n", iop->sid);
3159
3160 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3161 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3162 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3163 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3164
3165 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3166 printk("\tnext: 0x%p\n"
3167 "\tsaved_data: 0x%p\n"
3168 "\tsaved_data_buf: 0x%p\n",
3169 iop->ctx_out.next,
3170 iop->ctx_out.saved_data,
3171 iop->ctx_out.saved_data_buf);
3172
3173 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3174 printk("\tnext: 0x%p\n"
3175 "\tsaved_data: 0x%p\n"
3176 "\tsaved_data_buf: 0x%p\n",
3177 iop->ctx_in.next,
3178 iop->ctx_in.saved_data,
3179 iop->ctx_in.saved_data_buf);
3180
3181 i = 0;
3182 while (cdesc_out) {
3183 dma_descr_data *td;
3184 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3185 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3186 td = cdesc_out->dma_descr;
3187 printk("\n\tbuf: 0x%p\n"
3188 "\tafter: 0x%p\n"
3189 "\tmd: 0x%04x\n"
3190 "\tnext: 0x%p\n",
3191 td->buf,
3192 td->after,
3193 td->md,
3194 td->next);
3195 printk("flags:\n"
3196 "\twait:\t%d\n"
3197 "\teol:\t%d\n"
3198 "\touteop:\t%d\n"
3199 "\tineop:\t%d\n"
3200 "\tintr:\t%d\n",
3201 td->wait,
3202 td->eol,
3203 td->out_eop,
3204 td->in_eop,
3205 td->intr);
3206 cdesc_out = cdesc_out->next;
3207 i++;
3208 }
3209 i = 0;
3210 while (cdesc_in) {
3211 dma_descr_data *td;
3212 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3213 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3214 td = cdesc_in->dma_descr;
3215 printk("\n\tbuf: 0x%p\n"
3216 "\tafter: 0x%p\n"
3217 "\tmd: 0x%04x\n"
3218 "\tnext: 0x%p\n",
3219 td->buf,
3220 td->after,
3221 td->md,
3222 td->next);
3223 printk("flags:\n"
3224 "\twait:\t%d\n"
3225 "\teol:\t%d\n"
3226 "\touteop:\t%d\n"
3227 "\tineop:\t%d\n"
3228 "\tintr:\t%d\n",
3229 td->wait,
3230 td->eol,
3231 td->out_eop,
3232 td->in_eop,
3233 td->intr);
3234 cdesc_in = cdesc_in->next;
3235 i++;
3236 }
3237
3238 printk("print_dma_descriptors end\n");
3239 }
3240
3241
3242 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3243 {
3244 printk("print_strcop_crypto_op, 0x%p\n", cop);
3245
3246 /* Indata. */
3247 printk("indata=0x%p\n"
3248 "inlen=%d\n"
3249 "do_cipher=%d\n"
3250 "decrypt=%d\n"
3251 "cipher_explicit=%d\n"
3252 "cipher_start=%d\n"
3253 "cipher_len=%d\n"
3254 "outdata=0x%p\n"
3255 "outlen=%d\n",
3256 cop->indata,
3257 cop->inlen,
3258 cop->do_cipher,
3259 cop->decrypt,
3260 cop->cipher_explicit,
3261 cop->cipher_start,
3262 cop->cipher_len,
3263 cop->cipher_outdata,
3264 cop->cipher_outlen);
3265
3266 printk("do_digest=%d\n"
3267 "digest_start=%d\n"
3268 "digest_len=%d\n",
3269 cop->do_digest,
3270 cop->digest_start,
3271 cop->digest_len);
3272
3273 printk("do_csum=%d\n"
3274 "csum_start=%d\n"
3275 "csum_len=%d\n",
3276 cop->do_csum,
3277 cop->csum_start,
3278 cop->csum_len);
3279 }
3280
3281 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3282 {
3283 struct cryptocop_desc *d;
3284 struct cryptocop_tfrm_cfg *tc;
3285 struct cryptocop_desc_cfg *dc;
3286 int i;
3287
3288 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3289 printk("sid: %lld\n", cop->sid);
3290 printk("operation_status=%d\n"
3291 "use_dmalists=%d\n"
3292 "in_interrupt=%d\n"
3293 "fast_callback=%d\n",
3294 cop->operation_status,
3295 cop->use_dmalists,
3296 cop->in_interrupt,
3297 cop->fast_callback);
3298
3299 if (cop->use_dmalists){
3300 print_user_dma_lists(&cop->list_op);
3301 } else {
3302 printk("cop->tfrm_op\n"
3303 "tfrm_cfg=0x%p\n"
3304 "desc=0x%p\n"
3305 "indata=0x%p\n"
3306 "incount=%d\n"
3307 "inlen=%d\n"
3308 "outdata=0x%p\n"
3309 "outcount=%d\n"
3310 "outlen=%d\n\n",
3311 cop->tfrm_op.tfrm_cfg,
3312 cop->tfrm_op.desc,
3313 cop->tfrm_op.indata,
3314 cop->tfrm_op.incount,
3315 cop->tfrm_op.inlen,
3316 cop->tfrm_op.outdata,
3317 cop->tfrm_op.outcount,
3318 cop->tfrm_op.outlen);
3319
3320 tc = cop->tfrm_op.tfrm_cfg;
3321 while (tc){
3322 printk("tfrm_cfg, 0x%p\n"
3323 "tid=%d\n"
3324 "flags=%d\n"
3325 "inject_ix=%d\n"
3326 "next=0x%p\n",
3327 tc,
3328 tc->tid,
3329 tc->flags,
3330 tc->inject_ix,
3331 tc->next);
3332 tc = tc->next;
3333 }
3334 d = cop->tfrm_op.desc;
3335 while (d){
3336 printk("\n======================desc, 0x%p\n"
3337 "length=%d\n"
3338 "cfg=0x%p\n"
3339 "next=0x%p\n",
3340 d,
3341 d->length,
3342 d->cfg,
3343 d->next);
3344 dc = d->cfg;
3345 while (dc){
3346 printk("=========desc_cfg, 0x%p\n"
3347 "tid=%d\n"
3348 "src=%d\n"
3349 "last=%d\n"
3350 "next=0x%p\n",
3351 dc,
3352 dc->tid,
3353 dc->src,
3354 dc->last,
3355 dc->next);
3356 dc = dc->next;
3357 }
3358 d = d->next;
3359 }
3360 printk("\n====iniov\n");
3361 for (i = 0; i < cop->tfrm_op.incount; i++){
3362 printk("indata[%d]\n"
3363 "base=0x%p\n"
3364 "len=%d\n",
3365 i,
3366 cop->tfrm_op.indata[i].iov_base,
3367 cop->tfrm_op.indata[i].iov_len);
3368 }
3369 printk("\n====outiov\n");
3370 for (i = 0; i < cop->tfrm_op.outcount; i++){
3371 printk("outdata[%d]\n"
3372 "base=0x%p\n"
3373 "len=%d\n",
3374 i,
3375 cop->tfrm_op.outdata[i].iov_base,
3376 cop->tfrm_op.outdata[i].iov_len);
3377 }
3378 }
3379 printk("------------end print_cryptocop_operation\n");
3380 }
3381
3382
3383 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3384 {
3385 dma_descr_data *dd;
3386 int i;
3387
3388 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3389
3390 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3391 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3392
3393 printk("##############outlist\n");
3394 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3395 i = 0;
3396 while (dd != NULL) {
3397 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3398 printk("\n\tbuf: 0x%p\n"
3399 "\tafter: 0x%p\n"
3400 "\tmd: 0x%04x\n"
3401 "\tnext: 0x%p\n",
3402 dd->buf,
3403 dd->after,
3404 dd->md,
3405 dd->next);
3406 printk("flags:\n"
3407 "\twait:\t%d\n"
3408 "\teol:\t%d\n"
3409 "\touteop:\t%d\n"
3410 "\tineop:\t%d\n"
3411 "\tintr:\t%d\n",
3412 dd->wait,
3413 dd->eol,
3414 dd->out_eop,
3415 dd->in_eop,
3416 dd->intr);
3417 if (dd->eol)
3418 dd = NULL;
3419 else
3420 dd = phys_to_virt((unsigned long int)dd->next);
3421 ++i;
3422 }
3423
3424 printk("##############inlist\n");
3425 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3426 i = 0;
3427 while (dd != NULL) {
3428 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3429 printk("\n\tbuf: 0x%p\n"
3430 "\tafter: 0x%p\n"
3431 "\tmd: 0x%04x\n"
3432 "\tnext: 0x%p\n",
3433 dd->buf,
3434 dd->after,
3435 dd->md,
3436 dd->next);
3437 printk("flags:\n"
3438 "\twait:\t%d\n"
3439 "\teol:\t%d\n"
3440 "\touteop:\t%d\n"
3441 "\tineop:\t%d\n"
3442 "\tintr:\t%d\n",
3443 dd->wait,
3444 dd->eol,
3445 dd->out_eop,
3446 dd->in_eop,
3447 dd->intr);
3448 if (dd->eol)
3449 dd = NULL;
3450 else
3451 dd = phys_to_virt((unsigned long int)dd->next);
3452 ++i;
3453 }
3454 }
3455
3456
3457 static void print_lock_status(void)
3458 {
3459 printk("**********************print_lock_status\n");
3460 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3461 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3462 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3463 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3464 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3465 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3466 }
3467 #endif /* LDEBUG */
3468
3469
3470 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3471
3472 static int init_stream_coprocessor(void)
3473 {
3474 int err;
3475 int i;
3476 static int initialized = 0;
3477
3478 if (initialized)
3479 return 0;
3480
3481 initialized = 1;
3482
3483 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3484
3485 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3486 if (err < 0) {
3487 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3488 return err;
3489 }
3490
3491 err = init_cryptocop();
3492 if (err) {
3493 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3494 return err;
3495 }
3496 err = cryptocop_job_queue_init();
3497 if (err) {
3498 release_cryptocop();
3499 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3500 return err;
3501 }
3502 /* Init the descriptor pool. */
3503 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3504 descr_pool[i].from_pool = 1;
3505 descr_pool[i].next = &descr_pool[i + 1];
3506 }
3507 descr_pool[i].from_pool = 1;
3508 descr_pool[i].next = NULL;
3509 descr_pool_free_list = &descr_pool[0];
3510 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3511
3512 spin_lock_init(&cryptocop_completed_jobs_lock);
3513 spin_lock_init(&cryptocop_job_queue_lock);
3514 spin_lock_init(&descr_pool_lock);
3515 spin_lock_init(&cryptocop_sessions_lock);
3516 spin_lock_init(&running_job_lock);
3517 spin_lock_init(&cryptocop_process_lock);
3518
3519 cryptocop_sessions = NULL;
3520 next_sid = 1;
3521
3522 cryptocop_running_job = NULL;
3523
3524 printk("stream co-processor: init done.\n");
3525 return 0;
3526 }
3527
3528 static void __exit exit_stream_coprocessor(void)
3529 {
3530 release_cryptocop();
3531 cryptocop_job_queue_close();
3532 }
3533
3534 module_init(init_stream_coprocessor);
3535 module_exit(exit_stream_coprocessor);
3536
Linux with added FPC-III support