?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'perf-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel...
[cris-mirror.git] / arch / cris / arch-v32 / drivers / cryptocop.c
blob2081d8b45f061b37e117f46f66bc03c5b481fe0f
1 /*
2 * Stream co-processor driver for the ETRAX FS
3 *
4 * Copyright (C) 2003-2007 Axis Communications AB
5 */
6
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <linux/atomic.h>
20
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50
51 #define DESCR_ALLOC_PAD (31)
52
53 struct cryptocop_dma_desc {
54 char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 dma_descr_data *dma_descr;
56
57 unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58
59 unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 struct cryptocop_dma_desc *next;
61 };
62
63
64 struct cryptocop_int_operation{
65 void *alloc_ptr;
66 cryptocop_session_id sid;
67
68 dma_descr_context ctx_out;
69 dma_descr_context ctx_in;
70
71 /* DMA descriptors allocated by driver. */
72 struct cryptocop_dma_desc *cdesc_out;
73 struct cryptocop_dma_desc *cdesc_in;
74
75 /* Strcop config to use. */
76 cryptocop_3des_mode tdes_mode;
77 cryptocop_csum_type csum_mode;
78
79 /* DMA descrs provided by consumer. */
80 dma_descr_data *ddesc_out;
81 dma_descr_data *ddesc_in;
82 };
83
84
85 struct cryptocop_tfrm_ctx {
86 cryptocop_tfrm_id tid;
87 unsigned int blocklength;
88
89 unsigned int start_ix;
90
91 struct cryptocop_tfrm_cfg *tcfg;
92 struct cryptocop_transform_ctx *tctx;
93
94 unsigned char previous_src;
95 unsigned char current_src;
96
97 /* Values to use in metadata out. */
98 unsigned char hash_conf;
99 unsigned char hash_mode;
100 unsigned char ciph_conf;
101 unsigned char cbcmode;
102 unsigned char decrypt;
103
104 unsigned int requires_padding:1;
105 unsigned int strict_block_length:1;
106 unsigned int active:1;
107 unsigned int done:1;
108 size_t consumed;
109 size_t produced;
110
111 /* Pad (input) descriptors to put in the DMA out list when the transform
112 * output is put on the DMA in list. */
113 struct cryptocop_dma_desc *pad_descs;
114
115 struct cryptocop_tfrm_ctx *prev_src;
116 struct cryptocop_tfrm_ctx *curr_src;
117
118 /* Mapping to HW. */
119 unsigned char unit_no;
120 };
121
122
123 struct cryptocop_private{
124 cryptocop_session_id sid;
125 struct cryptocop_private *next;
126 };
127
128 /* Session list. */
129
130 struct cryptocop_transform_ctx{
131 struct cryptocop_transform_init init;
132 unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 unsigned int dec_key_set:1;
134
135 struct cryptocop_transform_ctx *next;
136 };
137
138
139 struct cryptocop_session{
140 cryptocop_session_id sid;
141
142 struct cryptocop_transform_ctx *tfrm_ctx;
143
144 struct cryptocop_session *next;
145 };
146
147 /* Priority levels for jobs sent to the cryptocop. Checksum operations from
148 kernel have highest priority since TCPIP stack processing must not
149 be a bottleneck. */
150 typedef enum {
151 cryptocop_prio_kernel_csum = 0,
152 cryptocop_prio_kernel = 1,
153 cryptocop_prio_user = 2,
154 cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156
157 struct cryptocop_prio_queue{
158 struct list_head jobs;
159 cryptocop_queue_priority prio;
160 };
161
162 struct cryptocop_prio_job{
163 struct list_head node;
164 cryptocop_queue_priority prio;
165
166 struct cryptocop_operation *oper;
167 struct cryptocop_int_operation *iop;
168 };
169
170 struct ioctl_job_cb_ctx {
171 unsigned int processed:1;
172 };
173
174
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197 of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206 * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211
212
213 /** Local functions. **/
214
215 static int cryptocop_open(struct inode *, struct file *);
216
217 static int cryptocop_release(struct inode *, struct file *);
218
219 static long cryptocop_ioctl(struct file *file,
220 unsigned int cmd, unsigned long arg);
221
222 static void cryptocop_start_job(void);
223
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength);
243
244 static int init_stream_coprocessor(void);
245
246 static void __exit exit_stream_coprocessor(void);
247
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263
264
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH (8)
267 #define AES_BLOCK_LENGTH (16)
268 #define MD5_BLOCK_LENGTH (64)
269 #define SHA1_BLOCK_LENGTH (64)
270 #define CSUM_BLOCK_LENGTH (2)
271 #define MD5_STATE_LENGTH (16)
272 #define SHA1_STATE_LENGTH (20)
273
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR (254)
276 #define CRYPTOCOP_MINOR (0)
277
278
279
280 const struct file_operations cryptocop_fops = {
281 .owner = THIS_MODULE,
282 .open = cryptocop_open,
283 .release = cryptocop_release,
284 .unlocked_ioctl = cryptocop_ioctl,
285 .llseek = noop_llseek,
286 };
287
288
289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290 {
291 DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
292 kfree(cdesc->free_buf);
293
294 if (cdesc->from_pool) {
295 unsigned long int flags;
296 spin_lock_irqsave(&descr_pool_lock, flags);
297 cdesc->next = descr_pool_free_list;
298 descr_pool_free_list = cdesc;
299 ++descr_pool_no_free;
300 spin_unlock_irqrestore(&descr_pool_lock, flags);
301 } else {
302 kfree(cdesc);
303 }
304 }
305
306
307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308 {
309 int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 struct cryptocop_dma_desc *cdesc;
311
312 if (use_pool) {
313 unsigned long int flags;
314 spin_lock_irqsave(&descr_pool_lock, flags);
315 if (!descr_pool_free_list) {
316 spin_unlock_irqrestore(&descr_pool_lock, flags);
317 DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 return NULL;
319 }
320 cdesc = descr_pool_free_list;
321 descr_pool_free_list = descr_pool_free_list->next;
322 --descr_pool_no_free;
323 spin_unlock_irqrestore(&descr_pool_lock, flags);
324 cdesc->from_pool = 1;
325 } else {
326 cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 if (!cdesc) {
328 DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 return NULL;
330 }
331 cdesc->from_pool = 0;
332 }
333 cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334
335 cdesc->next = NULL;
336
337 cdesc->free_buf = NULL;
338 cdesc->dma_descr->out_eop = 0;
339 cdesc->dma_descr->in_eop = 0;
340 cdesc->dma_descr->intr = 0;
341 cdesc->dma_descr->eol = 0;
342 cdesc->dma_descr->wait = 0;
343 cdesc->dma_descr->buf = NULL;
344 cdesc->dma_descr->after = NULL;
345
346 DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 return cdesc;
348 }
349
350
351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352 {
353 DEBUG(printk("setup_descr_chain: entering\n"));
354 while (cd) {
355 if (cd->next) {
356 cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 } else {
358 cd->dma_descr->next = NULL;
359 }
360 cd = cd->next;
361 }
362 DEBUG(printk("setup_descr_chain: exit\n"));
363 }
364
365
366 /* Create a pad descriptor for the transform.
367 * Return -1 for error, 0 if pad created. */
368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369 {
370 struct cryptocop_dma_desc *cdesc = NULL;
371 int error = 0;
372 struct strcop_meta_out mo = {
373 .ciphsel = src_none,
374 .hashsel = src_none,
375 .csumsel = src_none
376 };
377 char *pad;
378 size_t plen;
379
380 DEBUG(printk("create_pad_descriptor: start.\n"));
381 /* Setup pad descriptor. */
382
383 DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 cdesc = alloc_cdesc(alloc_flag);
385 if (!cdesc){
386 DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 goto error_cleanup;
388 }
389 switch (tc->unit_no) {
390 case src_md5:
391 error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 if (error){
393 DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 goto error_cleanup;
395 }
396 cdesc->free_buf = pad;
397 mo.hashsel = src_dma;
398 mo.hashconf = tc->hash_conf;
399 mo.hashmode = tc->hash_mode;
400 break;
401 case src_sha1:
402 error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 if (error){
404 DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 goto error_cleanup;
406 }
407 cdesc->free_buf = pad;
408 mo.hashsel = src_dma;
409 mo.hashconf = tc->hash_conf;
410 mo.hashmode = tc->hash_mode;
411 break;
412 case src_csum:
413 if (tc->consumed % tc->blocklength){
414 pad = (char*)csum_zero_pad;
415 plen = 1;
416 } else {
417 pad = (char*)cdesc; /* Use any pointer. */
418 plen = 0;
419 }
420 mo.csumsel = src_dma;
421 break;
422 }
423 cdesc->dma_descr->wait = 1;
424 cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed. EOP is ok here since the padded unit is the only one active. */
425 cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427
428 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 *pad_desc = cdesc;
430
431 return 0;
432
433 error_cleanup:
434 if (cdesc) free_cdesc(cdesc);
435 return -1;
436 }
437
438
439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440 {
441 struct cryptocop_dma_desc *key_desc = alloc_cdesc(alloc_flag);
442 struct strcop_meta_out mo = {0};
443
444 DEBUG(printk("setup_key_dl_desc\n"));
445
446 if (!key_desc) {
447 DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 return -ENOMEM;
449 }
450
451 /* Download key. */
452 if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 /* Precook the AES decrypt key. */
454 if (!tc->tctx->dec_key_set){
455 get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 tc->tctx->dec_key_set = 1;
457 }
458 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 } else {
461 key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 }
464 /* Setup metadata. */
465 mo.dlkey = 1;
466 switch (tc->tctx->init.keylen) {
467 case 64:
468 mo.decrypt = 0;
469 mo.hashmode = 0;
470 break;
471 case 128:
472 mo.decrypt = 0;
473 mo.hashmode = 1;
474 break;
475 case 192:
476 mo.decrypt = 1;
477 mo.hashmode = 0;
478 break;
479 case 256:
480 mo.decrypt = 1;
481 mo.hashmode = 1;
482 break;
483 default:
484 break;
485 }
486 mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488
489 key_desc->dma_descr->out_eop = 1;
490 key_desc->dma_descr->wait = 1;
491 key_desc->dma_descr->intr = 0;
492
493 *kd = key_desc;
494 return 0;
495 }
496
497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498 {
499 struct cryptocop_dma_desc *iv_desc = alloc_cdesc(alloc_flag);
500 struct strcop_meta_out mo = {0};
501
502 DEBUG(printk("setup_cipher_iv_desc\n"));
503
504 if (!iv_desc) {
505 DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 return -ENOMEM;
507 }
508 /* Download IV. */
509 iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511
512 /* Setup metadata. */
513 mo.hashsel = mo.csumsel = src_none;
514 mo.ciphsel = src_dma;
515 mo.ciphconf = tc->ciph_conf;
516 mo.cbcmode = tc->cbcmode;
517
518 iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519
520 iv_desc->dma_descr->out_eop = 0;
521 iv_desc->dma_descr->wait = 1;
522 iv_desc->dma_descr->intr = 0;
523
524 *id = iv_desc;
525 return 0;
526 }
527
528 /* Map the output length of the transform to operation output starting on the inject index. */
529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530 {
531 int err = 0;
532 struct cryptocop_dma_desc head = {0};
533 struct cryptocop_dma_desc *outdesc = &head;
534 size_t iov_offset = 0;
535 size_t out_ix = 0;
536 int outiov_ix = 0;
537 struct strcop_meta_in mi = {0};
538
539 size_t out_length = tc->produced;
540 int rem_length;
541 int dlength;
542
543 assert(out_length != 0);
544 if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 return -EINVAL;
547 }
548 /* Traverse the out iovec until the result inject index is reached. */
549 while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 outiov_ix++;
552 }
553 if (outiov_ix >= operation->tfrm_op.outcount){
554 DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 return -EINVAL;
556 }
557 iov_offset = tc->tcfg->inject_ix - out_ix;
558 mi.dmasel = tc->unit_no;
559
560 /* Setup the output descriptors. */
561 while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 outdesc->next = alloc_cdesc(alloc_flag);
563 if (!outdesc->next) {
564 DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 err = -ENOMEM;
566 goto error_cleanup;
567 }
568 outdesc = outdesc->next;
569 rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 dlength = (out_length < rem_length) ? out_length : rem_length;
571
572 DEBUG(printk("create_input_descriptors:\n"
573 "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 "outcount=%d, outiov_ix=%d\n",
576 outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577
578 outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581
582 out_length -= dlength;
583 iov_offset += dlength;
584 if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 iov_offset = 0;
586 ++outiov_ix;
587 }
588 }
589 if (out_length > 0){
590 DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 err = -EINVAL;
592 goto error_cleanup;
593 }
594 /* Set sync in last descriptor. */
595 mi.sync = 1;
596 outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597
598 *id = head.next;
599 return 0;
600
601 error_cleanup:
602 while (head.next) {
603 outdesc = head.next->next;
604 free_cdesc(head.next);
605 head.next = outdesc;
606 }
607 return err;
608 }
609
610
611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612 {
613 while (desc_len != 0) {
614 struct cryptocop_dma_desc *cdesc;
615 int rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 int dlength = (desc_len < rem_length) ? desc_len : rem_length;
617
618 cdesc = alloc_cdesc(alloc_flag);
619 if (!cdesc) {
620 DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 return -ENOMEM;
622 }
623 (*current_out_cdesc)->next = cdesc;
624 (*current_out_cdesc) = cdesc;
625
626 cdesc->free_buf = NULL;
627
628 cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630
631 assert(desc_len >= dlength);
632 desc_len -= dlength;
633 *iniov_offset += dlength;
634 if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 *iniov_offset = 0;
636 ++(*iniov_ix);
637 if (*iniov_ix > operation->tfrm_op.incount) {
638 DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 return -EINVAL;
640 }
641 }
642 cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 } /* while (desc_len != 0) */
644 /* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 (*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646
647 return 0;
648 }
649
650
651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652 {
653 DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 if (tc->tcfg) {
655 int failed = 0;
656 struct cryptocop_dma_desc *idescs = NULL;
657 DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 if (tc->pad_descs) {
659 DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 while (tc->pad_descs) {
661 DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 (*current_out_cdesc)->next = tc->pad_descs;
663 tc->pad_descs = tc->pad_descs->next;
664 (*current_out_cdesc) = (*current_out_cdesc)->next;
665 }
666 }
667
668 /* Setup and append output descriptors to DMA in list. */
669 if (tc->unit_no == src_dma){
670 /* mem2mem. Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 unsigned int start_ix = tc->start_ix;
673 while (start_ix){
674 unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 (*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 if (!(*current_in_cdesc)->next){
677 DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 return -ENOMEM;
679 }
680 (*current_in_cdesc) = (*current_in_cdesc)->next;
681 (*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 (*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 start_ix -= desclen;
685 }
686 mi.sync = 1;
687 (*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 }
689
690 failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 if (failed){
692 DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 return failed;
694 }
695 DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 while (idescs) {
697 DEBUG(printk("append descriptor 0x%p\n", idescs));
698 (*current_in_cdesc)->next = idescs;
699 idescs = idescs->next;
700 (*current_in_cdesc) = (*current_in_cdesc)->next;
701 }
702 }
703 return 0;
704 }
705
706
707
708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709 {
710 struct cryptocop_session *sess;
711 struct cryptocop_transform_ctx *tctx;
712
713 struct cryptocop_tfrm_ctx digest_ctx = {
714 .previous_src = src_none,
715 .current_src = src_none,
716 .start_ix = 0,
717 .requires_padding = 1,
718 .strict_block_length = 0,
719 .hash_conf = 0,
720 .hash_mode = 0,
721 .ciph_conf = 0,
722 .cbcmode = 0,
723 .decrypt = 0,
724 .consumed = 0,
725 .produced = 0,
726 .pad_descs = NULL,
727 .active = 0,
728 .done = 0,
729 .prev_src = NULL,
730 .curr_src = NULL,
731 .tcfg = NULL};
732 struct cryptocop_tfrm_ctx cipher_ctx = {
733 .previous_src = src_none,
734 .current_src = src_none,
735 .start_ix = 0,
736 .requires_padding = 0,
737 .strict_block_length = 1,
738 .hash_conf = 0,
739 .hash_mode = 0,
740 .ciph_conf = 0,
741 .cbcmode = 0,
742 .decrypt = 0,
743 .consumed = 0,
744 .produced = 0,
745 .pad_descs = NULL,
746 .active = 0,
747 .done = 0,
748 .prev_src = NULL,
749 .curr_src = NULL,
750 .tcfg = NULL};
751 struct cryptocop_tfrm_ctx csum_ctx = {
752 .previous_src = src_none,
753 .current_src = src_none,
754 .start_ix = 0,
755 .blocklength = 2,
756 .requires_padding = 1,
757 .strict_block_length = 0,
758 .hash_conf = 0,
759 .hash_mode = 0,
760 .ciph_conf = 0,
761 .cbcmode = 0,
762 .decrypt = 0,
763 .consumed = 0,
764 .produced = 0,
765 .pad_descs = NULL,
766 .active = 0,
767 .done = 0,
768 .tcfg = NULL,
769 .prev_src = NULL,
770 .curr_src = NULL,
771 .unit_no = src_csum};
772 struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773
774 unsigned int indata_ix = 0;
775
776 /* iovec accounting. */
777 int iniov_ix = 0;
778 int iniov_offset = 0;
779
780 /* Operation descriptor cfg traversal pointer. */
781 struct cryptocop_desc *odsc;
782
783 int failed = 0;
784 /* List heads for allocated descriptors. */
785 struct cryptocop_dma_desc out_cdesc_head = {0};
786 struct cryptocop_dma_desc in_cdesc_head = {0};
787
788 struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790
791 struct cryptocop_tfrm_ctx *output_tc = NULL;
792 void *iop_alloc_ptr;
793
794 assert(operation != NULL);
795 assert(int_op != NULL);
796
797 DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 DEBUG(print_cryptocop_operation(operation));
799
800 sess = get_session(operation->sid);
801 if (!sess) {
802 DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 failed = -EINVAL;
804 goto error_cleanup;
805 }
806 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 if (!iop_alloc_ptr) {
808 DEBUG_API(printk("cryptocop_setup_dma_list: kmalloc cryptocop_int_operation\n"));
809 failed = -ENOMEM;
810 goto error_cleanup;
811 }
812 (*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 (*int_op)->alloc_ptr = iop_alloc_ptr;
815 DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816
817 (*int_op)->sid = operation->sid;
818 (*int_op)->cdesc_out = NULL;
819 (*int_op)->cdesc_in = NULL;
820 (*int_op)->tdes_mode = cryptocop_3des_ede;
821 (*int_op)->csum_mode = cryptocop_csum_le;
822 (*int_op)->ddesc_out = NULL;
823 (*int_op)->ddesc_in = NULL;
824
825 /* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 if (!tcfg) {
827 DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 failed = -EINVAL;
829 goto error_cleanup;
830 }
831 while (tcfg) {
832 tctx = get_transform_ctx(sess, tcfg->tid);
833 if (!tctx) {
834 DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 failed = -EINVAL;
836 goto error_cleanup;
837 }
838 if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 failed = -EINVAL;
841 goto error_cleanup;
842 }
843 switch (tctx->init.alg){
844 case cryptocop_alg_mem2mem:
845 if (cipher_ctx.tcfg != NULL){
846 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 failed = -EINVAL;
848 goto error_cleanup;
849 }
850 /* mem2mem is handled as a NULL cipher. */
851 cipher_ctx.cbcmode = 0;
852 cipher_ctx.decrypt = 0;
853 cipher_ctx.blocklength = 1;
854 cipher_ctx.ciph_conf = 0;
855 cipher_ctx.unit_no = src_dma;
856 cipher_ctx.tcfg = tcfg;
857 cipher_ctx.tctx = tctx;
858 break;
859 case cryptocop_alg_des:
860 case cryptocop_alg_3des:
861 case cryptocop_alg_aes:
862 /* cipher */
863 if (cipher_ctx.tcfg != NULL){
864 DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 failed = -EINVAL;
866 goto error_cleanup;
867 }
868 cipher_ctx.tcfg = tcfg;
869 cipher_ctx.tctx = tctx;
870 if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 cipher_ctx.decrypt = 1;
872 }
873 switch (tctx->init.cipher_mode) {
874 case cryptocop_cipher_mode_ecb:
875 cipher_ctx.cbcmode = 0;
876 break;
877 case cryptocop_cipher_mode_cbc:
878 cipher_ctx.cbcmode = 1;
879 break;
880 default:
881 DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 failed = -EINVAL;
883 goto error_cleanup;
884 }
885 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 switch (tctx->init.alg){
887 case cryptocop_alg_des:
888 cipher_ctx.ciph_conf = 0;
889 cipher_ctx.unit_no = src_des;
890 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 break;
892 case cryptocop_alg_3des:
893 cipher_ctx.ciph_conf = 1;
894 cipher_ctx.unit_no = src_des;
895 cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 break;
897 case cryptocop_alg_aes:
898 cipher_ctx.ciph_conf = 2;
899 cipher_ctx.unit_no = src_aes;
900 cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 break;
902 default:
903 panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 }
905 (*int_op)->tdes_mode = tctx->init.tdes_mode;
906 break;
907 case cryptocop_alg_md5:
908 case cryptocop_alg_sha1:
909 /* digest */
910 if (digest_ctx.tcfg != NULL){
911 DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 failed = -EINVAL;
913 goto error_cleanup;
914 }
915 digest_ctx.tcfg = tcfg;
916 digest_ctx.tctx = tctx;
917 digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 switch (tctx->init.alg){
919 case cryptocop_alg_md5:
920 digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 digest_ctx.unit_no = src_md5;
922 digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 break;
924 case cryptocop_alg_sha1:
925 digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 digest_ctx.unit_no = src_sha1;
927 digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 break;
929 default:
930 panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 }
932 break;
933 case cryptocop_alg_csum:
934 /* digest */
935 if (csum_ctx.tcfg != NULL){
936 DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 failed = -EINVAL;
938 goto error_cleanup;
939 }
940 (*int_op)->csum_mode = tctx->init.csum_mode;
941 csum_ctx.tcfg = tcfg;
942 csum_ctx.tctx = tctx;
943 break;
944 default:
945 /* no algorithm. */
946 DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 failed = -EINVAL;
948 goto error_cleanup;
949 }
950 tcfg = tcfg->next;
951 }
952 /* Download key if a cipher is used. */
953 if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 struct cryptocop_dma_desc *key_desc = NULL;
955
956 failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 if (failed) {
958 DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 goto error_cleanup;
960 }
961 current_out_cdesc->next = key_desc;
962 current_out_cdesc = key_desc;
963 indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964
965 /* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 struct cryptocop_dma_desc *iv_desc = NULL;
968
969 DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970
971 failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 if (failed) {
973 DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 goto error_cleanup;
975 }
976 current_out_cdesc->next = iv_desc;
977 current_out_cdesc = iv_desc;
978 indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 }
980 }
981
982 /* Process descriptors. */
983 odsc = operation->tfrm_op.desc;
984 while (odsc) {
985 struct cryptocop_desc_cfg *dcfg = odsc->cfg;
986 struct strcop_meta_out meta_out = {0};
987 size_t desc_len = odsc->length;
988 int active_count, eop_needed_count;
989
990 output_tc = NULL;
991
992 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993
994 while (dcfg) {
995 struct cryptocop_tfrm_ctx *tc = NULL;
996
997 DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 /* Get the local context for the transform and mark it as the output unit if it produces output. */
999 if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 tc = &digest_ctx;
1001 } else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 tc = &cipher_ctx;
1003 } else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 tc = &csum_ctx;
1005 }
1006 if (!tc) {
1007 DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 failed = -EINVAL;
1009 goto error_cleanup;
1010 }
1011 if (tc->done) {
1012 DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 failed = -EINVAL;
1014 goto error_cleanup;
1015 }
1016 if (!tc->active) {
1017 tc->start_ix = indata_ix;
1018 tc->active = 1;
1019 }
1020
1021 tc->previous_src = tc->current_src;
1022 tc->prev_src = tc->curr_src;
1023 /* Map source unit id to DMA source config. */
1024 switch (dcfg->src){
1025 case cryptocop_source_dma:
1026 tc->current_src = src_dma;
1027 break;
1028 case cryptocop_source_des:
1029 tc->current_src = src_des;
1030 break;
1031 case cryptocop_source_3des:
1032 tc->current_src = src_des;
1033 break;
1034 case cryptocop_source_aes:
1035 tc->current_src = src_aes;
1036 break;
1037 case cryptocop_source_md5:
1038 case cryptocop_source_sha1:
1039 case cryptocop_source_csum:
1040 case cryptocop_source_none:
1041 default:
1042 /* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 */
1044 DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 failed = -EINVAL;
1046 goto error_cleanup;
1047 }
1048 if (tc->current_src != src_dma) {
1049 /* Find the unit we are sourcing from. */
1050 if (digest_ctx.unit_no == tc->current_src){
1051 tc->curr_src = &digest_ctx;
1052 } else if (cipher_ctx.unit_no == tc->current_src){
1053 tc->curr_src = &cipher_ctx;
1054 } else if (csum_ctx.unit_no == tc->current_src){
1055 tc->curr_src = &csum_ctx;
1056 }
1057 if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 failed = -EINVAL;
1060 goto error_cleanup;
1061 }
1062 } else {
1063 tc->curr_src = NULL;
1064 }
1065
1066 /* Detect source switch. */
1067 DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 if (tc->active && (tc->current_src != tc->previous_src)) {
1069 /* Only allow source switch when both the old source unit and the new one have
1070 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 * transform blocklength). */
1072 /* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 {
1076 DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 failed = -EINVAL;
1078 goto error_cleanup;
1079 }
1080 }
1081 /* Detect unit deactivation. */
1082 if (dcfg->last) {
1083 /* Length check of this is handled below. */
1084 tc->done = 1;
1085 }
1086 dcfg = dcfg->next;
1087 } /* while (dcfg) */
1088 DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089
1090 if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 failed = -EINVAL;
1093 goto error_cleanup;
1094 }
1095 if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 failed = -EINVAL;
1098 goto error_cleanup;
1099 }
1100 if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 failed = -EINVAL;
1103 goto error_cleanup;
1104 }
1105
1106 /* Update consumed and produced lengths.
1107
1108 The consumed length accounting here is actually cheating. If a unit source from DMA (or any
1109 other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 block processing unit, i.e. a cipher, it will be temporarily incorrect at some times. However
1111 since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 unit has processed an exact multiple of its block length the end result will be correct.
1113 Beware that if the source change restriction change this code will need to be (much) reworked.
1114 */
1115 DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116
1117 if (csum_ctx.active) {
1118 csum_ctx.consumed += desc_len;
1119 if (csum_ctx.done) {
1120 csum_ctx.produced = 2;
1121 }
1122 DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 }
1124 if (digest_ctx.active) {
1125 digest_ctx.consumed += desc_len;
1126 if (digest_ctx.done) {
1127 if (digest_ctx.unit_no == src_md5) {
1128 digest_ctx.produced = MD5_STATE_LENGTH;
1129 } else {
1130 digest_ctx.produced = SHA1_STATE_LENGTH;
1131 }
1132 }
1133 DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 }
1135 if (cipher_ctx.active) {
1136 /* Ciphers are allowed only to source from DMA out. That is filtered above. */
1137 assert(cipher_ctx.current_src == src_dma);
1138 cipher_ctx.consumed += desc_len;
1139 cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 }
1143 DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 }
1145
1146 /* Setup the DMA out descriptors. */
1147 /* Configure the metadata. */
1148 active_count = 0;
1149 eop_needed_count = 0;
1150 if (cipher_ctx.active) {
1151 ++active_count;
1152 if (cipher_ctx.unit_no == src_dma){
1153 /* mem2mem */
1154 meta_out.ciphsel = src_none;
1155 } else {
1156 meta_out.ciphsel = cipher_ctx.current_src;
1157 }
1158 meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 meta_out.cbcmode = cipher_ctx.cbcmode;
1160 meta_out.decrypt = cipher_ctx.decrypt;
1161 DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 if (cipher_ctx.done) ++eop_needed_count;
1163 } else {
1164 meta_out.ciphsel = src_none;
1165 }
1166
1167 if (digest_ctx.active) {
1168 ++active_count;
1169 meta_out.hashsel = digest_ctx.current_src;
1170 meta_out.hashconf = digest_ctx.hash_conf;
1171 meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 if (digest_ctx.done) {
1174 assert(digest_ctx.pad_descs == NULL);
1175 failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 if (failed) {
1177 DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 goto error_cleanup;
1179 }
1180 }
1181 } else {
1182 meta_out.hashsel = src_none;
1183 }
1184
1185 if (csum_ctx.active) {
1186 ++active_count;
1187 meta_out.csumsel = csum_ctx.current_src;
1188 if (csum_ctx.done) {
1189 assert(csum_ctx.pad_descs == NULL);
1190 failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 if (failed) {
1192 DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 goto error_cleanup;
1194 }
1195 }
1196 } else {
1197 meta_out.csumsel = src_none;
1198 }
1199 DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 /* Setup DMA out descriptors for the indata. */
1201 failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 if (failed) {
1203 DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 goto error_cleanup;
1205 }
1206 /* Setup out EOP. If there are active units that are not done here they cannot get an EOP
1207 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 */
1210 assert(active_count >= eop_needed_count);
1211 assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 if (eop_needed_count) {
1213 /* This means that the bulk operation (cipeher/m2m) is terminated. */
1214 if (active_count > 1) {
1215 /* Use zero length EOP descriptor. */
1216 struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 struct strcop_meta_out ed_mo = {0};
1218 if (!ed) {
1219 DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 failed = -ENOMEM;
1221 goto error_cleanup;
1222 }
1223
1224 assert(cipher_ctx.active && cipher_ctx.done);
1225
1226 if (cipher_ctx.unit_no == src_dma){
1227 /* mem2mem */
1228 ed_mo.ciphsel = src_none;
1229 } else {
1230 ed_mo.ciphsel = cipher_ctx.current_src;
1231 }
1232 ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 ed_mo.decrypt = cipher_ctx.decrypt;
1235
1236 ed->free_buf = NULL;
1237 ed->dma_descr->wait = 1;
1238 ed->dma_descr->out_eop = 1;
1239
1240 ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 ed->dma_descr->after = ed->dma_descr->buf;
1242 ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 current_out_cdesc->next = ed;
1244 current_out_cdesc = ed;
1245 } else {
1246 /* Set EOP in the current out descriptor since the only active module is
1247 * the one needing the EOP. */
1248
1249 current_out_cdesc->dma_descr->out_eop = 1;
1250 }
1251 }
1252
1253 if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 indata_ix += odsc->length;
1257 odsc = odsc->next;
1258 } /* while (odsc) */ /* Process descriptors. */
1259 DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 failed = -EINVAL;
1263 goto error_cleanup;
1264 }
1265 if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 failed = -EINVAL;
1268 goto error_cleanup;
1269 }
1270 if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 failed = -EINVAL;
1273 goto error_cleanup;
1274 }
1275
1276 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 if (failed){
1278 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 goto error_cleanup;
1280 }
1281 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 if (failed){
1283 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 goto error_cleanup;
1285 }
1286 failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 if (failed){
1288 DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 goto error_cleanup;
1290 }
1291
1292 DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 (*int_op)->cdesc_out = out_cdesc_head.next;
1294 (*int_op)->cdesc_in = in_cdesc_head.next;
1295 DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296
1297 setup_descr_chain(out_cdesc_head.next);
1298 setup_descr_chain(in_cdesc_head.next);
1299
1300 /* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 * last DMA out descriptor for EOL.
1302 */
1303 current_in_cdesc->dma_descr->intr = 1;
1304 current_in_cdesc->dma_descr->eol = 1;
1305 current_out_cdesc->dma_descr->eol = 1;
1306
1307 /* Setup DMA contexts. */
1308 (*int_op)->ctx_out.next = NULL;
1309 (*int_op)->ctx_out.eol = 1;
1310 (*int_op)->ctx_out.intr = 0;
1311 (*int_op)->ctx_out.store_mode = 0;
1312 (*int_op)->ctx_out.en = 0;
1313 (*int_op)->ctx_out.dis = 0;
1314 (*int_op)->ctx_out.md0 = 0;
1315 (*int_op)->ctx_out.md1 = 0;
1316 (*int_op)->ctx_out.md2 = 0;
1317 (*int_op)->ctx_out.md3 = 0;
1318 (*int_op)->ctx_out.md4 = 0;
1319 (*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 (*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321
1322 (*int_op)->ctx_in.next = NULL;
1323 (*int_op)->ctx_in.eol = 1;
1324 (*int_op)->ctx_in.intr = 0;
1325 (*int_op)->ctx_in.store_mode = 0;
1326 (*int_op)->ctx_in.en = 0;
1327 (*int_op)->ctx_in.dis = 0;
1328 (*int_op)->ctx_in.md0 = 0;
1329 (*int_op)->ctx_in.md1 = 0;
1330 (*int_op)->ctx_in.md2 = 0;
1331 (*int_op)->ctx_in.md3 = 0;
1332 (*int_op)->ctx_in.md4 = 0;
1333
1334 (*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 (*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336
1337 DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 return 0;
1339
1340 error_cleanup:
1341 {
1342 /* Free all allocated resources. */
1343 struct cryptocop_dma_desc *tmp_cdesc;
1344 while (digest_ctx.pad_descs){
1345 tmp_cdesc = digest_ctx.pad_descs->next;
1346 free_cdesc(digest_ctx.pad_descs);
1347 digest_ctx.pad_descs = tmp_cdesc;
1348 }
1349 while (csum_ctx.pad_descs){
1350 tmp_cdesc = csum_ctx.pad_descs->next;
1351 free_cdesc(csum_ctx.pad_descs);
1352 csum_ctx.pad_descs = tmp_cdesc;
1353 }
1354 assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355
1356 if (*int_op != NULL) delete_internal_operation(*int_op);
1357 }
1358 DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 return failed;
1360 }
1361
1362
1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365 void *ptr = iop->alloc_ptr;
1366 struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 struct cryptocop_dma_desc *next;
1368
1369 DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370
1371 while (cd) {
1372 next = cd->next;
1373 free_cdesc(cd);
1374 cd = next;
1375 }
1376 cd = iop->cdesc_in;
1377 while (cd) {
1378 next = cd->next;
1379 free_cdesc(cd);
1380 cd = next;
1381 }
1382 kfree(ptr);
1383 }
1384
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387
1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390 size_t padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 unsigned char *p;
1392 int i;
1393 unsigned long long int bit_length = hashed_length << 3;
1394
1395 if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396
1397 p = kzalloc(padlen, alloc_flag);
1398 if (!p) return -ENOMEM;
1399
1400 *p = 0x80;
1401
1402 DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1403
1404 i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1405 while (bit_length != 0){
1406 p[i++] = bit_length % 0x100;
1407 bit_length >>= 8;
1408 }
1409
1410 *pad = (char*)p;
1411 *pad_length = padlen;
1412
1413 return 0;
1414 }
1415
1416 #define SHA1_MIN_PAD_LENGTH (9)
1417 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1418
1419 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1420 {
1421 size_t padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1422 unsigned char *p;
1423 int i;
1424 unsigned long long int bit_length = hashed_length << 3;
1425
1426 if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1427
1428 p = kzalloc(padlen, alloc_flag);
1429 if (!p) return -ENOMEM;
1430
1431 *p = 0x80;
1432
1433 DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1434
1435 i = padlen - 1;
1436 while (bit_length != 0){
1437 p[i--] = bit_length % 0x100;
1438 bit_length >>= 8;
1439 }
1440
1441 *pad = (char*)p;
1442 *pad_length = padlen;
1443
1444 return 0;
1445 }
1446
1447
1448 static int transform_ok(struct cryptocop_transform_init *tinit)
1449 {
1450 switch (tinit->alg){
1451 case cryptocop_alg_csum:
1452 switch (tinit->csum_mode){
1453 case cryptocop_csum_le:
1454 case cryptocop_csum_be:
1455 break;
1456 default:
1457 DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1458 return -EINVAL;
1459 }
1460 case cryptocop_alg_mem2mem:
1461 case cryptocop_alg_md5:
1462 case cryptocop_alg_sha1:
1463 if (tinit->keylen != 0) {
1464 DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1465 return -EINVAL; /* This check is a bit strict. */
1466 }
1467 break;
1468 case cryptocop_alg_des:
1469 if (tinit->keylen != 64) {
1470 DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1471 return -EINVAL;
1472 }
1473 break;
1474 case cryptocop_alg_3des:
1475 if (tinit->keylen != 192) {
1476 DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1477 return -EINVAL;
1478 }
1479 break;
1480 case cryptocop_alg_aes:
1481 if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1482 DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1483 return -EINVAL;
1484 }
1485 break;
1486 case cryptocop_no_alg:
1487 default:
1488 DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1489 return -EINVAL;
1490 }
1491
1492 switch (tinit->alg){
1493 case cryptocop_alg_des:
1494 case cryptocop_alg_3des:
1495 case cryptocop_alg_aes:
1496 if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1497 default:
1498 break;
1499 }
1500 return 0;
1501 }
1502
1503
1504 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1505 {
1506 struct cryptocop_session *sess;
1507 struct cryptocop_transform_init *tfrm_in = tinit;
1508 struct cryptocop_transform_init *tmp_in;
1509 int no_tfrms = 0;
1510 int i;
1511 unsigned long int flags;
1512
1513 init_stream_coprocessor(); /* For safety if we are called early */
1514
1515 while (tfrm_in){
1516 int err;
1517 ++no_tfrms;
1518 if ((err = transform_ok(tfrm_in))) {
1519 DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1520 return err;
1521 }
1522 tfrm_in = tfrm_in->next;
1523 }
1524 if (0 == no_tfrms) {
1525 DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1526 return -EINVAL;
1527 }
1528
1529 sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1530 if (!sess){
1531 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1532 return -ENOMEM;
1533 }
1534
1535 sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1536 if (!sess->tfrm_ctx) {
1537 DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1538 kfree(sess);
1539 return -ENOMEM;
1540 }
1541
1542 tfrm_in = tinit;
1543 for (i = 0; i < no_tfrms; i++){
1544 tmp_in = tfrm_in->next;
1545 while (tmp_in){
1546 if (tmp_in->tid == tfrm_in->tid) {
1547 DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1548 kfree(sess->tfrm_ctx);
1549 kfree(sess);
1550 return -EINVAL;
1551 }
1552 tmp_in = tmp_in->next;
1553 }
1554 memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1555 sess->tfrm_ctx[i].dec_key_set = 0;
1556 sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1557
1558 tfrm_in = tfrm_in->next;
1559 }
1560 sess->tfrm_ctx[i-1].next = NULL;
1561
1562 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1563 sess->sid = next_sid;
1564 next_sid++;
1565 /* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1566 * OTOH 2^64 is a really large number of session. */
1567 if (next_sid == 0) next_sid = 1;
1568
1569 /* Prepend to session list. */
1570 sess->next = cryptocop_sessions;
1571 cryptocop_sessions = sess;
1572 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1573 *sid = sess->sid;
1574 return 0;
1575 }
1576
1577
1578 int cryptocop_free_session(cryptocop_session_id sid)
1579 {
1580 struct cryptocop_transform_ctx *tc;
1581 struct cryptocop_session *sess = NULL;
1582 struct cryptocop_session *psess = NULL;
1583 unsigned long int flags;
1584 int i;
1585 LIST_HEAD(remove_list);
1586 struct list_head *node, *tmp;
1587 struct cryptocop_prio_job *pj;
1588
1589 DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1590
1591 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1592 sess = cryptocop_sessions;
1593 while (sess && sess->sid != sid){
1594 psess = sess;
1595 sess = sess->next;
1596 }
1597 if (sess){
1598 if (psess){
1599 psess->next = sess->next;
1600 } else {
1601 cryptocop_sessions = sess->next;
1602 }
1603 }
1604 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1605
1606 if (!sess) return -EINVAL;
1607
1608 /* Remove queued jobs. */
1609 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1610
1611 for (i = 0; i < cryptocop_prio_no_prios; i++){
1612 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1613 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1614 pj = list_entry(node, struct cryptocop_prio_job, node);
1615 if (pj->oper->sid == sid) {
1616 list_move_tail(node, &remove_list);
1617 }
1618 }
1619 }
1620 }
1621 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1622
1623 list_for_each_safe(node, tmp, &remove_list) {
1624 list_del(node);
1625 pj = list_entry(node, struct cryptocop_prio_job, node);
1626 pj->oper->operation_status = -EAGAIN; /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1627 DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1628 pj->oper->cb(pj->oper, pj->oper->cb_data);
1629 delete_internal_operation(pj->iop);
1630 kfree(pj);
1631 }
1632
1633 tc = sess->tfrm_ctx;
1634 /* Erase keying data. */
1635 while (tc){
1636 DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1637 memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1638 memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1639 tc = tc->next;
1640 }
1641 kfree(sess->tfrm_ctx);
1642 kfree(sess);
1643
1644 return 0;
1645 }
1646
1647 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1648 {
1649 struct cryptocop_session *sess;
1650 unsigned long int flags;
1651
1652 spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1653 sess = cryptocop_sessions;
1654 while (sess && (sess->sid != sid)){
1655 sess = sess->next;
1656 }
1657 spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1658
1659 return sess;
1660 }
1661
1662 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1663 {
1664 struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1665
1666 DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1667 assert(sess != NULL);
1668 while (tc && tc->init.tid != tid){
1669 DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1670 tc = tc->next;
1671 }
1672 DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1673 return tc;
1674 }
1675
1676
1677
1678 /* The AES s-transform matrix (s-box). */
1679 static const u8 aes_sbox[256] = {
1680 99, 124, 119, 123, 242, 107, 111, 197, 48, 1, 103, 43, 254, 215, 171, 118,
1681 202, 130, 201, 125, 250, 89, 71, 240, 173, 212, 162, 175, 156, 164, 114, 192,
1682 183, 253, 147, 38, 54, 63, 247, 204, 52, 165, 229, 241, 113, 216, 49, 21,
1683 4, 199, 35, 195, 24, 150, 5, 154, 7, 18, 128, 226, 235, 39, 178, 117,
1684 9, 131, 44, 26, 27, 110, 90, 160, 82, 59, 214, 179, 41, 227, 47, 132,
1685 83, 209, 0, 237, 32, 252, 177, 91, 106, 203, 190, 57, 74, 76, 88, 207,
1686 208, 239, 170, 251, 67, 77, 51, 133, 69, 249, 2, 127, 80, 60, 159, 168,
1687 81, 163, 64, 143, 146, 157, 56, 245, 188, 182, 218, 33, 16, 255, 243, 210,
1688 205, 12, 19, 236, 95, 151, 68, 23, 196, 167, 126, 61, 100, 93, 25, 115,
1689 96, 129, 79, 220, 34, 42, 144, 136, 70, 238, 184, 20, 222, 94, 11, 219,
1690 224, 50, 58, 10, 73, 6, 36, 92, 194, 211, 172, 98, 145, 149, 228, 121,
1691 231, 200, 55, 109, 141, 213, 78, 169, 108, 86, 244, 234, 101, 122, 174, 8,
1692 186, 120, 37, 46, 28, 166, 180, 198, 232, 221, 116, 31, 75, 189, 139, 138,
1693 112, 62, 181, 102, 72, 3, 246, 14, 97, 53, 87, 185, 134, 193, 29, 158,
1694 225, 248, 152, 17, 105, 217, 142, 148, 155, 30, 135, 233, 206, 85, 40, 223,
1695 140, 161, 137, 13, 191, 230, 66, 104, 65, 153, 45, 15, 176, 84, 187, 22
1696 };
1697
1698 /* AES has a 32 bit word round constants for each round in the
1699 * key schedule. round_constant[i] is really Rcon[i+1] in FIPS187.
1700 */
1701 static u32 round_constant[11] = {
1702 0x01000000, 0x02000000, 0x04000000, 0x08000000,
1703 0x10000000, 0x20000000, 0x40000000, 0x80000000,
1704 0x1B000000, 0x36000000, 0x6C000000
1705 };
1706
1707 /* Apply the s-box to each of the four occtets in w. */
1708 static u32 aes_ks_subword(const u32 w)
1709 {
1710 u8 bytes[4];
1711
1712 *(u32*)(&bytes[0]) = w;
1713 bytes[0] = aes_sbox[bytes[0]];
1714 bytes[1] = aes_sbox[bytes[1]];
1715 bytes[2] = aes_sbox[bytes[2]];
1716 bytes[3] = aes_sbox[bytes[3]];
1717 return *(u32*)(&bytes[0]);
1718 }
1719
1720 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1721 * (Note that AES words are 32 bit long)
1722 *
1723 * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1724 * word temp
1725 * i = 0
1726 * while (i < Nk) {
1727 * w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1728 * i = i + 1
1729 * }
1730 * i = Nk
1731 *
1732 * while (i < (Nb * (Nr + 1))) {
1733 * temp = w[i - 1]
1734 * if ((i mod Nk) == 0) {
1735 * temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1736 * }
1737 * else if ((Nk > 6) && ((i mod Nk) == 4)) {
1738 * temp = SubWord(temp)
1739 * }
1740 * w[i] = w[i - Nk] xor temp
1741 * }
1742 * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1743 * SubWord(t) applies the AES s-box individually to each octet
1744 * in a 32 bit word.
1745 *
1746 * For AES Nk can have the values 4, 6, and 8 (corresponding to
1747 * values for Nr of 10, 12, and 14). Nb is always 4.
1748 *
1749 * To construct w[i], w[i - 1] and w[i - Nk] must be
1750 * available. Consequently we must keep a state of the last Nk words
1751 * to be able to create the last round keys.
1752 */
1753 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned char *key, unsigned int keylength)
1754 {
1755 u32 temp;
1756 u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1757 u8 w_last_ix;
1758 int i;
1759 u8 nr, nk;
1760
1761 switch (keylength){
1762 case 128:
1763 nk = 4;
1764 nr = 10;
1765 break;
1766 case 192:
1767 nk = 6;
1768 nr = 12;
1769 break;
1770 case 256:
1771 nk = 8;
1772 nr = 14;
1773 break;
1774 default:
1775 panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1776 };
1777
1778 /* Need to do host byte order correction here since key is byte oriented and the
1779 * kx algorithm is word (u32) oriented. */
1780 for (i = 0; i < nk; i+=1) {
1781 w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1782 }
1783
1784 i = (int)nk;
1785 w_last_ix = i - 1;
1786 while (i < (4 * (nr + 2))) {
1787 temp = w_ring[w_last_ix];
1788 if (!(i % nk)) {
1789 /* RotWord(temp) */
1790 temp = (temp << 8) | (temp >> 24);
1791 temp = aes_ks_subword(temp);
1792 temp ^= round_constant[i/nk - 1];
1793 } else if ((nk > 6) && ((i % nk) == 4)) {
1794 temp = aes_ks_subword(temp);
1795 }
1796 w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1797 temp ^= w_ring[w_last_ix];
1798 w_ring[w_last_ix] = temp;
1799
1800 /* We need the round keys for round Nr+1 and Nr+2 (round key
1801 * Nr+2 is the round key beyond the last one used when
1802 * encrypting). Rounds are numbered starting from 0, Nr=10
1803 * implies 11 rounds are used in encryption/decryption.
1804 */
1805 if (i >= (4 * nr)) {
1806 /* Need to do host byte order correction here, the key
1807 * is byte oriented. */
1808 *(u32*)dec_key = cpu_to_be32(temp);
1809 dec_key += 4;
1810 }
1811 ++i;
1812 }
1813 }
1814
1815
1816 /**** Job/operation management. ****/
1817
1818 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1819 {
1820 return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1821 }
1822
1823 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1824 {
1825 return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1826 }
1827
1828 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1829 {
1830 return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1831 }
1832
1833 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1834 {
1835 int ret;
1836 struct cryptocop_prio_job *pj = NULL;
1837 unsigned long int flags;
1838
1839 DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1840
1841 if (!operation || !operation->cb){
1842 DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1843 return -EINVAL;
1844 }
1845
1846 if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1847 DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1848 return ret;
1849 }
1850 assert(pj != NULL);
1851
1852 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1853 list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1854 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1855
1856 /* Make sure a job is running */
1857 cryptocop_start_job();
1858 return 0;
1859 }
1860
1861 static void cryptocop_do_tasklet(unsigned long unused);
1862 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1863
1864 static void cryptocop_do_tasklet(unsigned long unused)
1865 {
1866 struct list_head *node;
1867 struct cryptocop_prio_job *pj = NULL;
1868 unsigned long flags;
1869
1870 DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1871
1872 do {
1873 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1874 if (!list_empty(&cryptocop_completed_jobs)){
1875 node = cryptocop_completed_jobs.next;
1876 list_del(node);
1877 pj = list_entry(node, struct cryptocop_prio_job, node);
1878 } else {
1879 pj = NULL;
1880 }
1881 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1882 if (pj) {
1883 assert(pj->oper != NULL);
1884
1885 /* Notify consumer of operation completeness. */
1886 DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1887
1888 pj->oper->operation_status = 0; /* Job is completed. */
1889 pj->oper->cb(pj->oper, pj->oper->cb_data);
1890 delete_internal_operation(pj->iop);
1891 kfree(pj);
1892 }
1893 } while (pj != NULL);
1894
1895 DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1896 }
1897
1898 static irqreturn_t
1899 dma_done_interrupt(int irq, void *dev_id)
1900 {
1901 struct cryptocop_prio_job *done_job;
1902 reg_dma_rw_ack_intr ack_intr = {
1903 .data = 1,
1904 };
1905
1906 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1907
1908 DEBUG(printk("cryptocop DMA done\n"));
1909
1910 spin_lock(&running_job_lock);
1911 if (cryptocop_running_job == NULL){
1912 printk("stream co-processor got interrupt when not busy\n");
1913 spin_unlock(&running_job_lock);
1914 return IRQ_HANDLED;
1915 }
1916 done_job = cryptocop_running_job;
1917 cryptocop_running_job = NULL;
1918 spin_unlock(&running_job_lock);
1919
1920 /* Start processing a job. */
1921 if (!spin_trylock(&cryptocop_process_lock)){
1922 DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1923 } else {
1924 cryptocop_start_job();
1925 spin_unlock(&cryptocop_process_lock);
1926 }
1927
1928 done_job->oper->operation_status = 0; /* Job is completed. */
1929 if (done_job->oper->fast_callback){
1930 /* This operation wants callback from interrupt. */
1931 done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1932 delete_internal_operation(done_job->iop);
1933 kfree(done_job);
1934 } else {
1935 spin_lock(&cryptocop_completed_jobs_lock);
1936 list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1937 spin_unlock(&cryptocop_completed_jobs_lock);
1938 tasklet_schedule(&cryptocop_tasklet);
1939 }
1940
1941 DEBUG(printk("cryptocop leave irq handler\n"));
1942 return IRQ_HANDLED;
1943 }
1944
1945
1946 /* Setup interrupts and DMA channels. */
1947 static int init_cryptocop(void)
1948 {
1949 unsigned long flags;
1950 reg_dma_rw_cfg dma_cfg = {.en = 1};
1951 reg_dma_rw_intr_mask intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1952 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
1953 reg_strcop_rw_cfg strcop_cfg = {
1954 .ipend = regk_strcop_little,
1955 .td1 = regk_strcop_e,
1956 .td2 = regk_strcop_d,
1957 .td3 = regk_strcop_e,
1958 .ignore_sync = 0,
1959 .en = 1
1960 };
1961
1962 if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1963 "stream co-processor DMA", NULL))
1964 panic("request_irq stream co-processor irq dma9");
1965
1966 (void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1967 0, dma_strp);
1968 (void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 0, dma_strp);
1970
1971 local_irq_save(flags);
1972
1973 /* Reset and enable the cryptocop. */
1974 strcop_cfg.en = 0;
1975 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1976 strcop_cfg.en = 1;
1977 REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978
1979 /* Enable DMAs. */
1980 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1981 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1982
1983 /* Set up wordsize = 4 for DMAs. */
1984 DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1985 DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1986
1987 /* Enable interrupts. */
1988 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1989
1990 /* Clear intr ack. */
1991 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1992
1993 local_irq_restore(flags);
1994
1995 return 0;
1996 }
1997
1998 /* Free used cryptocop hw resources (interrupt and DMA channels). */
1999 static void release_cryptocop(void)
2000 {
2001 unsigned long flags;
2002 reg_dma_rw_cfg dma_cfg = {.en = 0};
2003 reg_dma_rw_intr_mask intr_mask_in = {0};
2004 reg_dma_rw_ack_intr ack_intr = {.data = 1,.in_eop = 1 };
2005
2006 local_irq_save(flags);
2007
2008 /* Clear intr ack. */
2009 REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2010
2011 /* Disable DMAs. */
2012 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2013 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2014
2015 /* Disable interrupts. */
2016 REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2017
2018 local_irq_restore(flags);
2019
2020 free_irq(DMA_IRQ, NULL);
2021
2022 (void)crisv32_free_dma(OUT_DMA);
2023 (void)crisv32_free_dma(IN_DMA);
2024 }
2025
2026
2027 /* Init job queue. */
2028 static int cryptocop_job_queue_init(void)
2029 {
2030 int i;
2031
2032 INIT_LIST_HEAD(&cryptocop_completed_jobs);
2033
2034 for (i = 0; i < cryptocop_prio_no_prios; i++){
2035 cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2036 INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2037 }
2038 return 0;
2039 }
2040
2041
2042 static void cryptocop_job_queue_close(void)
2043 {
2044 struct list_head *node, *tmp;
2045 struct cryptocop_prio_job *pj = NULL;
2046 unsigned long int process_flags, flags;
2047 int i;
2048
2049 /* FIXME: This is as yet untested code. */
2050
2051 /* Stop strcop from getting an operation to process while we are closing the
2052 module. */
2053 spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2054
2055 /* Empty the job queue. */
2056 for (i = 0; i < cryptocop_prio_no_prios; i++){
2057 if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2058 list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2059 pj = list_entry(node, struct cryptocop_prio_job, node);
2060 list_del(node);
2061
2062 /* Call callback to notify consumer of job removal. */
2063 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2064 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2065 pj->oper->cb(pj->oper, pj->oper->cb_data);
2066
2067 delete_internal_operation(pj->iop);
2068 kfree(pj);
2069 }
2070 }
2071 }
2072 spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2073
2074 /* Remove the running job, if any. */
2075 spin_lock_irqsave(&running_job_lock, flags);
2076 if (cryptocop_running_job){
2077 reg_strcop_rw_cfg rw_cfg;
2078 reg_dma_rw_cfg dma_out_cfg, dma_in_cfg;
2079
2080 /* Stop DMA. */
2081 dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2082 dma_out_cfg.en = regk_dma_no;
2083 REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2084
2085 dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2086 dma_in_cfg.en = regk_dma_no;
2087 REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2088
2089 /* Disble the cryptocop. */
2090 rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2091 rw_cfg.en = 0;
2092 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2093
2094 pj = cryptocop_running_job;
2095 cryptocop_running_job = NULL;
2096
2097 /* Call callback to notify consumer of job removal. */
2098 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2099 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2100 pj->oper->cb(pj->oper, pj->oper->cb_data);
2101
2102 delete_internal_operation(pj->iop);
2103 kfree(pj);
2104 }
2105 spin_unlock_irqrestore(&running_job_lock, flags);
2106
2107 /* Remove completed jobs, if any. */
2108 spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2109
2110 list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2111 pj = list_entry(node, struct cryptocop_prio_job, node);
2112 list_del(node);
2113 /* Call callback to notify consumer of job removal. */
2114 DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2115 pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2116 pj->oper->cb(pj->oper, pj->oper->cb_data);
2117
2118 delete_internal_operation(pj->iop);
2119 kfree(pj);
2120 }
2121 spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2122 }
2123
2124
2125 static void cryptocop_start_job(void)
2126 {
2127 int i;
2128 struct cryptocop_prio_job *pj;
2129 unsigned long int flags;
2130 unsigned long int running_job_flags;
2131 reg_strcop_rw_cfg rw_cfg = {.en = 1, .ignore_sync = 0};
2132
2133 DEBUG(printk("cryptocop_start_job: entering\n"));
2134
2135 spin_lock_irqsave(&running_job_lock, running_job_flags);
2136 if (cryptocop_running_job != NULL){
2137 /* Already running. */
2138 DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2139 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2140 return;
2141 }
2142 spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2143
2144 /* Check the queues in priority order. */
2145 for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2146 if (i == cryptocop_prio_no_prios) {
2147 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2148 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2149 DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2150 return; /* No jobs to run */
2151 }
2152 DEBUG(printk("starting job for prio %d\n", i));
2153
2154 /* TODO: Do not starve lower priority jobs. Let in a lower
2155 * prio job for every N-th processed higher prio job or some
2156 * other scheduling policy. This could reasonably be
2157 * tweakable since the optimal balance would depend on the
2158 * type of load on the system. */
2159
2160 /* Pull the DMA lists from the job and start the DMA client. */
2161 pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2162 list_del(&pj->node);
2163 spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2164 cryptocop_running_job = pj;
2165
2166 /* Set config register (3DES and CSUM modes). */
2167 switch (pj->iop->tdes_mode){
2168 case cryptocop_3des_eee:
2169 rw_cfg.td1 = regk_strcop_e;
2170 rw_cfg.td2 = regk_strcop_e;
2171 rw_cfg.td3 = regk_strcop_e;
2172 break;
2173 case cryptocop_3des_eed:
2174 rw_cfg.td1 = regk_strcop_e;
2175 rw_cfg.td2 = regk_strcop_e;
2176 rw_cfg.td3 = regk_strcop_d;
2177 break;
2178 case cryptocop_3des_ede:
2179 rw_cfg.td1 = regk_strcop_e;
2180 rw_cfg.td2 = regk_strcop_d;
2181 rw_cfg.td3 = regk_strcop_e;
2182 break;
2183 case cryptocop_3des_edd:
2184 rw_cfg.td1 = regk_strcop_e;
2185 rw_cfg.td2 = regk_strcop_d;
2186 rw_cfg.td3 = regk_strcop_d;
2187 break;
2188 case cryptocop_3des_dee:
2189 rw_cfg.td1 = regk_strcop_d;
2190 rw_cfg.td2 = regk_strcop_e;
2191 rw_cfg.td3 = regk_strcop_e;
2192 break;
2193 case cryptocop_3des_ded:
2194 rw_cfg.td1 = regk_strcop_d;
2195 rw_cfg.td2 = regk_strcop_e;
2196 rw_cfg.td3 = regk_strcop_d;
2197 break;
2198 case cryptocop_3des_dde:
2199 rw_cfg.td1 = regk_strcop_d;
2200 rw_cfg.td2 = regk_strcop_d;
2201 rw_cfg.td3 = regk_strcop_e;
2202 break;
2203 case cryptocop_3des_ddd:
2204 rw_cfg.td1 = regk_strcop_d;
2205 rw_cfg.td2 = regk_strcop_d;
2206 rw_cfg.td3 = regk_strcop_d;
2207 break;
2208 default:
2209 DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2210 }
2211 switch (pj->iop->csum_mode){
2212 case cryptocop_csum_le:
2213 rw_cfg.ipend = regk_strcop_little;
2214 break;
2215 case cryptocop_csum_be:
2216 rw_cfg.ipend = regk_strcop_big;
2217 break;
2218 default:
2219 DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2220 }
2221 REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2222
2223 DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2224 "ctx_in: 0x%p, phys: 0x%p\n"
2225 "ctx_out: 0x%p, phys: 0x%p\n",
2226 pj,
2227 &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2228 &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2229
2230 /* Start input DMA. */
2231 flush_dma_context(&pj->iop->ctx_in);
2232 DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2233
2234 /* Start output DMA. */
2235 DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2236
2237 spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2238 DEBUG(printk("cryptocop_start_job: exiting\n"));
2239 }
2240
2241
2242 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2243 {
2244 int err;
2245 int alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2246 void *iop_alloc_ptr = NULL;
2247
2248 *pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2249 if (!*pj) return -ENOMEM;
2250
2251 DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2252
2253 (*pj)->oper = operation;
2254 DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n", (*pj)->oper->cb, (*pj)->oper->cb_data));
2255
2256 if (operation->use_dmalists) {
2257 DEBUG(print_user_dma_lists(&operation->list_op));
2258 if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2259 DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2260 kfree(*pj);
2261 return -EINVAL;
2262 }
2263 iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2264 if (!iop_alloc_ptr) {
2265 DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2266 kfree(*pj);
2267 return -ENOMEM;
2268 }
2269 (*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2270 DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2271 (*pj)->iop->alloc_ptr = iop_alloc_ptr;
2272 (*pj)->iop->sid = operation->sid;
2273 (*pj)->iop->cdesc_out = NULL;
2274 (*pj)->iop->cdesc_in = NULL;
2275 (*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2276 (*pj)->iop->csum_mode = operation->list_op.csum_mode;
2277 (*pj)->iop->ddesc_out = operation->list_op.outlist;
2278 (*pj)->iop->ddesc_in = operation->list_op.inlist;
2279
2280 /* Setup DMA contexts. */
2281 (*pj)->iop->ctx_out.next = NULL;
2282 (*pj)->iop->ctx_out.eol = 1;
2283 (*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2284 (*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2285
2286 (*pj)->iop->ctx_in.next = NULL;
2287 (*pj)->iop->ctx_in.eol = 1;
2288 (*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2289 (*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2290 } else {
2291 if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2292 DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2293 kfree(*pj);
2294 return err;
2295 }
2296 }
2297 DEBUG(print_dma_descriptors((*pj)->iop));
2298
2299 DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2300
2301 return 0;
2302 }
2303
2304 static int cryptocop_open(struct inode *inode, struct file *filp)
2305 {
2306 int p = iminor(inode);
2307
2308 if (p != CRYPTOCOP_MINOR) return -EINVAL;
2309
2310 filp->private_data = NULL;
2311 return 0;
2312 }
2313
2314
2315 static int cryptocop_release(struct inode *inode, struct file *filp)
2316 {
2317 struct cryptocop_private *dev = filp->private_data;
2318 struct cryptocop_private *dev_next;
2319
2320 while (dev){
2321 dev_next = dev->next;
2322 if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2323 (void)cryptocop_free_session(dev->sid);
2324 }
2325 kfree(dev);
2326 dev = dev_next;
2327 }
2328
2329 return 0;
2330 }
2331
2332
2333 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2334 unsigned int cmd, unsigned long arg)
2335 {
2336 struct cryptocop_private *dev = filp->private_data;
2337 struct cryptocop_private *prev_dev = NULL;
2338 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2339 struct strcop_session_op sop;
2340 int err;
2341
2342 DEBUG(printk("cryptocop_ioctl_close_session\n"));
2343
2344 if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2345 return -EFAULT;
2346 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2347 if (err) return -EFAULT;
2348
2349 while (dev && (dev->sid != sop.ses_id)) {
2350 prev_dev = dev;
2351 dev = dev->next;
2352 }
2353 if (dev){
2354 if (prev_dev){
2355 prev_dev->next = dev->next;
2356 } else {
2357 filp->private_data = dev->next;
2358 }
2359 err = cryptocop_free_session(dev->sid);
2360 if (err) return -EFAULT;
2361 } else {
2362 DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2363 return -EINVAL;
2364 }
2365 return 0;
2366 }
2367
2368
2369 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2370 {
2371 struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2372
2373 DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2374
2375 jc->processed = 1;
2376 wake_up(&cryptocop_ioc_process_wq);
2377 }
2378
2379
2380 #define CRYPTOCOP_IOCTL_CIPHER_TID (1)
2381 #define CRYPTOCOP_IOCTL_DIGEST_TID (2)
2382 #define CRYPTOCOP_IOCTL_CSUM_TID (3)
2383
2384 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2385 {
2386 size_t ch_ix = 0;
2387
2388 if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2389 if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2390 if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2391
2392 DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2393 return ch_ix;
2394 }
2395
2396
2397 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2398 {
2399 size_t ch_ix = INT_MAX;
2400 size_t tmp_ix = 0;
2401
2402 if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2403 if (crp_op->cipher_start > ix) {
2404 ch_ix = crp_op->cipher_start;
2405 } else {
2406 ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2407 }
2408 }
2409 if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2410 if (crp_op->digest_start > ix) {
2411 tmp_ix = crp_op->digest_start;
2412 } else {
2413 tmp_ix = crp_op->digest_start + crp_op->digest_len;
2414 }
2415 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2416 }
2417 if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2418 if (crp_op->csum_start > ix) {
2419 tmp_ix = crp_op->csum_start;
2420 } else {
2421 tmp_ix = crp_op->csum_start + crp_op->csum_len;
2422 }
2423 if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2424 }
2425 if (ch_ix == INT_MAX) ch_ix = ix;
2426 DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2427 return ch_ix;
2428 }
2429
2430
2431 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2432 * Return -1 for ok, 0 for fail. */
2433 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2434 {
2435 int tmplen;
2436
2437 assert(iov != NULL);
2438 assert(iovix != NULL);
2439 assert(pages != NULL);
2440 assert(pageix != NULL);
2441 assert(pageoffset != NULL);
2442
2443 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2444
2445 while (map_length > 0){
2446 DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2447 if (*iovix >= iovlen){
2448 DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2449 return 0;
2450 }
2451 if (*pageix >= nopages){
2452 DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2453 return 0;
2454 }
2455 iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2456 tmplen = PAGE_SIZE - *pageoffset;
2457 if (tmplen < map_length){
2458 (*pageoffset) = 0;
2459 (*pageix)++;
2460 } else {
2461 tmplen = map_length;
2462 (*pageoffset) += map_length;
2463 }
2464 DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2465 iov[*iovix].iov_len = tmplen;
2466 map_length -= tmplen;
2467 (*iovix)++;
2468 }
2469 DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2470 return -1;
2471 }
2472
2473
2474
2475 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2476 {
2477 int i;
2478 struct cryptocop_private *dev = filp->private_data;
2479 struct strcop_crypto_op *crp_oper = (struct strcop_crypto_op *)arg;
2480 struct strcop_crypto_op oper = {0};
2481 int err = 0;
2482 struct cryptocop_operation *cop = NULL;
2483
2484 struct ioctl_job_cb_ctx *jc = NULL;
2485
2486 struct page **inpages = NULL;
2487 struct page **outpages = NULL;
2488 int noinpages = 0;
2489 int nooutpages = 0;
2490
2491 struct cryptocop_desc descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2492 * can get connected/disconnected on different places in the indata. */
2493 struct cryptocop_desc_cfg dcfgs[5*3];
2494 int desc_ix = 0;
2495 int dcfg_ix = 0;
2496 struct cryptocop_tfrm_cfg ciph_tcfg = {0};
2497 struct cryptocop_tfrm_cfg digest_tcfg = {0};
2498 struct cryptocop_tfrm_cfg csum_tcfg = {0};
2499
2500 unsigned char *digest_result = NULL;
2501 int digest_length = 0;
2502 int cblocklen = 0;
2503 unsigned char csum_result[CSUM_BLOCK_LENGTH];
2504 struct cryptocop_session *sess;
2505
2506 int iovlen = 0;
2507 int iovix = 0;
2508 int pageix = 0;
2509 int pageoffset = 0;
2510
2511 size_t prev_ix = 0;
2512 size_t next_ix;
2513
2514 int cipher_active, digest_active, csum_active;
2515 int end_digest, end_csum;
2516 int digest_done = 0;
2517 int cipher_done = 0;
2518 int csum_done = 0;
2519
2520 DEBUG(printk("cryptocop_ioctl_process\n"));
2521
2522 if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2523 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2524 return -EFAULT;
2525 }
2526 if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2527 DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2528 return -EFAULT;
2529 }
2530 DEBUG(print_strcop_crypto_op(&oper));
2531
2532 while (dev && dev->sid != oper.ses_id) dev = dev->next;
2533 if (!dev){
2534 DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2535 return -EINVAL;
2536 }
2537
2538 /* Check buffers. */
2539 if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2540 DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2541 return -EINVAL;
2542 }
2543
2544 if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2545 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2546 return -EFAULT;
2547 }
2548 if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2549 DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2550 return -EFAULT;
2551 }
2552
2553 cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2554 if (!cop) {
2555 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2556 return -ENOMEM;
2557 }
2558 jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2559 if (!jc) {
2560 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2561 err = -ENOMEM;
2562 goto error_cleanup;
2563 }
2564 jc->processed = 0;
2565
2566 cop->cb_data = jc;
2567 cop->cb = ioctl_process_job_callback;
2568 cop->operation_status = 0;
2569 cop->use_dmalists = 0;
2570 cop->in_interrupt = 0;
2571 cop->fast_callback = 0;
2572 cop->tfrm_op.tfrm_cfg = NULL;
2573 cop->tfrm_op.desc = NULL;
2574 cop->tfrm_op.indata = NULL;
2575 cop->tfrm_op.incount = 0;
2576 cop->tfrm_op.inlen = 0;
2577 cop->tfrm_op.outdata = NULL;
2578 cop->tfrm_op.outcount = 0;
2579 cop->tfrm_op.outlen = 0;
2580
2581 sess = get_session(oper.ses_id);
2582 if (!sess){
2583 DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2584 kfree(cop);
2585 kfree(jc);
2586 return -EINVAL;
2587 }
2588
2589 if (oper.do_cipher) {
2590 unsigned int cipher_outlen = 0;
2591 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2592 if (!tc) {
2593 DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2594 err = -EINVAL;
2595 goto error_cleanup;
2596 }
2597 ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2598 ciph_tcfg.inject_ix = 0;
2599 ciph_tcfg.flags = 0;
2600 if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2601 DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2602 kfree(cop);
2603 kfree(jc);
2604 return -EINVAL;
2605 }
2606 cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2607 if (oper.cipher_len % cblocklen) {
2608 kfree(cop);
2609 kfree(jc);
2610 DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2611 return -EINVAL;
2612 }
2613 cipher_outlen = oper.cipher_len;
2614 if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2615 if (oper.cipher_explicit) {
2616 ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2617 memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2618 } else {
2619 cipher_outlen = oper.cipher_len - cblocklen;
2620 }
2621 } else {
2622 if (oper.cipher_explicit){
2623 kfree(cop);
2624 kfree(jc);
2625 DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2626 return -EINVAL;
2627 }
2628 }
2629 if (oper.cipher_outlen != cipher_outlen) {
2630 kfree(cop);
2631 kfree(jc);
2632 DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2633 return -EINVAL;
2634 }
2635
2636 if (oper.decrypt){
2637 ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2638 } else {
2639 ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2640 }
2641 ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2642 cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2643 }
2644 if (oper.do_digest){
2645 struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2646 if (!tc) {
2647 DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2648 err = -EINVAL;
2649 goto error_cleanup;
2650 }
2651 digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2652 digest_result = kmalloc(digest_length, GFP_KERNEL);
2653 if (!digest_result) {
2654 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2655 err = -EINVAL;
2656 goto error_cleanup;
2657 }
2658 DEBUG(memset(digest_result, 0xff, digest_length));
2659
2660 digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2661 digest_tcfg.inject_ix = 0;
2662 ciph_tcfg.inject_ix += digest_length;
2663 if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2664 DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2665 err = -EINVAL;
2666 goto error_cleanup;
2667 }
2668
2669 digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2670 cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2671 }
2672 if (oper.do_csum){
2673 csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2674 csum_tcfg.inject_ix = digest_length;
2675 ciph_tcfg.inject_ix += 2;
2676
2677 if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2678 DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2679 kfree(cop);
2680 kfree(jc);
2681 return -EINVAL;
2682 }
2683
2684 csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2685 cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2686 }
2687
2688 prev_ix = first_cfg_change_ix(&oper);
2689 if (prev_ix > oper.inlen) {
2690 DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2691 nooutpages = noinpages = 0;
2692 err = -EINVAL;
2693 goto error_cleanup;
2694 }
2695 DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2696
2697 /* Map user pages for in and out data of the operation. */
2698 noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2699 DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2700 inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2701 if (!inpages){
2702 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2703 nooutpages = noinpages = 0;
2704 err = -ENOMEM;
2705 goto error_cleanup;
2706 }
2707 if (oper.do_cipher){
2708 nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2709 DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2710 outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2711 if (!outpages){
2712 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2713 nooutpages = noinpages = 0;
2714 err = -ENOMEM;
2715 goto error_cleanup;
2716 }
2717 }
2718
2719 /* Acquire the mm page semaphore. */
2720 down_read(&current->mm->mmap_sem);
2721
2722 err = get_user_pages((unsigned long int)(oper.indata + prev_ix),
2723 noinpages,
2724 0, /* read access only for in data */
2725 0, /* no force */
2726 inpages,
2727 NULL);
2728
2729 if (err < 0) {
2730 up_read(&current->mm->mmap_sem);
2731 nooutpages = noinpages = 0;
2732 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2733 goto error_cleanup;
2734 }
2735 noinpages = err;
2736 if (oper.do_cipher){
2737 err = get_user_pages((unsigned long int)oper.cipher_outdata,
2738 nooutpages,
2739 1, /* write access for out data */
2740 0, /* no force */
2741 outpages,
2742 NULL);
2743 up_read(&current->mm->mmap_sem);
2744 if (err < 0) {
2745 nooutpages = 0;
2746 DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2747 goto error_cleanup;
2748 }
2749 nooutpages = err;
2750 } else {
2751 up_read(&current->mm->mmap_sem);
2752 }
2753
2754 /* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2755 * csum output and splits when units are (dis-)connected. */
2756 cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2757 cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2758 if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2759 DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2760 err = -ENOMEM;
2761 goto error_cleanup;
2762 }
2763
2764 cop->tfrm_op.inlen = oper.inlen - prev_ix;
2765 cop->tfrm_op.outlen = 0;
2766 if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2767 if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2768 if (oper.do_csum) cop->tfrm_op.outlen += 2;
2769
2770 /* Setup the in iovecs. */
2771 cop->tfrm_op.incount = noinpages;
2772 if (noinpages > 1){
2773 size_t tmplen = cop->tfrm_op.inlen;
2774
2775 cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2776 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2777 tmplen -= cop->tfrm_op.indata[0].iov_len;
2778 for (i = 1; i<noinpages; i++){
2779 cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2780 cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2781 tmplen -= PAGE_SIZE;
2782 }
2783 } else {
2784 cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2785 cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2786 }
2787
2788 iovlen = nooutpages + 6;
2789 pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2790
2791 next_ix = next_cfg_change_ix(&oper, prev_ix);
2792 if (prev_ix == next_ix){
2793 DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2794 err = -EINVAL; /* This should be impossible barring bugs. */
2795 goto error_cleanup;
2796 }
2797 while (prev_ix != next_ix){
2798 end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2799 descs[desc_ix].cfg = NULL;
2800 descs[desc_ix].length = next_ix - prev_ix;
2801
2802 if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2803 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2804 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2805 cipher_active = 1;
2806
2807 if (next_ix == (oper.cipher_start + oper.cipher_len)){
2808 cipher_done = 1;
2809 dcfgs[dcfg_ix].last = 1;
2810 } else {
2811 dcfgs[dcfg_ix].last = 0;
2812 }
2813 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2814 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2815 ++dcfg_ix;
2816 }
2817 if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2818 digest_active = 1;
2819 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2820 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2821 if (next_ix == (oper.digest_start + oper.digest_len)){
2822 assert(!digest_done);
2823 digest_done = 1;
2824 dcfgs[dcfg_ix].last = 1;
2825 } else {
2826 dcfgs[dcfg_ix].last = 0;
2827 }
2828 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2829 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2830 ++dcfg_ix;
2831 }
2832 if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2833 csum_active = 1;
2834 dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2835 dcfgs[dcfg_ix].src = cryptocop_source_dma;
2836 if (next_ix == (oper.csum_start + oper.csum_len)){
2837 csum_done = 1;
2838 dcfgs[dcfg_ix].last = 1;
2839 } else {
2840 dcfgs[dcfg_ix].last = 0;
2841 }
2842 dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2843 descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2844 ++dcfg_ix;
2845 }
2846 if (!descs[desc_ix].cfg){
2847 DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2848 err = -EINVAL;
2849 goto error_cleanup;
2850 }
2851 descs[desc_ix].next = &(descs[desc_ix]) + 1;
2852 ++desc_ix;
2853 prev_ix = next_ix;
2854 next_ix = next_cfg_change_ix(&oper, prev_ix);
2855 }
2856 if (desc_ix > 0){
2857 descs[desc_ix-1].next = NULL;
2858 } else {
2859 descs[0].next = NULL;
2860 }
2861 if (oper.do_digest) {
2862 DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2863 /* Add outdata iovec, length == <length of type of digest> */
2864 cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2865 cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2866 ++iovix;
2867 }
2868 if (oper.do_csum) {
2869 /* Add outdata iovec, length == 2, the length of csum. */
2870 DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2871 /* Add outdata iovec, length == <length of type of digest> */
2872 cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2873 cop->tfrm_op.outdata[iovix].iov_len = 2;
2874 ++iovix;
2875 }
2876 if (oper.do_cipher) {
2877 if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2878 DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2879 err = -ENOSYS; /* This should be impossible barring bugs. */
2880 goto error_cleanup;
2881 }
2882 }
2883 DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2884 cop->tfrm_op.outcount = iovix;
2885 assert(iovix <= (nooutpages + 6));
2886
2887 cop->sid = oper.ses_id;
2888 cop->tfrm_op.desc = &descs[0];
2889
2890 DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2891
2892 if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2893 DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2894 err = -EINVAL;
2895 goto error_cleanup;
2896 }
2897
2898 DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2899
2900 wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2901 DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2902 if (!jc->processed){
2903 printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2904 err = -EIO;
2905 goto error_cleanup;
2906 }
2907
2908 /* Job process done. Cipher output should already be correct in job so no post processing of outdata. */
2909 DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2910 if (cop->operation_status == 0){
2911 if (oper.do_digest){
2912 DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2913 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2914 if (0 != err){
2915 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2916 err = -EFAULT;
2917 goto error_cleanup;
2918 }
2919 }
2920 if (oper.do_csum){
2921 DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2922 err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2923 if (0 != err){
2924 DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2925 err = -EFAULT;
2926 goto error_cleanup;
2927 }
2928 }
2929 err = 0;
2930 } else {
2931 DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2932 err = cop->operation_status;
2933 }
2934
2935 error_cleanup:
2936 /* Release page caches. */
2937 for (i = 0; i < noinpages; i++){
2938 put_page(inpages[i]);
2939 }
2940 for (i = 0; i < nooutpages; i++){
2941 int spdl_err;
2942 /* Mark output pages dirty. */
2943 spdl_err = set_page_dirty_lock(outpages[i]);
2944 DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2945 }
2946 for (i = 0; i < nooutpages; i++){
2947 put_page(outpages[i]);
2948 }
2949
2950 kfree(digest_result);
2951 kfree(inpages);
2952 kfree(outpages);
2953 if (cop){
2954 kfree(cop->tfrm_op.indata);
2955 kfree(cop->tfrm_op.outdata);
2956 kfree(cop);
2957 }
2958 kfree(jc);
2959
2960 DEBUG(print_lock_status());
2961
2962 return err;
2963 }
2964
2965
2966 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2967 {
2968 cryptocop_session_id sid;
2969 int err;
2970 struct cryptocop_private *dev;
2971 struct strcop_session_op *sess_op = (struct strcop_session_op *)arg;
2972 struct strcop_session_op sop;
2973 struct cryptocop_transform_init *tis = NULL;
2974 struct cryptocop_transform_init ti_cipher = {0};
2975 struct cryptocop_transform_init ti_digest = {0};
2976 struct cryptocop_transform_init ti_csum = {0};
2977
2978 if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2979 return -EFAULT;
2980 err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2981 if (err) return -EFAULT;
2982 if (sop.cipher != cryptocop_cipher_none) {
2983 if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2984 }
2985 DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2986
2987 DEBUG(printk("\tcipher:%d\n"
2988 "\tcipher_mode:%d\n"
2989 "\tdigest:%d\n"
2990 "\tcsum:%d\n",
2991 (int)sop.cipher,
2992 (int)sop.cmode,
2993 (int)sop.digest,
2994 (int)sop.csum));
2995
2996 if (sop.cipher != cryptocop_cipher_none){
2997 /* Init the cipher. */
2998 switch (sop.cipher){
2999 case cryptocop_cipher_des:
3000 ti_cipher.alg = cryptocop_alg_des;
3001 break;
3002 case cryptocop_cipher_3des:
3003 ti_cipher.alg = cryptocop_alg_3des;
3004 break;
3005 case cryptocop_cipher_aes:
3006 ti_cipher.alg = cryptocop_alg_aes;
3007 break;
3008 default:
3009 DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3010 return -EINVAL;
3011 };
3012 DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3013 copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3014 ti_cipher.keylen = sop.keylen;
3015 switch (sop.cmode){
3016 case cryptocop_cipher_mode_cbc:
3017 case cryptocop_cipher_mode_ecb:
3018 ti_cipher.cipher_mode = sop.cmode;
3019 break;
3020 default:
3021 DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3022 return -EINVAL;
3023 }
3024 DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3025 switch (sop.des3_mode){
3026 case cryptocop_3des_eee:
3027 case cryptocop_3des_eed:
3028 case cryptocop_3des_ede:
3029 case cryptocop_3des_edd:
3030 case cryptocop_3des_dee:
3031 case cryptocop_3des_ded:
3032 case cryptocop_3des_dde:
3033 case cryptocop_3des_ddd:
3034 ti_cipher.tdes_mode = sop.des3_mode;
3035 break;
3036 default:
3037 DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3038 return -EINVAL;
3039 }
3040 ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3041 ti_cipher.next = tis;
3042 tis = &ti_cipher;
3043 } /* if (sop.cipher != cryptocop_cipher_none) */
3044 if (sop.digest != cryptocop_digest_none){
3045 DEBUG(printk("setting digest transform\n"));
3046 switch (sop.digest){
3047 case cryptocop_digest_md5:
3048 ti_digest.alg = cryptocop_alg_md5;
3049 break;
3050 case cryptocop_digest_sha1:
3051 ti_digest.alg = cryptocop_alg_sha1;
3052 break;
3053 default:
3054 DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3055 return -EINVAL;
3056 }
3057 ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3058 ti_digest.next = tis;
3059 tis = &ti_digest;
3060 } /* if (sop.digest != cryptocop_digest_none) */
3061 if (sop.csum != cryptocop_csum_none){
3062 DEBUG(printk("setting csum transform\n"));
3063 switch (sop.csum){
3064 case cryptocop_csum_le:
3065 case cryptocop_csum_be:
3066 ti_csum.csum_mode = sop.csum;
3067 break;
3068 default:
3069 DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3070 return -EINVAL;
3071 }
3072 ti_csum.alg = cryptocop_alg_csum;
3073 ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3074 ti_csum.next = tis;
3075 tis = &ti_csum;
3076 } /* (sop.csum != cryptocop_csum_none) */
3077 dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3078 if (!dev){
3079 DEBUG_API(printk("create session, alloc dev\n"));
3080 return -ENOMEM;
3081 }
3082
3083 err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3084 DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3085
3086 if (err) {
3087 kfree(dev);
3088 return err;
3089 }
3090 sess_op->ses_id = sid;
3091 dev->sid = sid;
3092 dev->next = filp->private_data;
3093 filp->private_data = dev;
3094
3095 return 0;
3096 }
3097
3098 static long cryptocop_ioctl_unlocked(struct inode *inode,
3099 struct file *filp, unsigned int cmd, unsigned long arg)
3100 {
3101 int err = 0;
3102 if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3103 DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3104 return -ENOTTY;
3105 }
3106 if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3107 return -ENOTTY;
3108 }
3109 /* Access check of the argument. Some commands, e.g. create session and process op,
3110 needs additional checks. Those are handled in the command handling functions. */
3111 if (_IOC_DIR(cmd) & _IOC_READ)
3112 err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3113 else if (_IOC_DIR(cmd) & _IOC_WRITE)
3114 err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3115 if (err) return -EFAULT;
3116
3117 switch (cmd) {
3118 case CRYPTOCOP_IO_CREATE_SESSION:
3119 return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3120 case CRYPTOCOP_IO_CLOSE_SESSION:
3121 return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3122 case CRYPTOCOP_IO_PROCESS_OP:
3123 return cryptocop_ioctl_process(inode, filp, cmd, arg);
3124 default:
3125 DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3126 return -ENOTTY;
3127 }
3128 return 0;
3129 }
3130
3131 static long
3132 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3133 {
3134 long ret;
3135
3136 mutex_lock(&cryptocop_mutex);
3137 ret = cryptocop_ioctl_unlocked(file_inode(filp), filp, cmd, arg);
3138 mutex_unlock(&cryptocop_mutex);
3139
3140 return ret;
3141 }
3142
3143
3144 #ifdef LDEBUG
3145 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3146 {
3147 struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3148 struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3149 int i;
3150
3151 printk("print_dma_descriptors start\n");
3152
3153 printk("iop:\n");
3154 printk("\tsid: 0x%lld\n", iop->sid);
3155
3156 printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3157 printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3158 printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3159 printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3160
3161 printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3162 printk("\tnext: 0x%p\n"
3163 "\tsaved_data: 0x%p\n"
3164 "\tsaved_data_buf: 0x%p\n",
3165 iop->ctx_out.next,
3166 iop->ctx_out.saved_data,
3167 iop->ctx_out.saved_data_buf);
3168
3169 printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3170 printk("\tnext: 0x%p\n"
3171 "\tsaved_data: 0x%p\n"
3172 "\tsaved_data_buf: 0x%p\n",
3173 iop->ctx_in.next,
3174 iop->ctx_in.saved_data,
3175 iop->ctx_in.saved_data_buf);
3176
3177 i = 0;
3178 while (cdesc_out) {
3179 dma_descr_data *td;
3180 printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3181 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3182 td = cdesc_out->dma_descr;
3183 printk("\n\tbuf: 0x%p\n"
3184 "\tafter: 0x%p\n"
3185 "\tmd: 0x%04x\n"
3186 "\tnext: 0x%p\n",
3187 td->buf,
3188 td->after,
3189 td->md,
3190 td->next);
3191 printk("flags:\n"
3192 "\twait:\t%d\n"
3193 "\teol:\t%d\n"
3194 "\touteop:\t%d\n"
3195 "\tineop:\t%d\n"
3196 "\tintr:\t%d\n",
3197 td->wait,
3198 td->eol,
3199 td->out_eop,
3200 td->in_eop,
3201 td->intr);
3202 cdesc_out = cdesc_out->next;
3203 i++;
3204 }
3205 i = 0;
3206 while (cdesc_in) {
3207 dma_descr_data *td;
3208 printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3209 printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3210 td = cdesc_in->dma_descr;
3211 printk("\n\tbuf: 0x%p\n"
3212 "\tafter: 0x%p\n"
3213 "\tmd: 0x%04x\n"
3214 "\tnext: 0x%p\n",
3215 td->buf,
3216 td->after,
3217 td->md,
3218 td->next);
3219 printk("flags:\n"
3220 "\twait:\t%d\n"
3221 "\teol:\t%d\n"
3222 "\touteop:\t%d\n"
3223 "\tineop:\t%d\n"
3224 "\tintr:\t%d\n",
3225 td->wait,
3226 td->eol,
3227 td->out_eop,
3228 td->in_eop,
3229 td->intr);
3230 cdesc_in = cdesc_in->next;
3231 i++;
3232 }
3233
3234 printk("print_dma_descriptors end\n");
3235 }
3236
3237
3238 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3239 {
3240 printk("print_strcop_crypto_op, 0x%p\n", cop);
3241
3242 /* Indata. */
3243 printk("indata=0x%p\n"
3244 "inlen=%d\n"
3245 "do_cipher=%d\n"
3246 "decrypt=%d\n"
3247 "cipher_explicit=%d\n"
3248 "cipher_start=%d\n"
3249 "cipher_len=%d\n"
3250 "outdata=0x%p\n"
3251 "outlen=%d\n",
3252 cop->indata,
3253 cop->inlen,
3254 cop->do_cipher,
3255 cop->decrypt,
3256 cop->cipher_explicit,
3257 cop->cipher_start,
3258 cop->cipher_len,
3259 cop->cipher_outdata,
3260 cop->cipher_outlen);
3261
3262 printk("do_digest=%d\n"
3263 "digest_start=%d\n"
3264 "digest_len=%d\n",
3265 cop->do_digest,
3266 cop->digest_start,
3267 cop->digest_len);
3268
3269 printk("do_csum=%d\n"
3270 "csum_start=%d\n"
3271 "csum_len=%d\n",
3272 cop->do_csum,
3273 cop->csum_start,
3274 cop->csum_len);
3275 }
3276
3277 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3278 {
3279 struct cryptocop_desc *d;
3280 struct cryptocop_tfrm_cfg *tc;
3281 struct cryptocop_desc_cfg *dc;
3282 int i;
3283
3284 printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3285 printk("sid: %lld\n", cop->sid);
3286 printk("operation_status=%d\n"
3287 "use_dmalists=%d\n"
3288 "in_interrupt=%d\n"
3289 "fast_callback=%d\n",
3290 cop->operation_status,
3291 cop->use_dmalists,
3292 cop->in_interrupt,
3293 cop->fast_callback);
3294
3295 if (cop->use_dmalists){
3296 print_user_dma_lists(&cop->list_op);
3297 } else {
3298 printk("cop->tfrm_op\n"
3299 "tfrm_cfg=0x%p\n"
3300 "desc=0x%p\n"
3301 "indata=0x%p\n"
3302 "incount=%d\n"
3303 "inlen=%d\n"
3304 "outdata=0x%p\n"
3305 "outcount=%d\n"
3306 "outlen=%d\n\n",
3307 cop->tfrm_op.tfrm_cfg,
3308 cop->tfrm_op.desc,
3309 cop->tfrm_op.indata,
3310 cop->tfrm_op.incount,
3311 cop->tfrm_op.inlen,
3312 cop->tfrm_op.outdata,
3313 cop->tfrm_op.outcount,
3314 cop->tfrm_op.outlen);
3315
3316 tc = cop->tfrm_op.tfrm_cfg;
3317 while (tc){
3318 printk("tfrm_cfg, 0x%p\n"
3319 "tid=%d\n"
3320 "flags=%d\n"
3321 "inject_ix=%d\n"
3322 "next=0x%p\n",
3323 tc,
3324 tc->tid,
3325 tc->flags,
3326 tc->inject_ix,
3327 tc->next);
3328 tc = tc->next;
3329 }
3330 d = cop->tfrm_op.desc;
3331 while (d){
3332 printk("\n======================desc, 0x%p\n"
3333 "length=%d\n"
3334 "cfg=0x%p\n"
3335 "next=0x%p\n",
3336 d,
3337 d->length,
3338 d->cfg,
3339 d->next);
3340 dc = d->cfg;
3341 while (dc){
3342 printk("=========desc_cfg, 0x%p\n"
3343 "tid=%d\n"
3344 "src=%d\n"
3345 "last=%d\n"
3346 "next=0x%p\n",
3347 dc,
3348 dc->tid,
3349 dc->src,
3350 dc->last,
3351 dc->next);
3352 dc = dc->next;
3353 }
3354 d = d->next;
3355 }
3356 printk("\n====iniov\n");
3357 for (i = 0; i < cop->tfrm_op.incount; i++){
3358 printk("indata[%d]\n"
3359 "base=0x%p\n"
3360 "len=%d\n",
3361 i,
3362 cop->tfrm_op.indata[i].iov_base,
3363 cop->tfrm_op.indata[i].iov_len);
3364 }
3365 printk("\n====outiov\n");
3366 for (i = 0; i < cop->tfrm_op.outcount; i++){
3367 printk("outdata[%d]\n"
3368 "base=0x%p\n"
3369 "len=%d\n",
3370 i,
3371 cop->tfrm_op.outdata[i].iov_base,
3372 cop->tfrm_op.outdata[i].iov_len);
3373 }
3374 }
3375 printk("------------end print_cryptocop_operation\n");
3376 }
3377
3378
3379 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3380 {
3381 dma_descr_data *dd;
3382 int i;
3383
3384 printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3385
3386 printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3387 printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3388
3389 printk("##############outlist\n");
3390 dd = phys_to_virt((unsigned long int)dma_op->outlist);
3391 i = 0;
3392 while (dd != NULL) {
3393 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3394 printk("\n\tbuf: 0x%p\n"
3395 "\tafter: 0x%p\n"
3396 "\tmd: 0x%04x\n"
3397 "\tnext: 0x%p\n",
3398 dd->buf,
3399 dd->after,
3400 dd->md,
3401 dd->next);
3402 printk("flags:\n"
3403 "\twait:\t%d\n"
3404 "\teol:\t%d\n"
3405 "\touteop:\t%d\n"
3406 "\tineop:\t%d\n"
3407 "\tintr:\t%d\n",
3408 dd->wait,
3409 dd->eol,
3410 dd->out_eop,
3411 dd->in_eop,
3412 dd->intr);
3413 if (dd->eol)
3414 dd = NULL;
3415 else
3416 dd = phys_to_virt((unsigned long int)dd->next);
3417 ++i;
3418 }
3419
3420 printk("##############inlist\n");
3421 dd = phys_to_virt((unsigned long int)dma_op->inlist);
3422 i = 0;
3423 while (dd != NULL) {
3424 printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3425 printk("\n\tbuf: 0x%p\n"
3426 "\tafter: 0x%p\n"
3427 "\tmd: 0x%04x\n"
3428 "\tnext: 0x%p\n",
3429 dd->buf,
3430 dd->after,
3431 dd->md,
3432 dd->next);
3433 printk("flags:\n"
3434 "\twait:\t%d\n"
3435 "\teol:\t%d\n"
3436 "\touteop:\t%d\n"
3437 "\tineop:\t%d\n"
3438 "\tintr:\t%d\n",
3439 dd->wait,
3440 dd->eol,
3441 dd->out_eop,
3442 dd->in_eop,
3443 dd->intr);
3444 if (dd->eol)
3445 dd = NULL;
3446 else
3447 dd = phys_to_virt((unsigned long int)dd->next);
3448 ++i;
3449 }
3450 }
3451
3452
3453 static void print_lock_status(void)
3454 {
3455 printk("**********************print_lock_status\n");
3456 printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3457 printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3458 printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3459 printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3460 printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3461 printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3462 }
3463 #endif /* LDEBUG */
3464
3465
3466 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3467
3468 static int init_stream_coprocessor(void)
3469 {
3470 int err;
3471 int i;
3472 static int initialized = 0;
3473
3474 if (initialized)
3475 return 0;
3476
3477 initialized = 1;
3478
3479 printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3480
3481 err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3482 if (err < 0) {
3483 printk(KERN_ERR "stream co-processor: could not get major number.\n");
3484 return err;
3485 }
3486
3487 err = init_cryptocop();
3488 if (err) {
3489 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3490 return err;
3491 }
3492 err = cryptocop_job_queue_init();
3493 if (err) {
3494 release_cryptocop();
3495 (void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3496 return err;
3497 }
3498 /* Init the descriptor pool. */
3499 for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3500 descr_pool[i].from_pool = 1;
3501 descr_pool[i].next = &descr_pool[i + 1];
3502 }
3503 descr_pool[i].from_pool = 1;
3504 descr_pool[i].next = NULL;
3505 descr_pool_free_list = &descr_pool[0];
3506 descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3507
3508 spin_lock_init(&cryptocop_completed_jobs_lock);
3509 spin_lock_init(&cryptocop_job_queue_lock);
3510 spin_lock_init(&descr_pool_lock);
3511 spin_lock_init(&cryptocop_sessions_lock);
3512 spin_lock_init(&running_job_lock);
3513 spin_lock_init(&cryptocop_process_lock);
3514
3515 cryptocop_sessions = NULL;
3516 next_sid = 1;
3517
3518 cryptocop_running_job = NULL;
3519
3520 printk("stream co-processor: init done.\n");
3521 return 0;
3522 }
3523
3524 static void __exit exit_stream_coprocessor(void)
3525 {
3526 release_cryptocop();
3527 cryptocop_job_queue_close();
3528 }
3529
3530 module_init(init_stream_coprocessor);
3531 module_exit(exit_stream_coprocessor);
3532
CRIS linux kernel tree