?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
PM / sleep: Asynchronous threads for suspend_noirq
[linux/fpc-iii.git] / drivers / crypto / ccp / ccp-ops.c
blob71ed3ade7e12c58ef6a4cd780d9d884d4d2e7f13
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/pci_ids.h>
17 #include <linux/kthread.h>
18 #include <linux/sched.h>
19 #include <linux/interrupt.h>
20 #include <linux/spinlock.h>
21 #include <linux/mutex.h>
22 #include <linux/delay.h>
23 #include <linux/ccp.h>
24 #include <linux/scatterlist.h>
25 #include <crypto/scatterwalk.h>
26
27 #include "ccp-dev.h"
28
29
30 enum ccp_memtype {
31 CCP_MEMTYPE_SYSTEM = 0,
32 CCP_MEMTYPE_KSB,
33 CCP_MEMTYPE_LOCAL,
34 CCP_MEMTYPE__LAST,
35 };
36
37 struct ccp_dma_info {
38 dma_addr_t address;
39 unsigned int offset;
40 unsigned int length;
41 enum dma_data_direction dir;
42 };
43
44 struct ccp_dm_workarea {
45 struct device *dev;
46 struct dma_pool *dma_pool;
47 unsigned int length;
48
49 u8 *address;
50 struct ccp_dma_info dma;
51 };
52
53 struct ccp_sg_workarea {
54 struct scatterlist *sg;
55 unsigned int nents;
56 unsigned int length;
57
58 struct scatterlist *dma_sg;
59 struct device *dma_dev;
60 unsigned int dma_count;
61 enum dma_data_direction dma_dir;
62
63 unsigned int sg_used;
64
65 u64 bytes_left;
66 };
67
68 struct ccp_data {
69 struct ccp_sg_workarea sg_wa;
70 struct ccp_dm_workarea dm_wa;
71 };
72
73 struct ccp_mem {
74 enum ccp_memtype type;
75 union {
76 struct ccp_dma_info dma;
77 u32 ksb;
78 } u;
79 };
80
81 struct ccp_aes_op {
82 enum ccp_aes_type type;
83 enum ccp_aes_mode mode;
84 enum ccp_aes_action action;
85 };
86
87 struct ccp_xts_aes_op {
88 enum ccp_aes_action action;
89 enum ccp_xts_aes_unit_size unit_size;
90 };
91
92 struct ccp_sha_op {
93 enum ccp_sha_type type;
94 u64 msg_bits;
95 };
96
97 struct ccp_rsa_op {
98 u32 mod_size;
99 u32 input_len;
100 };
101
102 struct ccp_passthru_op {
103 enum ccp_passthru_bitwise bit_mod;
104 enum ccp_passthru_byteswap byte_swap;
105 };
106
107 struct ccp_ecc_op {
108 enum ccp_ecc_function function;
109 };
110
111 struct ccp_op {
112 struct ccp_cmd_queue *cmd_q;
113
114 u32 jobid;
115 u32 ioc;
116 u32 soc;
117 u32 ksb_key;
118 u32 ksb_ctx;
119 u32 init;
120 u32 eom;
121
122 struct ccp_mem src;
123 struct ccp_mem dst;
124
125 union {
126 struct ccp_aes_op aes;
127 struct ccp_xts_aes_op xts;
128 struct ccp_sha_op sha;
129 struct ccp_rsa_op rsa;
130 struct ccp_passthru_op passthru;
131 struct ccp_ecc_op ecc;
132 } u;
133 };
134
135 /* The CCP cannot perform zero-length sha operations so the caller
136 * is required to buffer data for the final operation. However, a
137 * sha operation for a message with a total length of zero is valid
138 * so known values are required to supply the result.
139 */
140 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = {
141 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
142 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
143 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00,
144 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
145 };
146
147 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = {
148 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9,
149 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4,
150 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a,
151 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00,
152 };
153
154 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = {
155 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
156 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
157 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
158 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55,
159 };
160
161 static u32 ccp_addr_lo(struct ccp_dma_info *info)
162 {
163 return lower_32_bits(info->address + info->offset);
164 }
165
166 static u32 ccp_addr_hi(struct ccp_dma_info *info)
167 {
168 return upper_32_bits(info->address + info->offset) & 0x0000ffff;
169 }
170
171 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
172 {
173 struct ccp_cmd_queue *cmd_q = op->cmd_q;
174 struct ccp_device *ccp = cmd_q->ccp;
175 void __iomem *cr_addr;
176 u32 cr0, cmd;
177 unsigned int i;
178 int ret = 0;
179
180 /* We could read a status register to see how many free slots
181 * are actually available, but reading that register resets it
182 * and you could lose some error information.
183 */
184 cmd_q->free_slots--;
185
186 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
187 | (op->jobid << REQ0_JOBID_SHIFT)
188 | REQ0_WAIT_FOR_WRITE;
189
190 if (op->soc)
191 cr0 |= REQ0_STOP_ON_COMPLETE
192 | REQ0_INT_ON_COMPLETE;
193
194 if (op->ioc || !cmd_q->free_slots)
195 cr0 |= REQ0_INT_ON_COMPLETE;
196
197 /* Start at CMD_REQ1 */
198 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
199
200 mutex_lock(&ccp->req_mutex);
201
202 /* Write CMD_REQ1 through CMD_REQx first */
203 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
204 iowrite32(*(cr + i), cr_addr);
205
206 /* Tell the CCP to start */
207 wmb();
208 iowrite32(cr0, ccp->io_regs + CMD_REQ0);
209
210 mutex_unlock(&ccp->req_mutex);
211
212 if (cr0 & REQ0_INT_ON_COMPLETE) {
213 /* Wait for the job to complete */
214 ret = wait_event_interruptible(cmd_q->int_queue,
215 cmd_q->int_rcvd);
216 if (ret || cmd_q->cmd_error) {
217 /* On error delete all related jobs from the queue */
218 cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
219 | op->jobid;
220
221 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
222
223 if (!ret)
224 ret = -EIO;
225 } else if (op->soc) {
226 /* Delete just head job from the queue on SoC */
227 cmd = DEL_Q_ACTIVE
228 | (cmd_q->id << DEL_Q_ID_SHIFT)
229 | op->jobid;
230
231 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
232 }
233
234 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
235
236 cmd_q->int_rcvd = 0;
237 }
238
239 return ret;
240 }
241
242 static int ccp_perform_aes(struct ccp_op *op)
243 {
244 u32 cr[6];
245
246 /* Fill out the register contents for REQ1 through REQ6 */
247 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
248 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
249 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
250 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
251 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
252 cr[1] = op->src.u.dma.length - 1;
253 cr[2] = ccp_addr_lo(&op->src.u.dma);
254 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
255 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
256 | ccp_addr_hi(&op->src.u.dma);
257 cr[4] = ccp_addr_lo(&op->dst.u.dma);
258 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
259 | ccp_addr_hi(&op->dst.u.dma);
260
261 if (op->u.aes.mode == CCP_AES_MODE_CFB)
262 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
263
264 if (op->eom)
265 cr[0] |= REQ1_EOM;
266
267 if (op->init)
268 cr[0] |= REQ1_INIT;
269
270 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
271 }
272
273 static int ccp_perform_xts_aes(struct ccp_op *op)
274 {
275 u32 cr[6];
276
277 /* Fill out the register contents for REQ1 through REQ6 */
278 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
279 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
280 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
281 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
282 cr[1] = op->src.u.dma.length - 1;
283 cr[2] = ccp_addr_lo(&op->src.u.dma);
284 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
285 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
286 | ccp_addr_hi(&op->src.u.dma);
287 cr[4] = ccp_addr_lo(&op->dst.u.dma);
288 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
289 | ccp_addr_hi(&op->dst.u.dma);
290
291 if (op->eom)
292 cr[0] |= REQ1_EOM;
293
294 if (op->init)
295 cr[0] |= REQ1_INIT;
296
297 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
298 }
299
300 static int ccp_perform_sha(struct ccp_op *op)
301 {
302 u32 cr[6];
303
304 /* Fill out the register contents for REQ1 through REQ6 */
305 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
306 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
307 | REQ1_INIT;
308 cr[1] = op->src.u.dma.length - 1;
309 cr[2] = ccp_addr_lo(&op->src.u.dma);
310 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
311 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
312 | ccp_addr_hi(&op->src.u.dma);
313
314 if (op->eom) {
315 cr[0] |= REQ1_EOM;
316 cr[4] = lower_32_bits(op->u.sha.msg_bits);
317 cr[5] = upper_32_bits(op->u.sha.msg_bits);
318 } else {
319 cr[4] = 0;
320 cr[5] = 0;
321 }
322
323 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
324 }
325
326 static int ccp_perform_rsa(struct ccp_op *op)
327 {
328 u32 cr[6];
329
330 /* Fill out the register contents for REQ1 through REQ6 */
331 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
332 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
333 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
334 | REQ1_EOM;
335 cr[1] = op->u.rsa.input_len - 1;
336 cr[2] = ccp_addr_lo(&op->src.u.dma);
337 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
338 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
339 | ccp_addr_hi(&op->src.u.dma);
340 cr[4] = ccp_addr_lo(&op->dst.u.dma);
341 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
342 | ccp_addr_hi(&op->dst.u.dma);
343
344 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
345 }
346
347 static int ccp_perform_passthru(struct ccp_op *op)
348 {
349 u32 cr[6];
350
351 /* Fill out the register contents for REQ1 through REQ6 */
352 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
353 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
354 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
355
356 if (op->src.type == CCP_MEMTYPE_SYSTEM)
357 cr[1] = op->src.u.dma.length - 1;
358 else
359 cr[1] = op->dst.u.dma.length - 1;
360
361 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
362 cr[2] = ccp_addr_lo(&op->src.u.dma);
363 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
364 | ccp_addr_hi(&op->src.u.dma);
365
366 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
367 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
368 } else {
369 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
370 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
371 }
372
373 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
374 cr[4] = ccp_addr_lo(&op->dst.u.dma);
375 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
376 | ccp_addr_hi(&op->dst.u.dma);
377 } else {
378 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
379 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
380 }
381
382 if (op->eom)
383 cr[0] |= REQ1_EOM;
384
385 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
386 }
387
388 static int ccp_perform_ecc(struct ccp_op *op)
389 {
390 u32 cr[6];
391
392 /* Fill out the register contents for REQ1 through REQ6 */
393 cr[0] = REQ1_ECC_AFFINE_CONVERT
394 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
395 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
396 | REQ1_EOM;
397 cr[1] = op->src.u.dma.length - 1;
398 cr[2] = ccp_addr_lo(&op->src.u.dma);
399 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
400 | ccp_addr_hi(&op->src.u.dma);
401 cr[4] = ccp_addr_lo(&op->dst.u.dma);
402 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
403 | ccp_addr_hi(&op->dst.u.dma);
404
405 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
406 }
407
408 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
409 {
410 int start;
411
412 for (;;) {
413 mutex_lock(&ccp->ksb_mutex);
414
415 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
416 ccp->ksb_count,
417 ccp->ksb_start,
418 count, 0);
419 if (start <= ccp->ksb_count) {
420 bitmap_set(ccp->ksb, start, count);
421
422 mutex_unlock(&ccp->ksb_mutex);
423 break;
424 }
425
426 ccp->ksb_avail = 0;
427
428 mutex_unlock(&ccp->ksb_mutex);
429
430 /* Wait for KSB entries to become available */
431 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
432 return 0;
433 }
434
435 return KSB_START + start;
436 }
437
438 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
439 unsigned int count)
440 {
441 if (!start)
442 return;
443
444 mutex_lock(&ccp->ksb_mutex);
445
446 bitmap_clear(ccp->ksb, start - KSB_START, count);
447
448 ccp->ksb_avail = 1;
449
450 mutex_unlock(&ccp->ksb_mutex);
451
452 wake_up_interruptible_all(&ccp->ksb_queue);
453 }
454
455 static u32 ccp_gen_jobid(struct ccp_device *ccp)
456 {
457 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
458 }
459
460 static void ccp_sg_free(struct ccp_sg_workarea *wa)
461 {
462 if (wa->dma_count)
463 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
464
465 wa->dma_count = 0;
466 }
467
468 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
469 struct scatterlist *sg, u64 len,
470 enum dma_data_direction dma_dir)
471 {
472 memset(wa, 0, sizeof(*wa));
473
474 wa->sg = sg;
475 if (!sg)
476 return 0;
477
478 wa->nents = sg_nents(sg);
479 wa->length = sg->length;
480 wa->bytes_left = len;
481 wa->sg_used = 0;
482
483 if (len == 0)
484 return 0;
485
486 if (dma_dir == DMA_NONE)
487 return 0;
488
489 wa->dma_sg = sg;
490 wa->dma_dev = dev;
491 wa->dma_dir = dma_dir;
492 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
493 if (!wa->dma_count)
494 return -ENOMEM;
495
496
497 return 0;
498 }
499
500 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
501 {
502 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
503
504 if (!wa->sg)
505 return;
506
507 wa->sg_used += nbytes;
508 wa->bytes_left -= nbytes;
509 if (wa->sg_used == wa->sg->length) {
510 wa->sg = sg_next(wa->sg);
511 wa->sg_used = 0;
512 }
513 }
514
515 static void ccp_dm_free(struct ccp_dm_workarea *wa)
516 {
517 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
518 if (wa->address)
519 dma_pool_free(wa->dma_pool, wa->address,
520 wa->dma.address);
521 } else {
522 if (wa->dma.address)
523 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
524 wa->dma.dir);
525 kfree(wa->address);
526 }
527
528 wa->address = NULL;
529 wa->dma.address = 0;
530 }
531
532 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
533 struct ccp_cmd_queue *cmd_q,
534 unsigned int len,
535 enum dma_data_direction dir)
536 {
537 memset(wa, 0, sizeof(*wa));
538
539 if (!len)
540 return 0;
541
542 wa->dev = cmd_q->ccp->dev;
543 wa->length = len;
544
545 if (len <= CCP_DMAPOOL_MAX_SIZE) {
546 wa->dma_pool = cmd_q->dma_pool;
547
548 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
549 &wa->dma.address);
550 if (!wa->address)
551 return -ENOMEM;
552
553 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
554
555 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
556 } else {
557 wa->address = kzalloc(len, GFP_KERNEL);
558 if (!wa->address)
559 return -ENOMEM;
560
561 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
562 dir);
563 if (!wa->dma.address)
564 return -ENOMEM;
565
566 wa->dma.length = len;
567 }
568 wa->dma.dir = dir;
569
570 return 0;
571 }
572
573 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
574 struct scatterlist *sg, unsigned int sg_offset,
575 unsigned int len)
576 {
577 WARN_ON(!wa->address);
578
579 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
580 0);
581 }
582
583 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
584 struct scatterlist *sg, unsigned int sg_offset,
585 unsigned int len)
586 {
587 WARN_ON(!wa->address);
588
589 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
590 1);
591 }
592
593 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
594 struct scatterlist *sg,
595 unsigned int len, unsigned int se_len,
596 bool sign_extend)
597 {
598 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
599 u8 buffer[CCP_REVERSE_BUF_SIZE];
600
601 BUG_ON(se_len > sizeof(buffer));
602
603 sg_offset = len;
604 dm_offset = 0;
605 nbytes = len;
606 while (nbytes) {
607 ksb_len = min_t(unsigned int, nbytes, se_len);
608 sg_offset -= ksb_len;
609
610 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
611 for (i = 0; i < ksb_len; i++)
612 wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
613
614 dm_offset += ksb_len;
615 nbytes -= ksb_len;
616
617 if ((ksb_len != se_len) && sign_extend) {
618 /* Must sign-extend to nearest sign-extend length */
619 if (wa->address[dm_offset - 1] & 0x80)
620 memset(wa->address + dm_offset, 0xff,
621 se_len - ksb_len);
622 }
623 }
624 }
625
626 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
627 struct scatterlist *sg,
628 unsigned int len)
629 {
630 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
631 u8 buffer[CCP_REVERSE_BUF_SIZE];
632
633 sg_offset = 0;
634 dm_offset = len;
635 nbytes = len;
636 while (nbytes) {
637 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
638 dm_offset -= ksb_len;
639
640 for (i = 0; i < ksb_len; i++)
641 buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
642 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
643
644 sg_offset += ksb_len;
645 nbytes -= ksb_len;
646 }
647 }
648
649 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
650 {
651 ccp_dm_free(&data->dm_wa);
652 ccp_sg_free(&data->sg_wa);
653 }
654
655 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
656 struct scatterlist *sg, u64 sg_len,
657 unsigned int dm_len,
658 enum dma_data_direction dir)
659 {
660 int ret;
661
662 memset(data, 0, sizeof(*data));
663
664 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
665 dir);
666 if (ret)
667 goto e_err;
668
669 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
670 if (ret)
671 goto e_err;
672
673 return 0;
674
675 e_err:
676 ccp_free_data(data, cmd_q);
677
678 return ret;
679 }
680
681 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
682 {
683 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
684 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
685 unsigned int buf_count, nbytes;
686
687 /* Clear the buffer if setting it */
688 if (!from)
689 memset(dm_wa->address, 0, dm_wa->length);
690
691 if (!sg_wa->sg)
692 return 0;
693
694 /* Perform the copy operation
695 * nbytes will always be <= UINT_MAX because dm_wa->length is
696 * an unsigned int
697 */
698 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
699 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
700 nbytes, from);
701
702 /* Update the structures and generate the count */
703 buf_count = 0;
704 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
705 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
706 dm_wa->length - buf_count);
707 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
708
709 buf_count += nbytes;
710 ccp_update_sg_workarea(sg_wa, nbytes);
711 }
712
713 return buf_count;
714 }
715
716 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
717 {
718 return ccp_queue_buf(data, 0);
719 }
720
721 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
722 {
723 return ccp_queue_buf(data, 1);
724 }
725
726 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
727 struct ccp_op *op, unsigned int block_size,
728 bool blocksize_op)
729 {
730 unsigned int sg_src_len, sg_dst_len, op_len;
731
732 /* The CCP can only DMA from/to one address each per operation. This
733 * requires that we find the smallest DMA area between the source
734 * and destination. The resulting len values will always be <= UINT_MAX
735 * because the dma length is an unsigned int.
736 */
737 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
738 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
739
740 if (dst) {
741 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
742 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
743 op_len = min(sg_src_len, sg_dst_len);
744 } else
745 op_len = sg_src_len;
746
747 /* The data operation length will be at least block_size in length
748 * or the smaller of available sg room remaining for the source or
749 * the destination
750 */
751 op_len = max(op_len, block_size);
752
753 /* Unless we have to buffer data, there's no reason to wait */
754 op->soc = 0;
755
756 if (sg_src_len < block_size) {
757 /* Not enough data in the sg element, so it
758 * needs to be buffered into a blocksize chunk
759 */
760 int cp_len = ccp_fill_queue_buf(src);
761
762 op->soc = 1;
763 op->src.u.dma.address = src->dm_wa.dma.address;
764 op->src.u.dma.offset = 0;
765 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
766 } else {
767 /* Enough data in the sg element, but we need to
768 * adjust for any previously copied data
769 */
770 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
771 op->src.u.dma.offset = src->sg_wa.sg_used;
772 op->src.u.dma.length = op_len & ~(block_size - 1);
773
774 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
775 }
776
777 if (dst) {
778 if (sg_dst_len < block_size) {
779 /* Not enough room in the sg element or we're on the
780 * last piece of data (when using padding), so the
781 * output needs to be buffered into a blocksize chunk
782 */
783 op->soc = 1;
784 op->dst.u.dma.address = dst->dm_wa.dma.address;
785 op->dst.u.dma.offset = 0;
786 op->dst.u.dma.length = op->src.u.dma.length;
787 } else {
788 /* Enough room in the sg element, but we need to
789 * adjust for any previously used area
790 */
791 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
792 op->dst.u.dma.offset = dst->sg_wa.sg_used;
793 op->dst.u.dma.length = op->src.u.dma.length;
794 }
795 }
796 }
797
798 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
799 struct ccp_op *op)
800 {
801 op->init = 0;
802
803 if (dst) {
804 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
805 ccp_empty_queue_buf(dst);
806 else
807 ccp_update_sg_workarea(&dst->sg_wa,
808 op->dst.u.dma.length);
809 }
810 }
811
812 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
813 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
814 u32 byte_swap, bool from)
815 {
816 struct ccp_op op;
817
818 memset(&op, 0, sizeof(op));
819
820 op.cmd_q = cmd_q;
821 op.jobid = jobid;
822 op.eom = 1;
823
824 if (from) {
825 op.soc = 1;
826 op.src.type = CCP_MEMTYPE_KSB;
827 op.src.u.ksb = ksb;
828 op.dst.type = CCP_MEMTYPE_SYSTEM;
829 op.dst.u.dma.address = wa->dma.address;
830 op.dst.u.dma.length = wa->length;
831 } else {
832 op.src.type = CCP_MEMTYPE_SYSTEM;
833 op.src.u.dma.address = wa->dma.address;
834 op.src.u.dma.length = wa->length;
835 op.dst.type = CCP_MEMTYPE_KSB;
836 op.dst.u.ksb = ksb;
837 }
838
839 op.u.passthru.byte_swap = byte_swap;
840
841 return ccp_perform_passthru(&op);
842 }
843
844 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
845 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
846 u32 byte_swap)
847 {
848 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
849 }
850
851 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
852 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
853 u32 byte_swap)
854 {
855 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
856 }
857
858 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
859 struct ccp_cmd *cmd)
860 {
861 struct ccp_aes_engine *aes = &cmd->u.aes;
862 struct ccp_dm_workarea key, ctx;
863 struct ccp_data src;
864 struct ccp_op op;
865 unsigned int dm_offset;
866 int ret;
867
868 if (!((aes->key_len == AES_KEYSIZE_128) ||
869 (aes->key_len == AES_KEYSIZE_192) ||
870 (aes->key_len == AES_KEYSIZE_256)))
871 return -EINVAL;
872
873 if (aes->src_len & (AES_BLOCK_SIZE - 1))
874 return -EINVAL;
875
876 if (aes->iv_len != AES_BLOCK_SIZE)
877 return -EINVAL;
878
879 if (!aes->key || !aes->iv || !aes->src)
880 return -EINVAL;
881
882 if (aes->cmac_final) {
883 if (aes->cmac_key_len != AES_BLOCK_SIZE)
884 return -EINVAL;
885
886 if (!aes->cmac_key)
887 return -EINVAL;
888 }
889
890 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
891 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
892
893 ret = -EIO;
894 memset(&op, 0, sizeof(op));
895 op.cmd_q = cmd_q;
896 op.jobid = ccp_gen_jobid(cmd_q->ccp);
897 op.ksb_key = cmd_q->ksb_key;
898 op.ksb_ctx = cmd_q->ksb_ctx;
899 op.init = 1;
900 op.u.aes.type = aes->type;
901 op.u.aes.mode = aes->mode;
902 op.u.aes.action = aes->action;
903
904 /* All supported key sizes fit in a single (32-byte) KSB entry
905 * and must be in little endian format. Use the 256-bit byte
906 * swap passthru option to convert from big endian to little
907 * endian.
908 */
909 ret = ccp_init_dm_workarea(&key, cmd_q,
910 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
911 DMA_TO_DEVICE);
912 if (ret)
913 return ret;
914
915 dm_offset = CCP_KSB_BYTES - aes->key_len;
916 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
917 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
918 CCP_PASSTHRU_BYTESWAP_256BIT);
919 if (ret) {
920 cmd->engine_error = cmd_q->cmd_error;
921 goto e_key;
922 }
923
924 /* The AES context fits in a single (32-byte) KSB entry and
925 * must be in little endian format. Use the 256-bit byte swap
926 * passthru option to convert from big endian to little endian.
927 */
928 ret = ccp_init_dm_workarea(&ctx, cmd_q,
929 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
930 DMA_BIDIRECTIONAL);
931 if (ret)
932 goto e_key;
933
934 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
935 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
936 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
937 CCP_PASSTHRU_BYTESWAP_256BIT);
938 if (ret) {
939 cmd->engine_error = cmd_q->cmd_error;
940 goto e_ctx;
941 }
942
943 /* Send data to the CCP AES engine */
944 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
945 AES_BLOCK_SIZE, DMA_TO_DEVICE);
946 if (ret)
947 goto e_ctx;
948
949 while (src.sg_wa.bytes_left) {
950 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
951 if (aes->cmac_final && !src.sg_wa.bytes_left) {
952 op.eom = 1;
953
954 /* Push the K1/K2 key to the CCP now */
955 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
956 op.ksb_ctx,
957 CCP_PASSTHRU_BYTESWAP_256BIT);
958 if (ret) {
959 cmd->engine_error = cmd_q->cmd_error;
960 goto e_src;
961 }
962
963 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
964 aes->cmac_key_len);
965 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
966 CCP_PASSTHRU_BYTESWAP_256BIT);
967 if (ret) {
968 cmd->engine_error = cmd_q->cmd_error;
969 goto e_src;
970 }
971 }
972
973 ret = ccp_perform_aes(&op);
974 if (ret) {
975 cmd->engine_error = cmd_q->cmd_error;
976 goto e_src;
977 }
978
979 ccp_process_data(&src, NULL, &op);
980 }
981
982 /* Retrieve the AES context - convert from LE to BE using
983 * 32-byte (256-bit) byteswapping
984 */
985 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
986 CCP_PASSTHRU_BYTESWAP_256BIT);
987 if (ret) {
988 cmd->engine_error = cmd_q->cmd_error;
989 goto e_src;
990 }
991
992 /* ...but we only need AES_BLOCK_SIZE bytes */
993 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
994 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
995
996 e_src:
997 ccp_free_data(&src, cmd_q);
998
999 e_ctx:
1000 ccp_dm_free(&ctx);
1001
1002 e_key:
1003 ccp_dm_free(&key);
1004
1005 return ret;
1006 }
1007
1008 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1009 {
1010 struct ccp_aes_engine *aes = &cmd->u.aes;
1011 struct ccp_dm_workarea key, ctx;
1012 struct ccp_data src, dst;
1013 struct ccp_op op;
1014 unsigned int dm_offset;
1015 bool in_place = false;
1016 int ret;
1017
1018 if (aes->mode == CCP_AES_MODE_CMAC)
1019 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1020
1021 if (!((aes->key_len == AES_KEYSIZE_128) ||
1022 (aes->key_len == AES_KEYSIZE_192) ||
1023 (aes->key_len == AES_KEYSIZE_256)))
1024 return -EINVAL;
1025
1026 if (((aes->mode == CCP_AES_MODE_ECB) ||
1027 (aes->mode == CCP_AES_MODE_CBC) ||
1028 (aes->mode == CCP_AES_MODE_CFB)) &&
1029 (aes->src_len & (AES_BLOCK_SIZE - 1)))
1030 return -EINVAL;
1031
1032 if (!aes->key || !aes->src || !aes->dst)
1033 return -EINVAL;
1034
1035 if (aes->mode != CCP_AES_MODE_ECB) {
1036 if (aes->iv_len != AES_BLOCK_SIZE)
1037 return -EINVAL;
1038
1039 if (!aes->iv)
1040 return -EINVAL;
1041 }
1042
1043 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1044 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1045
1046 ret = -EIO;
1047 memset(&op, 0, sizeof(op));
1048 op.cmd_q = cmd_q;
1049 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1050 op.ksb_key = cmd_q->ksb_key;
1051 op.ksb_ctx = cmd_q->ksb_ctx;
1052 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1053 op.u.aes.type = aes->type;
1054 op.u.aes.mode = aes->mode;
1055 op.u.aes.action = aes->action;
1056
1057 /* All supported key sizes fit in a single (32-byte) KSB entry
1058 * and must be in little endian format. Use the 256-bit byte
1059 * swap passthru option to convert from big endian to little
1060 * endian.
1061 */
1062 ret = ccp_init_dm_workarea(&key, cmd_q,
1063 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1064 DMA_TO_DEVICE);
1065 if (ret)
1066 return ret;
1067
1068 dm_offset = CCP_KSB_BYTES - aes->key_len;
1069 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1070 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1071 CCP_PASSTHRU_BYTESWAP_256BIT);
1072 if (ret) {
1073 cmd->engine_error = cmd_q->cmd_error;
1074 goto e_key;
1075 }
1076
1077 /* The AES context fits in a single (32-byte) KSB entry and
1078 * must be in little endian format. Use the 256-bit byte swap
1079 * passthru option to convert from big endian to little endian.
1080 */
1081 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1082 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1083 DMA_BIDIRECTIONAL);
1084 if (ret)
1085 goto e_key;
1086
1087 if (aes->mode != CCP_AES_MODE_ECB) {
1088 /* Load the AES context - conver to LE */
1089 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1090 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1091 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1092 CCP_PASSTHRU_BYTESWAP_256BIT);
1093 if (ret) {
1094 cmd->engine_error = cmd_q->cmd_error;
1095 goto e_ctx;
1096 }
1097 }
1098
1099 /* Prepare the input and output data workareas. For in-place
1100 * operations we need to set the dma direction to BIDIRECTIONAL
1101 * and copy the src workarea to the dst workarea.
1102 */
1103 if (sg_virt(aes->src) == sg_virt(aes->dst))
1104 in_place = true;
1105
1106 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1107 AES_BLOCK_SIZE,
1108 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1109 if (ret)
1110 goto e_ctx;
1111
1112 if (in_place)
1113 dst = src;
1114 else {
1115 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1116 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1117 if (ret)
1118 goto e_src;
1119 }
1120
1121 /* Send data to the CCP AES engine */
1122 while (src.sg_wa.bytes_left) {
1123 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1124 if (!src.sg_wa.bytes_left) {
1125 op.eom = 1;
1126
1127 /* Since we don't retrieve the AES context in ECB
1128 * mode we have to wait for the operation to complete
1129 * on the last piece of data
1130 */
1131 if (aes->mode == CCP_AES_MODE_ECB)
1132 op.soc = 1;
1133 }
1134
1135 ret = ccp_perform_aes(&op);
1136 if (ret) {
1137 cmd->engine_error = cmd_q->cmd_error;
1138 goto e_dst;
1139 }
1140
1141 ccp_process_data(&src, &dst, &op);
1142 }
1143
1144 if (aes->mode != CCP_AES_MODE_ECB) {
1145 /* Retrieve the AES context - convert from LE to BE using
1146 * 32-byte (256-bit) byteswapping
1147 */
1148 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1149 CCP_PASSTHRU_BYTESWAP_256BIT);
1150 if (ret) {
1151 cmd->engine_error = cmd_q->cmd_error;
1152 goto e_dst;
1153 }
1154
1155 /* ...but we only need AES_BLOCK_SIZE bytes */
1156 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1157 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1158 }
1159
1160 e_dst:
1161 if (!in_place)
1162 ccp_free_data(&dst, cmd_q);
1163
1164 e_src:
1165 ccp_free_data(&src, cmd_q);
1166
1167 e_ctx:
1168 ccp_dm_free(&ctx);
1169
1170 e_key:
1171 ccp_dm_free(&key);
1172
1173 return ret;
1174 }
1175
1176 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1177 struct ccp_cmd *cmd)
1178 {
1179 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1180 struct ccp_dm_workarea key, ctx;
1181 struct ccp_data src, dst;
1182 struct ccp_op op;
1183 unsigned int unit_size, dm_offset;
1184 bool in_place = false;
1185 int ret;
1186
1187 switch (xts->unit_size) {
1188 case CCP_XTS_AES_UNIT_SIZE_16:
1189 unit_size = 16;
1190 break;
1191 case CCP_XTS_AES_UNIT_SIZE_512:
1192 unit_size = 512;
1193 break;
1194 case CCP_XTS_AES_UNIT_SIZE_1024:
1195 unit_size = 1024;
1196 break;
1197 case CCP_XTS_AES_UNIT_SIZE_2048:
1198 unit_size = 2048;
1199 break;
1200 case CCP_XTS_AES_UNIT_SIZE_4096:
1201 unit_size = 4096;
1202 break;
1203
1204 default:
1205 return -EINVAL;
1206 }
1207
1208 if (xts->key_len != AES_KEYSIZE_128)
1209 return -EINVAL;
1210
1211 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1212 return -EINVAL;
1213
1214 if (xts->iv_len != AES_BLOCK_SIZE)
1215 return -EINVAL;
1216
1217 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1218 return -EINVAL;
1219
1220 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1221 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1222
1223 ret = -EIO;
1224 memset(&op, 0, sizeof(op));
1225 op.cmd_q = cmd_q;
1226 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1227 op.ksb_key = cmd_q->ksb_key;
1228 op.ksb_ctx = cmd_q->ksb_ctx;
1229 op.init = 1;
1230 op.u.xts.action = xts->action;
1231 op.u.xts.unit_size = xts->unit_size;
1232
1233 /* All supported key sizes fit in a single (32-byte) KSB entry
1234 * and must be in little endian format. Use the 256-bit byte
1235 * swap passthru option to convert from big endian to little
1236 * endian.
1237 */
1238 ret = ccp_init_dm_workarea(&key, cmd_q,
1239 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1240 DMA_TO_DEVICE);
1241 if (ret)
1242 return ret;
1243
1244 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1245 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1246 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1247 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1248 CCP_PASSTHRU_BYTESWAP_256BIT);
1249 if (ret) {
1250 cmd->engine_error = cmd_q->cmd_error;
1251 goto e_key;
1252 }
1253
1254 /* The AES context fits in a single (32-byte) KSB entry and
1255 * for XTS is already in little endian format so no byte swapping
1256 * is needed.
1257 */
1258 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1259 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1260 DMA_BIDIRECTIONAL);
1261 if (ret)
1262 goto e_key;
1263
1264 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1265 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1266 CCP_PASSTHRU_BYTESWAP_NOOP);
1267 if (ret) {
1268 cmd->engine_error = cmd_q->cmd_error;
1269 goto e_ctx;
1270 }
1271
1272 /* Prepare the input and output data workareas. For in-place
1273 * operations we need to set the dma direction to BIDIRECTIONAL
1274 * and copy the src workarea to the dst workarea.
1275 */
1276 if (sg_virt(xts->src) == sg_virt(xts->dst))
1277 in_place = true;
1278
1279 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1280 unit_size,
1281 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1282 if (ret)
1283 goto e_ctx;
1284
1285 if (in_place)
1286 dst = src;
1287 else {
1288 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1289 unit_size, DMA_FROM_DEVICE);
1290 if (ret)
1291 goto e_src;
1292 }
1293
1294 /* Send data to the CCP AES engine */
1295 while (src.sg_wa.bytes_left) {
1296 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1297 if (!src.sg_wa.bytes_left)
1298 op.eom = 1;
1299
1300 ret = ccp_perform_xts_aes(&op);
1301 if (ret) {
1302 cmd->engine_error = cmd_q->cmd_error;
1303 goto e_dst;
1304 }
1305
1306 ccp_process_data(&src, &dst, &op);
1307 }
1308
1309 /* Retrieve the AES context - convert from LE to BE using
1310 * 32-byte (256-bit) byteswapping
1311 */
1312 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1313 CCP_PASSTHRU_BYTESWAP_256BIT);
1314 if (ret) {
1315 cmd->engine_error = cmd_q->cmd_error;
1316 goto e_dst;
1317 }
1318
1319 /* ...but we only need AES_BLOCK_SIZE bytes */
1320 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1321 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1322
1323 e_dst:
1324 if (!in_place)
1325 ccp_free_data(&dst, cmd_q);
1326
1327 e_src:
1328 ccp_free_data(&src, cmd_q);
1329
1330 e_ctx:
1331 ccp_dm_free(&ctx);
1332
1333 e_key:
1334 ccp_dm_free(&key);
1335
1336 return ret;
1337 }
1338
1339 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1340 {
1341 struct ccp_sha_engine *sha = &cmd->u.sha;
1342 struct ccp_dm_workarea ctx;
1343 struct ccp_data src;
1344 struct ccp_op op;
1345 int ret;
1346
1347 if (sha->ctx_len != CCP_SHA_CTXSIZE)
1348 return -EINVAL;
1349
1350 if (!sha->ctx)
1351 return -EINVAL;
1352
1353 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1354 return -EINVAL;
1355
1356 if (!sha->src_len) {
1357 const u8 *sha_zero;
1358
1359 /* Not final, just return */
1360 if (!sha->final)
1361 return 0;
1362
1363 /* CCP can't do a zero length sha operation so the caller
1364 * must buffer the data.
1365 */
1366 if (sha->msg_bits)
1367 return -EINVAL;
1368
1369 /* A sha operation for a message with a total length of zero,
1370 * return known result.
1371 */
1372 switch (sha->type) {
1373 case CCP_SHA_TYPE_1:
1374 sha_zero = ccp_sha1_zero;
1375 break;
1376 case CCP_SHA_TYPE_224:
1377 sha_zero = ccp_sha224_zero;
1378 break;
1379 case CCP_SHA_TYPE_256:
1380 sha_zero = ccp_sha256_zero;
1381 break;
1382 default:
1383 return -EINVAL;
1384 }
1385
1386 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1387 sha->ctx_len, 1);
1388
1389 return 0;
1390 }
1391
1392 if (!sha->src)
1393 return -EINVAL;
1394
1395 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1396
1397 memset(&op, 0, sizeof(op));
1398 op.cmd_q = cmd_q;
1399 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1400 op.ksb_ctx = cmd_q->ksb_ctx;
1401 op.u.sha.type = sha->type;
1402 op.u.sha.msg_bits = sha->msg_bits;
1403
1404 /* The SHA context fits in a single (32-byte) KSB entry and
1405 * must be in little endian format. Use the 256-bit byte swap
1406 * passthru option to convert from big endian to little endian.
1407 */
1408 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1409 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1410 DMA_BIDIRECTIONAL);
1411 if (ret)
1412 return ret;
1413
1414 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1415 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1416 CCP_PASSTHRU_BYTESWAP_256BIT);
1417 if (ret) {
1418 cmd->engine_error = cmd_q->cmd_error;
1419 goto e_ctx;
1420 }
1421
1422 /* Send data to the CCP SHA engine */
1423 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1424 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1425 if (ret)
1426 goto e_ctx;
1427
1428 while (src.sg_wa.bytes_left) {
1429 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1430 if (sha->final && !src.sg_wa.bytes_left)
1431 op.eom = 1;
1432
1433 ret = ccp_perform_sha(&op);
1434 if (ret) {
1435 cmd->engine_error = cmd_q->cmd_error;
1436 goto e_data;
1437 }
1438
1439 ccp_process_data(&src, NULL, &op);
1440 }
1441
1442 /* Retrieve the SHA context - convert from LE to BE using
1443 * 32-byte (256-bit) byteswapping to BE
1444 */
1445 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1446 CCP_PASSTHRU_BYTESWAP_256BIT);
1447 if (ret) {
1448 cmd->engine_error = cmd_q->cmd_error;
1449 goto e_data;
1450 }
1451
1452 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1453
1454 e_data:
1455 ccp_free_data(&src, cmd_q);
1456
1457 e_ctx:
1458 ccp_dm_free(&ctx);
1459
1460 return ret;
1461 }
1462
1463 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1464 {
1465 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1466 struct ccp_dm_workarea exp, src;
1467 struct ccp_data dst;
1468 struct ccp_op op;
1469 unsigned int ksb_count, i_len, o_len;
1470 int ret;
1471
1472 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1473 return -EINVAL;
1474
1475 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1476 return -EINVAL;
1477
1478 /* The RSA modulus must precede the message being acted upon, so
1479 * it must be copied to a DMA area where the message and the
1480 * modulus can be concatenated. Therefore the input buffer
1481 * length required is twice the output buffer length (which
1482 * must be a multiple of 256-bits).
1483 */
1484 o_len = ((rsa->key_size + 255) / 256) * 32;
1485 i_len = o_len * 2;
1486
1487 ksb_count = o_len / CCP_KSB_BYTES;
1488
1489 memset(&op, 0, sizeof(op));
1490 op.cmd_q = cmd_q;
1491 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1492 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1493 if (!op.ksb_key)
1494 return -EIO;
1495
1496 /* The RSA exponent may span multiple (32-byte) KSB entries and must
1497 * be in little endian format. Reverse copy each 32-byte chunk
1498 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1499 * and each byte within that chunk and do not perform any byte swap
1500 * operations on the passthru operation.
1501 */
1502 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1503 if (ret)
1504 goto e_ksb;
1505
1506 ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES,
1507 true);
1508 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1509 CCP_PASSTHRU_BYTESWAP_NOOP);
1510 if (ret) {
1511 cmd->engine_error = cmd_q->cmd_error;
1512 goto e_exp;
1513 }
1514
1515 /* Concatenate the modulus and the message. Both the modulus and
1516 * the operands must be in little endian format. Since the input
1517 * is in big endian format it must be converted.
1518 */
1519 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1520 if (ret)
1521 goto e_exp;
1522
1523 ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES,
1524 true);
1525 src.address += o_len; /* Adjust the address for the copy operation */
1526 ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES,
1527 true);
1528 src.address -= o_len; /* Reset the address to original value */
1529
1530 /* Prepare the output area for the operation */
1531 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1532 o_len, DMA_FROM_DEVICE);
1533 if (ret)
1534 goto e_src;
1535
1536 op.soc = 1;
1537 op.src.u.dma.address = src.dma.address;
1538 op.src.u.dma.offset = 0;
1539 op.src.u.dma.length = i_len;
1540 op.dst.u.dma.address = dst.dm_wa.dma.address;
1541 op.dst.u.dma.offset = 0;
1542 op.dst.u.dma.length = o_len;
1543
1544 op.u.rsa.mod_size = rsa->key_size;
1545 op.u.rsa.input_len = i_len;
1546
1547 ret = ccp_perform_rsa(&op);
1548 if (ret) {
1549 cmd->engine_error = cmd_q->cmd_error;
1550 goto e_dst;
1551 }
1552
1553 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1554
1555 e_dst:
1556 ccp_free_data(&dst, cmd_q);
1557
1558 e_src:
1559 ccp_dm_free(&src);
1560
1561 e_exp:
1562 ccp_dm_free(&exp);
1563
1564 e_ksb:
1565 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1566
1567 return ret;
1568 }
1569
1570 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1571 struct ccp_cmd *cmd)
1572 {
1573 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1574 struct ccp_dm_workarea mask;
1575 struct ccp_data src, dst;
1576 struct ccp_op op;
1577 bool in_place = false;
1578 unsigned int i;
1579 int ret;
1580
1581 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1582 return -EINVAL;
1583
1584 if (!pt->src || !pt->dst)
1585 return -EINVAL;
1586
1587 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1588 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1589 return -EINVAL;
1590 if (!pt->mask)
1591 return -EINVAL;
1592 }
1593
1594 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1595
1596 memset(&op, 0, sizeof(op));
1597 op.cmd_q = cmd_q;
1598 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1599
1600 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1601 /* Load the mask */
1602 op.ksb_key = cmd_q->ksb_key;
1603
1604 ret = ccp_init_dm_workarea(&mask, cmd_q,
1605 CCP_PASSTHRU_KSB_COUNT *
1606 CCP_KSB_BYTES,
1607 DMA_TO_DEVICE);
1608 if (ret)
1609 return ret;
1610
1611 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1612 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1613 CCP_PASSTHRU_BYTESWAP_NOOP);
1614 if (ret) {
1615 cmd->engine_error = cmd_q->cmd_error;
1616 goto e_mask;
1617 }
1618 }
1619
1620 /* Prepare the input and output data workareas. For in-place
1621 * operations we need to set the dma direction to BIDIRECTIONAL
1622 * and copy the src workarea to the dst workarea.
1623 */
1624 if (sg_virt(pt->src) == sg_virt(pt->dst))
1625 in_place = true;
1626
1627 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1628 CCP_PASSTHRU_MASKSIZE,
1629 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1630 if (ret)
1631 goto e_mask;
1632
1633 if (in_place)
1634 dst = src;
1635 else {
1636 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1637 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1638 if (ret)
1639 goto e_src;
1640 }
1641
1642 /* Send data to the CCP Passthru engine
1643 * Because the CCP engine works on a single source and destination
1644 * dma address at a time, each entry in the source scatterlist
1645 * (after the dma_map_sg call) must be less than or equal to the
1646 * (remaining) length in the destination scatterlist entry and the
1647 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1648 */
1649 dst.sg_wa.sg_used = 0;
1650 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1651 if (!dst.sg_wa.sg ||
1652 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1653 ret = -EINVAL;
1654 goto e_dst;
1655 }
1656
1657 if (i == src.sg_wa.dma_count) {
1658 op.eom = 1;
1659 op.soc = 1;
1660 }
1661
1662 op.src.type = CCP_MEMTYPE_SYSTEM;
1663 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1664 op.src.u.dma.offset = 0;
1665 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1666
1667 op.dst.type = CCP_MEMTYPE_SYSTEM;
1668 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1669 op.src.u.dma.offset = dst.sg_wa.sg_used;
1670 op.src.u.dma.length = op.src.u.dma.length;
1671
1672 ret = ccp_perform_passthru(&op);
1673 if (ret) {
1674 cmd->engine_error = cmd_q->cmd_error;
1675 goto e_dst;
1676 }
1677
1678 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1679 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1680 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1681 dst.sg_wa.sg_used = 0;
1682 }
1683 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1684 }
1685
1686 e_dst:
1687 if (!in_place)
1688 ccp_free_data(&dst, cmd_q);
1689
1690 e_src:
1691 ccp_free_data(&src, cmd_q);
1692
1693 e_mask:
1694 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1695 ccp_dm_free(&mask);
1696
1697 return ret;
1698 }
1699
1700 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1701 {
1702 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1703 struct ccp_dm_workarea src, dst;
1704 struct ccp_op op;
1705 int ret;
1706 u8 *save;
1707
1708 if (!ecc->u.mm.operand_1 ||
1709 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1710 return -EINVAL;
1711
1712 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1713 if (!ecc->u.mm.operand_2 ||
1714 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1715 return -EINVAL;
1716
1717 if (!ecc->u.mm.result ||
1718 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1719 return -EINVAL;
1720
1721 memset(&op, 0, sizeof(op));
1722 op.cmd_q = cmd_q;
1723 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1724
1725 /* Concatenate the modulus and the operands. Both the modulus and
1726 * the operands must be in little endian format. Since the input
1727 * is in big endian format it must be converted and placed in a
1728 * fixed length buffer.
1729 */
1730 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1731 DMA_TO_DEVICE);
1732 if (ret)
1733 return ret;
1734
1735 /* Save the workarea address since it is updated in order to perform
1736 * the concatenation
1737 */
1738 save = src.address;
1739
1740 /* Copy the ECC modulus */
1741 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1742 CCP_ECC_OPERAND_SIZE, true);
1743 src.address += CCP_ECC_OPERAND_SIZE;
1744
1745 /* Copy the first operand */
1746 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1747 ecc->u.mm.operand_1_len,
1748 CCP_ECC_OPERAND_SIZE, true);
1749 src.address += CCP_ECC_OPERAND_SIZE;
1750
1751 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1752 /* Copy the second operand */
1753 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1754 ecc->u.mm.operand_2_len,
1755 CCP_ECC_OPERAND_SIZE, true);
1756 src.address += CCP_ECC_OPERAND_SIZE;
1757 }
1758
1759 /* Restore the workarea address */
1760 src.address = save;
1761
1762 /* Prepare the output area for the operation */
1763 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1764 DMA_FROM_DEVICE);
1765 if (ret)
1766 goto e_src;
1767
1768 op.soc = 1;
1769 op.src.u.dma.address = src.dma.address;
1770 op.src.u.dma.offset = 0;
1771 op.src.u.dma.length = src.length;
1772 op.dst.u.dma.address = dst.dma.address;
1773 op.dst.u.dma.offset = 0;
1774 op.dst.u.dma.length = dst.length;
1775
1776 op.u.ecc.function = cmd->u.ecc.function;
1777
1778 ret = ccp_perform_ecc(&op);
1779 if (ret) {
1780 cmd->engine_error = cmd_q->cmd_error;
1781 goto e_dst;
1782 }
1783
1784 ecc->ecc_result = le16_to_cpup(
1785 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1786 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1787 ret = -EIO;
1788 goto e_dst;
1789 }
1790
1791 /* Save the ECC result */
1792 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1793
1794 e_dst:
1795 ccp_dm_free(&dst);
1796
1797 e_src:
1798 ccp_dm_free(&src);
1799
1800 return ret;
1801 }
1802
1803 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1804 {
1805 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1806 struct ccp_dm_workarea src, dst;
1807 struct ccp_op op;
1808 int ret;
1809 u8 *save;
1810
1811 if (!ecc->u.pm.point_1.x ||
1812 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1813 !ecc->u.pm.point_1.y ||
1814 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1815 return -EINVAL;
1816
1817 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1818 if (!ecc->u.pm.point_2.x ||
1819 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1820 !ecc->u.pm.point_2.y ||
1821 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1822 return -EINVAL;
1823 } else {
1824 if (!ecc->u.pm.domain_a ||
1825 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1826 return -EINVAL;
1827
1828 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1829 if (!ecc->u.pm.scalar ||
1830 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1831 return -EINVAL;
1832 }
1833
1834 if (!ecc->u.pm.result.x ||
1835 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1836 !ecc->u.pm.result.y ||
1837 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1838 return -EINVAL;
1839
1840 memset(&op, 0, sizeof(op));
1841 op.cmd_q = cmd_q;
1842 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1843
1844 /* Concatenate the modulus and the operands. Both the modulus and
1845 * the operands must be in little endian format. Since the input
1846 * is in big endian format it must be converted and placed in a
1847 * fixed length buffer.
1848 */
1849 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1850 DMA_TO_DEVICE);
1851 if (ret)
1852 return ret;
1853
1854 /* Save the workarea address since it is updated in order to perform
1855 * the concatenation
1856 */
1857 save = src.address;
1858
1859 /* Copy the ECC modulus */
1860 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1861 CCP_ECC_OPERAND_SIZE, true);
1862 src.address += CCP_ECC_OPERAND_SIZE;
1863
1864 /* Copy the first point X and Y coordinate */
1865 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1866 ecc->u.pm.point_1.x_len,
1867 CCP_ECC_OPERAND_SIZE, true);
1868 src.address += CCP_ECC_OPERAND_SIZE;
1869 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1870 ecc->u.pm.point_1.y_len,
1871 CCP_ECC_OPERAND_SIZE, true);
1872 src.address += CCP_ECC_OPERAND_SIZE;
1873
1874 /* Set the first point Z coordianate to 1 */
1875 *(src.address) = 0x01;
1876 src.address += CCP_ECC_OPERAND_SIZE;
1877
1878 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1879 /* Copy the second point X and Y coordinate */
1880 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1881 ecc->u.pm.point_2.x_len,
1882 CCP_ECC_OPERAND_SIZE, true);
1883 src.address += CCP_ECC_OPERAND_SIZE;
1884 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1885 ecc->u.pm.point_2.y_len,
1886 CCP_ECC_OPERAND_SIZE, true);
1887 src.address += CCP_ECC_OPERAND_SIZE;
1888
1889 /* Set the second point Z coordianate to 1 */
1890 *(src.address) = 0x01;
1891 src.address += CCP_ECC_OPERAND_SIZE;
1892 } else {
1893 /* Copy the Domain "a" parameter */
1894 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1895 ecc->u.pm.domain_a_len,
1896 CCP_ECC_OPERAND_SIZE, true);
1897 src.address += CCP_ECC_OPERAND_SIZE;
1898
1899 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
1900 /* Copy the scalar value */
1901 ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
1902 ecc->u.pm.scalar_len,
1903 CCP_ECC_OPERAND_SIZE, true);
1904 src.address += CCP_ECC_OPERAND_SIZE;
1905 }
1906 }
1907
1908 /* Restore the workarea address */
1909 src.address = save;
1910
1911 /* Prepare the output area for the operation */
1912 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1913 DMA_FROM_DEVICE);
1914 if (ret)
1915 goto e_src;
1916
1917 op.soc = 1;
1918 op.src.u.dma.address = src.dma.address;
1919 op.src.u.dma.offset = 0;
1920 op.src.u.dma.length = src.length;
1921 op.dst.u.dma.address = dst.dma.address;
1922 op.dst.u.dma.offset = 0;
1923 op.dst.u.dma.length = dst.length;
1924
1925 op.u.ecc.function = cmd->u.ecc.function;
1926
1927 ret = ccp_perform_ecc(&op);
1928 if (ret) {
1929 cmd->engine_error = cmd_q->cmd_error;
1930 goto e_dst;
1931 }
1932
1933 ecc->ecc_result = le16_to_cpup(
1934 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1935 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1936 ret = -EIO;
1937 goto e_dst;
1938 }
1939
1940 /* Save the workarea address since it is updated as we walk through
1941 * to copy the point math result
1942 */
1943 save = dst.address;
1944
1945 /* Save the ECC result X and Y coordinates */
1946 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
1947 CCP_ECC_MODULUS_BYTES);
1948 dst.address += CCP_ECC_OUTPUT_SIZE;
1949 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
1950 CCP_ECC_MODULUS_BYTES);
1951 dst.address += CCP_ECC_OUTPUT_SIZE;
1952
1953 /* Restore the workarea address */
1954 dst.address = save;
1955
1956 e_dst:
1957 ccp_dm_free(&dst);
1958
1959 e_src:
1960 ccp_dm_free(&src);
1961
1962 return ret;
1963 }
1964
1965 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1966 {
1967 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1968
1969 ecc->ecc_result = 0;
1970
1971 if (!ecc->mod ||
1972 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
1973 return -EINVAL;
1974
1975 switch (ecc->function) {
1976 case CCP_ECC_FUNCTION_MMUL_384BIT:
1977 case CCP_ECC_FUNCTION_MADD_384BIT:
1978 case CCP_ECC_FUNCTION_MINV_384BIT:
1979 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
1980
1981 case CCP_ECC_FUNCTION_PADD_384BIT:
1982 case CCP_ECC_FUNCTION_PMUL_384BIT:
1983 case CCP_ECC_FUNCTION_PDBL_384BIT:
1984 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
1985
1986 default:
1987 return -EINVAL;
1988 }
1989 }
1990
1991 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1992 {
1993 int ret;
1994
1995 cmd->engine_error = 0;
1996 cmd_q->cmd_error = 0;
1997 cmd_q->int_rcvd = 0;
1998 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
1999
2000 switch (cmd->engine) {
2001 case CCP_ENGINE_AES:
2002 ret = ccp_run_aes_cmd(cmd_q, cmd);
2003 break;
2004 case CCP_ENGINE_XTS_AES_128:
2005 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2006 break;
2007 case CCP_ENGINE_SHA:
2008 ret = ccp_run_sha_cmd(cmd_q, cmd);
2009 break;
2010 case CCP_ENGINE_RSA:
2011 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2012 break;
2013 case CCP_ENGINE_PASSTHRU:
2014 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2015 break;
2016 case CCP_ENGINE_ECC:
2017 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2018 break;
2019 default:
2020 ret = -EINVAL;
2021 }
2022
2023 return ret;
2024 }
Linux with added FPC-III support