?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.1.16
[linux/fpc-iii.git] / drivers / crypto / ccp / ccp-ops.c
blob71f2e3c8942416b719a53c84a1ae124d0d4363b9
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/pci_ids.h>
17 #include <linux/kthread.h>
18 #include <linux/sched.h>
19 #include <linux/interrupt.h>
20 #include <linux/spinlock.h>
21 #include <linux/mutex.h>
22 #include <linux/delay.h>
23 #include <linux/ccp.h>
24 #include <linux/scatterlist.h>
25 #include <crypto/scatterwalk.h>
26 #include <crypto/sha.h>
27
28 #include "ccp-dev.h"
29
30 enum ccp_memtype {
31 CCP_MEMTYPE_SYSTEM = 0,
32 CCP_MEMTYPE_KSB,
33 CCP_MEMTYPE_LOCAL,
34 CCP_MEMTYPE__LAST,
35 };
36
37 struct ccp_dma_info {
38 dma_addr_t address;
39 unsigned int offset;
40 unsigned int length;
41 enum dma_data_direction dir;
42 };
43
44 struct ccp_dm_workarea {
45 struct device *dev;
46 struct dma_pool *dma_pool;
47 unsigned int length;
48
49 u8 *address;
50 struct ccp_dma_info dma;
51 };
52
53 struct ccp_sg_workarea {
54 struct scatterlist *sg;
55 unsigned int nents;
56 unsigned int length;
57
58 struct scatterlist *dma_sg;
59 struct device *dma_dev;
60 unsigned int dma_count;
61 enum dma_data_direction dma_dir;
62
63 unsigned int sg_used;
64
65 u64 bytes_left;
66 };
67
68 struct ccp_data {
69 struct ccp_sg_workarea sg_wa;
70 struct ccp_dm_workarea dm_wa;
71 };
72
73 struct ccp_mem {
74 enum ccp_memtype type;
75 union {
76 struct ccp_dma_info dma;
77 u32 ksb;
78 } u;
79 };
80
81 struct ccp_aes_op {
82 enum ccp_aes_type type;
83 enum ccp_aes_mode mode;
84 enum ccp_aes_action action;
85 };
86
87 struct ccp_xts_aes_op {
88 enum ccp_aes_action action;
89 enum ccp_xts_aes_unit_size unit_size;
90 };
91
92 struct ccp_sha_op {
93 enum ccp_sha_type type;
94 u64 msg_bits;
95 };
96
97 struct ccp_rsa_op {
98 u32 mod_size;
99 u32 input_len;
100 };
101
102 struct ccp_passthru_op {
103 enum ccp_passthru_bitwise bit_mod;
104 enum ccp_passthru_byteswap byte_swap;
105 };
106
107 struct ccp_ecc_op {
108 enum ccp_ecc_function function;
109 };
110
111 struct ccp_op {
112 struct ccp_cmd_queue *cmd_q;
113
114 u32 jobid;
115 u32 ioc;
116 u32 soc;
117 u32 ksb_key;
118 u32 ksb_ctx;
119 u32 init;
120 u32 eom;
121
122 struct ccp_mem src;
123 struct ccp_mem dst;
124
125 union {
126 struct ccp_aes_op aes;
127 struct ccp_xts_aes_op xts;
128 struct ccp_sha_op sha;
129 struct ccp_rsa_op rsa;
130 struct ccp_passthru_op passthru;
131 struct ccp_ecc_op ecc;
132 } u;
133 };
134
135 /* SHA initial context values */
136 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
137 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
138 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
139 cpu_to_be32(SHA1_H4), 0, 0, 0,
140 };
141
142 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
143 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
144 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
145 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
146 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
147 };
148
149 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
150 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
151 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
152 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
153 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
154 };
155
156 /* The CCP cannot perform zero-length sha operations so the caller
157 * is required to buffer data for the final operation. However, a
158 * sha operation for a message with a total length of zero is valid
159 * so known values are required to supply the result.
160 */
161 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = {
162 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
163 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
164 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00,
165 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
166 };
167
168 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = {
169 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9,
170 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4,
171 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a,
172 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00,
173 };
174
175 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = {
176 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
177 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
178 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
179 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55,
180 };
181
182 static u32 ccp_addr_lo(struct ccp_dma_info *info)
183 {
184 return lower_32_bits(info->address + info->offset);
185 }
186
187 static u32 ccp_addr_hi(struct ccp_dma_info *info)
188 {
189 return upper_32_bits(info->address + info->offset) & 0x0000ffff;
190 }
191
192 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
193 {
194 struct ccp_cmd_queue *cmd_q = op->cmd_q;
195 struct ccp_device *ccp = cmd_q->ccp;
196 void __iomem *cr_addr;
197 u32 cr0, cmd;
198 unsigned int i;
199 int ret = 0;
200
201 /* We could read a status register to see how many free slots
202 * are actually available, but reading that register resets it
203 * and you could lose some error information.
204 */
205 cmd_q->free_slots--;
206
207 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
208 | (op->jobid << REQ0_JOBID_SHIFT)
209 | REQ0_WAIT_FOR_WRITE;
210
211 if (op->soc)
212 cr0 |= REQ0_STOP_ON_COMPLETE
213 | REQ0_INT_ON_COMPLETE;
214
215 if (op->ioc || !cmd_q->free_slots)
216 cr0 |= REQ0_INT_ON_COMPLETE;
217
218 /* Start at CMD_REQ1 */
219 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
220
221 mutex_lock(&ccp->req_mutex);
222
223 /* Write CMD_REQ1 through CMD_REQx first */
224 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
225 iowrite32(*(cr + i), cr_addr);
226
227 /* Tell the CCP to start */
228 wmb();
229 iowrite32(cr0, ccp->io_regs + CMD_REQ0);
230
231 mutex_unlock(&ccp->req_mutex);
232
233 if (cr0 & REQ0_INT_ON_COMPLETE) {
234 /* Wait for the job to complete */
235 ret = wait_event_interruptible(cmd_q->int_queue,
236 cmd_q->int_rcvd);
237 if (ret || cmd_q->cmd_error) {
238 /* On error delete all related jobs from the queue */
239 cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
240 | op->jobid;
241
242 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
243
244 if (!ret)
245 ret = -EIO;
246 } else if (op->soc) {
247 /* Delete just head job from the queue on SoC */
248 cmd = DEL_Q_ACTIVE
249 | (cmd_q->id << DEL_Q_ID_SHIFT)
250 | op->jobid;
251
252 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
253 }
254
255 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
256
257 cmd_q->int_rcvd = 0;
258 }
259
260 return ret;
261 }
262
263 static int ccp_perform_aes(struct ccp_op *op)
264 {
265 u32 cr[6];
266
267 /* Fill out the register contents for REQ1 through REQ6 */
268 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
269 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
270 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
271 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
272 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
273 cr[1] = op->src.u.dma.length - 1;
274 cr[2] = ccp_addr_lo(&op->src.u.dma);
275 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
276 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
277 | ccp_addr_hi(&op->src.u.dma);
278 cr[4] = ccp_addr_lo(&op->dst.u.dma);
279 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
280 | ccp_addr_hi(&op->dst.u.dma);
281
282 if (op->u.aes.mode == CCP_AES_MODE_CFB)
283 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
284
285 if (op->eom)
286 cr[0] |= REQ1_EOM;
287
288 if (op->init)
289 cr[0] |= REQ1_INIT;
290
291 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
292 }
293
294 static int ccp_perform_xts_aes(struct ccp_op *op)
295 {
296 u32 cr[6];
297
298 /* Fill out the register contents for REQ1 through REQ6 */
299 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
300 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
301 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
302 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
303 cr[1] = op->src.u.dma.length - 1;
304 cr[2] = ccp_addr_lo(&op->src.u.dma);
305 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
306 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
307 | ccp_addr_hi(&op->src.u.dma);
308 cr[4] = ccp_addr_lo(&op->dst.u.dma);
309 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
310 | ccp_addr_hi(&op->dst.u.dma);
311
312 if (op->eom)
313 cr[0] |= REQ1_EOM;
314
315 if (op->init)
316 cr[0] |= REQ1_INIT;
317
318 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
319 }
320
321 static int ccp_perform_sha(struct ccp_op *op)
322 {
323 u32 cr[6];
324
325 /* Fill out the register contents for REQ1 through REQ6 */
326 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
327 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
328 | REQ1_INIT;
329 cr[1] = op->src.u.dma.length - 1;
330 cr[2] = ccp_addr_lo(&op->src.u.dma);
331 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
332 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
333 | ccp_addr_hi(&op->src.u.dma);
334
335 if (op->eom) {
336 cr[0] |= REQ1_EOM;
337 cr[4] = lower_32_bits(op->u.sha.msg_bits);
338 cr[5] = upper_32_bits(op->u.sha.msg_bits);
339 } else {
340 cr[4] = 0;
341 cr[5] = 0;
342 }
343
344 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
345 }
346
347 static int ccp_perform_rsa(struct ccp_op *op)
348 {
349 u32 cr[6];
350
351 /* Fill out the register contents for REQ1 through REQ6 */
352 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
353 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
354 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
355 | REQ1_EOM;
356 cr[1] = op->u.rsa.input_len - 1;
357 cr[2] = ccp_addr_lo(&op->src.u.dma);
358 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
359 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
360 | ccp_addr_hi(&op->src.u.dma);
361 cr[4] = ccp_addr_lo(&op->dst.u.dma);
362 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
363 | ccp_addr_hi(&op->dst.u.dma);
364
365 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
366 }
367
368 static int ccp_perform_passthru(struct ccp_op *op)
369 {
370 u32 cr[6];
371
372 /* Fill out the register contents for REQ1 through REQ6 */
373 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
374 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
375 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
376
377 if (op->src.type == CCP_MEMTYPE_SYSTEM)
378 cr[1] = op->src.u.dma.length - 1;
379 else
380 cr[1] = op->dst.u.dma.length - 1;
381
382 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
383 cr[2] = ccp_addr_lo(&op->src.u.dma);
384 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
385 | ccp_addr_hi(&op->src.u.dma);
386
387 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
388 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
389 } else {
390 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
391 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
392 }
393
394 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
395 cr[4] = ccp_addr_lo(&op->dst.u.dma);
396 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
397 | ccp_addr_hi(&op->dst.u.dma);
398 } else {
399 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
400 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
401 }
402
403 if (op->eom)
404 cr[0] |= REQ1_EOM;
405
406 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
407 }
408
409 static int ccp_perform_ecc(struct ccp_op *op)
410 {
411 u32 cr[6];
412
413 /* Fill out the register contents for REQ1 through REQ6 */
414 cr[0] = REQ1_ECC_AFFINE_CONVERT
415 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
416 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
417 | REQ1_EOM;
418 cr[1] = op->src.u.dma.length - 1;
419 cr[2] = ccp_addr_lo(&op->src.u.dma);
420 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
421 | ccp_addr_hi(&op->src.u.dma);
422 cr[4] = ccp_addr_lo(&op->dst.u.dma);
423 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
424 | ccp_addr_hi(&op->dst.u.dma);
425
426 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
427 }
428
429 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
430 {
431 int start;
432
433 for (;;) {
434 mutex_lock(&ccp->ksb_mutex);
435
436 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
437 ccp->ksb_count,
438 ccp->ksb_start,
439 count, 0);
440 if (start <= ccp->ksb_count) {
441 bitmap_set(ccp->ksb, start, count);
442
443 mutex_unlock(&ccp->ksb_mutex);
444 break;
445 }
446
447 ccp->ksb_avail = 0;
448
449 mutex_unlock(&ccp->ksb_mutex);
450
451 /* Wait for KSB entries to become available */
452 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
453 return 0;
454 }
455
456 return KSB_START + start;
457 }
458
459 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
460 unsigned int count)
461 {
462 if (!start)
463 return;
464
465 mutex_lock(&ccp->ksb_mutex);
466
467 bitmap_clear(ccp->ksb, start - KSB_START, count);
468
469 ccp->ksb_avail = 1;
470
471 mutex_unlock(&ccp->ksb_mutex);
472
473 wake_up_interruptible_all(&ccp->ksb_queue);
474 }
475
476 static u32 ccp_gen_jobid(struct ccp_device *ccp)
477 {
478 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
479 }
480
481 static void ccp_sg_free(struct ccp_sg_workarea *wa)
482 {
483 if (wa->dma_count)
484 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
485
486 wa->dma_count = 0;
487 }
488
489 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
490 struct scatterlist *sg, u64 len,
491 enum dma_data_direction dma_dir)
492 {
493 memset(wa, 0, sizeof(*wa));
494
495 wa->sg = sg;
496 if (!sg)
497 return 0;
498
499 wa->nents = sg_nents(sg);
500 wa->length = sg->length;
501 wa->bytes_left = len;
502 wa->sg_used = 0;
503
504 if (len == 0)
505 return 0;
506
507 if (dma_dir == DMA_NONE)
508 return 0;
509
510 wa->dma_sg = sg;
511 wa->dma_dev = dev;
512 wa->dma_dir = dma_dir;
513 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
514 if (!wa->dma_count)
515 return -ENOMEM;
516
517 return 0;
518 }
519
520 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
521 {
522 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
523
524 if (!wa->sg)
525 return;
526
527 wa->sg_used += nbytes;
528 wa->bytes_left -= nbytes;
529 if (wa->sg_used == wa->sg->length) {
530 wa->sg = sg_next(wa->sg);
531 wa->sg_used = 0;
532 }
533 }
534
535 static void ccp_dm_free(struct ccp_dm_workarea *wa)
536 {
537 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
538 if (wa->address)
539 dma_pool_free(wa->dma_pool, wa->address,
540 wa->dma.address);
541 } else {
542 if (wa->dma.address)
543 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
544 wa->dma.dir);
545 kfree(wa->address);
546 }
547
548 wa->address = NULL;
549 wa->dma.address = 0;
550 }
551
552 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
553 struct ccp_cmd_queue *cmd_q,
554 unsigned int len,
555 enum dma_data_direction dir)
556 {
557 memset(wa, 0, sizeof(*wa));
558
559 if (!len)
560 return 0;
561
562 wa->dev = cmd_q->ccp->dev;
563 wa->length = len;
564
565 if (len <= CCP_DMAPOOL_MAX_SIZE) {
566 wa->dma_pool = cmd_q->dma_pool;
567
568 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
569 &wa->dma.address);
570 if (!wa->address)
571 return -ENOMEM;
572
573 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
574
575 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
576 } else {
577 wa->address = kzalloc(len, GFP_KERNEL);
578 if (!wa->address)
579 return -ENOMEM;
580
581 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
582 dir);
583 if (!wa->dma.address)
584 return -ENOMEM;
585
586 wa->dma.length = len;
587 }
588 wa->dma.dir = dir;
589
590 return 0;
591 }
592
593 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
594 struct scatterlist *sg, unsigned int sg_offset,
595 unsigned int len)
596 {
597 WARN_ON(!wa->address);
598
599 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
600 0);
601 }
602
603 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
604 struct scatterlist *sg, unsigned int sg_offset,
605 unsigned int len)
606 {
607 WARN_ON(!wa->address);
608
609 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
610 1);
611 }
612
613 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
614 struct scatterlist *sg,
615 unsigned int len, unsigned int se_len,
616 bool sign_extend)
617 {
618 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
619 u8 buffer[CCP_REVERSE_BUF_SIZE];
620
621 BUG_ON(se_len > sizeof(buffer));
622
623 sg_offset = len;
624 dm_offset = 0;
625 nbytes = len;
626 while (nbytes) {
627 ksb_len = min_t(unsigned int, nbytes, se_len);
628 sg_offset -= ksb_len;
629
630 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
631 for (i = 0; i < ksb_len; i++)
632 wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
633
634 dm_offset += ksb_len;
635 nbytes -= ksb_len;
636
637 if ((ksb_len != se_len) && sign_extend) {
638 /* Must sign-extend to nearest sign-extend length */
639 if (wa->address[dm_offset - 1] & 0x80)
640 memset(wa->address + dm_offset, 0xff,
641 se_len - ksb_len);
642 }
643 }
644 }
645
646 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
647 struct scatterlist *sg,
648 unsigned int len)
649 {
650 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
651 u8 buffer[CCP_REVERSE_BUF_SIZE];
652
653 sg_offset = 0;
654 dm_offset = len;
655 nbytes = len;
656 while (nbytes) {
657 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
658 dm_offset -= ksb_len;
659
660 for (i = 0; i < ksb_len; i++)
661 buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
662 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
663
664 sg_offset += ksb_len;
665 nbytes -= ksb_len;
666 }
667 }
668
669 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
670 {
671 ccp_dm_free(&data->dm_wa);
672 ccp_sg_free(&data->sg_wa);
673 }
674
675 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
676 struct scatterlist *sg, u64 sg_len,
677 unsigned int dm_len,
678 enum dma_data_direction dir)
679 {
680 int ret;
681
682 memset(data, 0, sizeof(*data));
683
684 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
685 dir);
686 if (ret)
687 goto e_err;
688
689 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
690 if (ret)
691 goto e_err;
692
693 return 0;
694
695 e_err:
696 ccp_free_data(data, cmd_q);
697
698 return ret;
699 }
700
701 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
702 {
703 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
704 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
705 unsigned int buf_count, nbytes;
706
707 /* Clear the buffer if setting it */
708 if (!from)
709 memset(dm_wa->address, 0, dm_wa->length);
710
711 if (!sg_wa->sg)
712 return 0;
713
714 /* Perform the copy operation
715 * nbytes will always be <= UINT_MAX because dm_wa->length is
716 * an unsigned int
717 */
718 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
719 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
720 nbytes, from);
721
722 /* Update the structures and generate the count */
723 buf_count = 0;
724 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
725 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
726 dm_wa->length - buf_count);
727 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
728
729 buf_count += nbytes;
730 ccp_update_sg_workarea(sg_wa, nbytes);
731 }
732
733 return buf_count;
734 }
735
736 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
737 {
738 return ccp_queue_buf(data, 0);
739 }
740
741 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
742 {
743 return ccp_queue_buf(data, 1);
744 }
745
746 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
747 struct ccp_op *op, unsigned int block_size,
748 bool blocksize_op)
749 {
750 unsigned int sg_src_len, sg_dst_len, op_len;
751
752 /* The CCP can only DMA from/to one address each per operation. This
753 * requires that we find the smallest DMA area between the source
754 * and destination. The resulting len values will always be <= UINT_MAX
755 * because the dma length is an unsigned int.
756 */
757 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
758 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
759
760 if (dst) {
761 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
762 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
763 op_len = min(sg_src_len, sg_dst_len);
764 } else {
765 op_len = sg_src_len;
766 }
767
768 /* The data operation length will be at least block_size in length
769 * or the smaller of available sg room remaining for the source or
770 * the destination
771 */
772 op_len = max(op_len, block_size);
773
774 /* Unless we have to buffer data, there's no reason to wait */
775 op->soc = 0;
776
777 if (sg_src_len < block_size) {
778 /* Not enough data in the sg element, so it
779 * needs to be buffered into a blocksize chunk
780 */
781 int cp_len = ccp_fill_queue_buf(src);
782
783 op->soc = 1;
784 op->src.u.dma.address = src->dm_wa.dma.address;
785 op->src.u.dma.offset = 0;
786 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
787 } else {
788 /* Enough data in the sg element, but we need to
789 * adjust for any previously copied data
790 */
791 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
792 op->src.u.dma.offset = src->sg_wa.sg_used;
793 op->src.u.dma.length = op_len & ~(block_size - 1);
794
795 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
796 }
797
798 if (dst) {
799 if (sg_dst_len < block_size) {
800 /* Not enough room in the sg element or we're on the
801 * last piece of data (when using padding), so the
802 * output needs to be buffered into a blocksize chunk
803 */
804 op->soc = 1;
805 op->dst.u.dma.address = dst->dm_wa.dma.address;
806 op->dst.u.dma.offset = 0;
807 op->dst.u.dma.length = op->src.u.dma.length;
808 } else {
809 /* Enough room in the sg element, but we need to
810 * adjust for any previously used area
811 */
812 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
813 op->dst.u.dma.offset = dst->sg_wa.sg_used;
814 op->dst.u.dma.length = op->src.u.dma.length;
815 }
816 }
817 }
818
819 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
820 struct ccp_op *op)
821 {
822 op->init = 0;
823
824 if (dst) {
825 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
826 ccp_empty_queue_buf(dst);
827 else
828 ccp_update_sg_workarea(&dst->sg_wa,
829 op->dst.u.dma.length);
830 }
831 }
832
833 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
834 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
835 u32 byte_swap, bool from)
836 {
837 struct ccp_op op;
838
839 memset(&op, 0, sizeof(op));
840
841 op.cmd_q = cmd_q;
842 op.jobid = jobid;
843 op.eom = 1;
844
845 if (from) {
846 op.soc = 1;
847 op.src.type = CCP_MEMTYPE_KSB;
848 op.src.u.ksb = ksb;
849 op.dst.type = CCP_MEMTYPE_SYSTEM;
850 op.dst.u.dma.address = wa->dma.address;
851 op.dst.u.dma.length = wa->length;
852 } else {
853 op.src.type = CCP_MEMTYPE_SYSTEM;
854 op.src.u.dma.address = wa->dma.address;
855 op.src.u.dma.length = wa->length;
856 op.dst.type = CCP_MEMTYPE_KSB;
857 op.dst.u.ksb = ksb;
858 }
859
860 op.u.passthru.byte_swap = byte_swap;
861
862 return ccp_perform_passthru(&op);
863 }
864
865 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
866 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
867 u32 byte_swap)
868 {
869 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
870 }
871
872 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
873 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
874 u32 byte_swap)
875 {
876 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
877 }
878
879 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
880 struct ccp_cmd *cmd)
881 {
882 struct ccp_aes_engine *aes = &cmd->u.aes;
883 struct ccp_dm_workarea key, ctx;
884 struct ccp_data src;
885 struct ccp_op op;
886 unsigned int dm_offset;
887 int ret;
888
889 if (!((aes->key_len == AES_KEYSIZE_128) ||
890 (aes->key_len == AES_KEYSIZE_192) ||
891 (aes->key_len == AES_KEYSIZE_256)))
892 return -EINVAL;
893
894 if (aes->src_len & (AES_BLOCK_SIZE - 1))
895 return -EINVAL;
896
897 if (aes->iv_len != AES_BLOCK_SIZE)
898 return -EINVAL;
899
900 if (!aes->key || !aes->iv || !aes->src)
901 return -EINVAL;
902
903 if (aes->cmac_final) {
904 if (aes->cmac_key_len != AES_BLOCK_SIZE)
905 return -EINVAL;
906
907 if (!aes->cmac_key)
908 return -EINVAL;
909 }
910
911 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
912 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
913
914 ret = -EIO;
915 memset(&op, 0, sizeof(op));
916 op.cmd_q = cmd_q;
917 op.jobid = ccp_gen_jobid(cmd_q->ccp);
918 op.ksb_key = cmd_q->ksb_key;
919 op.ksb_ctx = cmd_q->ksb_ctx;
920 op.init = 1;
921 op.u.aes.type = aes->type;
922 op.u.aes.mode = aes->mode;
923 op.u.aes.action = aes->action;
924
925 /* All supported key sizes fit in a single (32-byte) KSB entry
926 * and must be in little endian format. Use the 256-bit byte
927 * swap passthru option to convert from big endian to little
928 * endian.
929 */
930 ret = ccp_init_dm_workarea(&key, cmd_q,
931 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
932 DMA_TO_DEVICE);
933 if (ret)
934 return ret;
935
936 dm_offset = CCP_KSB_BYTES - aes->key_len;
937 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
938 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
939 CCP_PASSTHRU_BYTESWAP_256BIT);
940 if (ret) {
941 cmd->engine_error = cmd_q->cmd_error;
942 goto e_key;
943 }
944
945 /* The AES context fits in a single (32-byte) KSB entry and
946 * must be in little endian format. Use the 256-bit byte swap
947 * passthru option to convert from big endian to little endian.
948 */
949 ret = ccp_init_dm_workarea(&ctx, cmd_q,
950 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
951 DMA_BIDIRECTIONAL);
952 if (ret)
953 goto e_key;
954
955 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
956 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
957 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
958 CCP_PASSTHRU_BYTESWAP_256BIT);
959 if (ret) {
960 cmd->engine_error = cmd_q->cmd_error;
961 goto e_ctx;
962 }
963
964 /* Send data to the CCP AES engine */
965 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
966 AES_BLOCK_SIZE, DMA_TO_DEVICE);
967 if (ret)
968 goto e_ctx;
969
970 while (src.sg_wa.bytes_left) {
971 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
972 if (aes->cmac_final && !src.sg_wa.bytes_left) {
973 op.eom = 1;
974
975 /* Push the K1/K2 key to the CCP now */
976 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
977 op.ksb_ctx,
978 CCP_PASSTHRU_BYTESWAP_256BIT);
979 if (ret) {
980 cmd->engine_error = cmd_q->cmd_error;
981 goto e_src;
982 }
983
984 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
985 aes->cmac_key_len);
986 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
987 CCP_PASSTHRU_BYTESWAP_256BIT);
988 if (ret) {
989 cmd->engine_error = cmd_q->cmd_error;
990 goto e_src;
991 }
992 }
993
994 ret = ccp_perform_aes(&op);
995 if (ret) {
996 cmd->engine_error = cmd_q->cmd_error;
997 goto e_src;
998 }
999
1000 ccp_process_data(&src, NULL, &op);
1001 }
1002
1003 /* Retrieve the AES context - convert from LE to BE using
1004 * 32-byte (256-bit) byteswapping
1005 */
1006 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1007 CCP_PASSTHRU_BYTESWAP_256BIT);
1008 if (ret) {
1009 cmd->engine_error = cmd_q->cmd_error;
1010 goto e_src;
1011 }
1012
1013 /* ...but we only need AES_BLOCK_SIZE bytes */
1014 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1015 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1016
1017 e_src:
1018 ccp_free_data(&src, cmd_q);
1019
1020 e_ctx:
1021 ccp_dm_free(&ctx);
1022
1023 e_key:
1024 ccp_dm_free(&key);
1025
1026 return ret;
1027 }
1028
1029 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1030 {
1031 struct ccp_aes_engine *aes = &cmd->u.aes;
1032 struct ccp_dm_workarea key, ctx;
1033 struct ccp_data src, dst;
1034 struct ccp_op op;
1035 unsigned int dm_offset;
1036 bool in_place = false;
1037 int ret;
1038
1039 if (aes->mode == CCP_AES_MODE_CMAC)
1040 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1041
1042 if (!((aes->key_len == AES_KEYSIZE_128) ||
1043 (aes->key_len == AES_KEYSIZE_192) ||
1044 (aes->key_len == AES_KEYSIZE_256)))
1045 return -EINVAL;
1046
1047 if (((aes->mode == CCP_AES_MODE_ECB) ||
1048 (aes->mode == CCP_AES_MODE_CBC) ||
1049 (aes->mode == CCP_AES_MODE_CFB)) &&
1050 (aes->src_len & (AES_BLOCK_SIZE - 1)))
1051 return -EINVAL;
1052
1053 if (!aes->key || !aes->src || !aes->dst)
1054 return -EINVAL;
1055
1056 if (aes->mode != CCP_AES_MODE_ECB) {
1057 if (aes->iv_len != AES_BLOCK_SIZE)
1058 return -EINVAL;
1059
1060 if (!aes->iv)
1061 return -EINVAL;
1062 }
1063
1064 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1065 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1066
1067 ret = -EIO;
1068 memset(&op, 0, sizeof(op));
1069 op.cmd_q = cmd_q;
1070 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1071 op.ksb_key = cmd_q->ksb_key;
1072 op.ksb_ctx = cmd_q->ksb_ctx;
1073 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1074 op.u.aes.type = aes->type;
1075 op.u.aes.mode = aes->mode;
1076 op.u.aes.action = aes->action;
1077
1078 /* All supported key sizes fit in a single (32-byte) KSB entry
1079 * and must be in little endian format. Use the 256-bit byte
1080 * swap passthru option to convert from big endian to little
1081 * endian.
1082 */
1083 ret = ccp_init_dm_workarea(&key, cmd_q,
1084 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1085 DMA_TO_DEVICE);
1086 if (ret)
1087 return ret;
1088
1089 dm_offset = CCP_KSB_BYTES - aes->key_len;
1090 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1091 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1092 CCP_PASSTHRU_BYTESWAP_256BIT);
1093 if (ret) {
1094 cmd->engine_error = cmd_q->cmd_error;
1095 goto e_key;
1096 }
1097
1098 /* The AES context fits in a single (32-byte) KSB entry and
1099 * must be in little endian format. Use the 256-bit byte swap
1100 * passthru option to convert from big endian to little endian.
1101 */
1102 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1103 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1104 DMA_BIDIRECTIONAL);
1105 if (ret)
1106 goto e_key;
1107
1108 if (aes->mode != CCP_AES_MODE_ECB) {
1109 /* Load the AES context - conver to LE */
1110 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1111 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1112 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1113 CCP_PASSTHRU_BYTESWAP_256BIT);
1114 if (ret) {
1115 cmd->engine_error = cmd_q->cmd_error;
1116 goto e_ctx;
1117 }
1118 }
1119
1120 /* Prepare the input and output data workareas. For in-place
1121 * operations we need to set the dma direction to BIDIRECTIONAL
1122 * and copy the src workarea to the dst workarea.
1123 */
1124 if (sg_virt(aes->src) == sg_virt(aes->dst))
1125 in_place = true;
1126
1127 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1128 AES_BLOCK_SIZE,
1129 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1130 if (ret)
1131 goto e_ctx;
1132
1133 if (in_place) {
1134 dst = src;
1135 } else {
1136 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1137 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1138 if (ret)
1139 goto e_src;
1140 }
1141
1142 /* Send data to the CCP AES engine */
1143 while (src.sg_wa.bytes_left) {
1144 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1145 if (!src.sg_wa.bytes_left) {
1146 op.eom = 1;
1147
1148 /* Since we don't retrieve the AES context in ECB
1149 * mode we have to wait for the operation to complete
1150 * on the last piece of data
1151 */
1152 if (aes->mode == CCP_AES_MODE_ECB)
1153 op.soc = 1;
1154 }
1155
1156 ret = ccp_perform_aes(&op);
1157 if (ret) {
1158 cmd->engine_error = cmd_q->cmd_error;
1159 goto e_dst;
1160 }
1161
1162 ccp_process_data(&src, &dst, &op);
1163 }
1164
1165 if (aes->mode != CCP_AES_MODE_ECB) {
1166 /* Retrieve the AES context - convert from LE to BE using
1167 * 32-byte (256-bit) byteswapping
1168 */
1169 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1170 CCP_PASSTHRU_BYTESWAP_256BIT);
1171 if (ret) {
1172 cmd->engine_error = cmd_q->cmd_error;
1173 goto e_dst;
1174 }
1175
1176 /* ...but we only need AES_BLOCK_SIZE bytes */
1177 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1178 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1179 }
1180
1181 e_dst:
1182 if (!in_place)
1183 ccp_free_data(&dst, cmd_q);
1184
1185 e_src:
1186 ccp_free_data(&src, cmd_q);
1187
1188 e_ctx:
1189 ccp_dm_free(&ctx);
1190
1191 e_key:
1192 ccp_dm_free(&key);
1193
1194 return ret;
1195 }
1196
1197 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1198 struct ccp_cmd *cmd)
1199 {
1200 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1201 struct ccp_dm_workarea key, ctx;
1202 struct ccp_data src, dst;
1203 struct ccp_op op;
1204 unsigned int unit_size, dm_offset;
1205 bool in_place = false;
1206 int ret;
1207
1208 switch (xts->unit_size) {
1209 case CCP_XTS_AES_UNIT_SIZE_16:
1210 unit_size = 16;
1211 break;
1212 case CCP_XTS_AES_UNIT_SIZE_512:
1213 unit_size = 512;
1214 break;
1215 case CCP_XTS_AES_UNIT_SIZE_1024:
1216 unit_size = 1024;
1217 break;
1218 case CCP_XTS_AES_UNIT_SIZE_2048:
1219 unit_size = 2048;
1220 break;
1221 case CCP_XTS_AES_UNIT_SIZE_4096:
1222 unit_size = 4096;
1223 break;
1224
1225 default:
1226 return -EINVAL;
1227 }
1228
1229 if (xts->key_len != AES_KEYSIZE_128)
1230 return -EINVAL;
1231
1232 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1233 return -EINVAL;
1234
1235 if (xts->iv_len != AES_BLOCK_SIZE)
1236 return -EINVAL;
1237
1238 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1239 return -EINVAL;
1240
1241 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1242 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1243
1244 ret = -EIO;
1245 memset(&op, 0, sizeof(op));
1246 op.cmd_q = cmd_q;
1247 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1248 op.ksb_key = cmd_q->ksb_key;
1249 op.ksb_ctx = cmd_q->ksb_ctx;
1250 op.init = 1;
1251 op.u.xts.action = xts->action;
1252 op.u.xts.unit_size = xts->unit_size;
1253
1254 /* All supported key sizes fit in a single (32-byte) KSB entry
1255 * and must be in little endian format. Use the 256-bit byte
1256 * swap passthru option to convert from big endian to little
1257 * endian.
1258 */
1259 ret = ccp_init_dm_workarea(&key, cmd_q,
1260 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1261 DMA_TO_DEVICE);
1262 if (ret)
1263 return ret;
1264
1265 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1266 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1267 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1268 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1269 CCP_PASSTHRU_BYTESWAP_256BIT);
1270 if (ret) {
1271 cmd->engine_error = cmd_q->cmd_error;
1272 goto e_key;
1273 }
1274
1275 /* The AES context fits in a single (32-byte) KSB entry and
1276 * for XTS is already in little endian format so no byte swapping
1277 * is needed.
1278 */
1279 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1280 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1281 DMA_BIDIRECTIONAL);
1282 if (ret)
1283 goto e_key;
1284
1285 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1286 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1287 CCP_PASSTHRU_BYTESWAP_NOOP);
1288 if (ret) {
1289 cmd->engine_error = cmd_q->cmd_error;
1290 goto e_ctx;
1291 }
1292
1293 /* Prepare the input and output data workareas. For in-place
1294 * operations we need to set the dma direction to BIDIRECTIONAL
1295 * and copy the src workarea to the dst workarea.
1296 */
1297 if (sg_virt(xts->src) == sg_virt(xts->dst))
1298 in_place = true;
1299
1300 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1301 unit_size,
1302 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1303 if (ret)
1304 goto e_ctx;
1305
1306 if (in_place) {
1307 dst = src;
1308 } else {
1309 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1310 unit_size, DMA_FROM_DEVICE);
1311 if (ret)
1312 goto e_src;
1313 }
1314
1315 /* Send data to the CCP AES engine */
1316 while (src.sg_wa.bytes_left) {
1317 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1318 if (!src.sg_wa.bytes_left)
1319 op.eom = 1;
1320
1321 ret = ccp_perform_xts_aes(&op);
1322 if (ret) {
1323 cmd->engine_error = cmd_q->cmd_error;
1324 goto e_dst;
1325 }
1326
1327 ccp_process_data(&src, &dst, &op);
1328 }
1329
1330 /* Retrieve the AES context - convert from LE to BE using
1331 * 32-byte (256-bit) byteswapping
1332 */
1333 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1334 CCP_PASSTHRU_BYTESWAP_256BIT);
1335 if (ret) {
1336 cmd->engine_error = cmd_q->cmd_error;
1337 goto e_dst;
1338 }
1339
1340 /* ...but we only need AES_BLOCK_SIZE bytes */
1341 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1342 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1343
1344 e_dst:
1345 if (!in_place)
1346 ccp_free_data(&dst, cmd_q);
1347
1348 e_src:
1349 ccp_free_data(&src, cmd_q);
1350
1351 e_ctx:
1352 ccp_dm_free(&ctx);
1353
1354 e_key:
1355 ccp_dm_free(&key);
1356
1357 return ret;
1358 }
1359
1360 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1361 {
1362 struct ccp_sha_engine *sha = &cmd->u.sha;
1363 struct ccp_dm_workarea ctx;
1364 struct ccp_data src;
1365 struct ccp_op op;
1366 int ret;
1367
1368 if (sha->ctx_len != CCP_SHA_CTXSIZE)
1369 return -EINVAL;
1370
1371 if (!sha->ctx)
1372 return -EINVAL;
1373
1374 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1375 return -EINVAL;
1376
1377 if (!sha->src_len) {
1378 const u8 *sha_zero;
1379
1380 /* Not final, just return */
1381 if (!sha->final)
1382 return 0;
1383
1384 /* CCP can't do a zero length sha operation so the caller
1385 * must buffer the data.
1386 */
1387 if (sha->msg_bits)
1388 return -EINVAL;
1389
1390 /* A sha operation for a message with a total length of zero,
1391 * return known result.
1392 */
1393 switch (sha->type) {
1394 case CCP_SHA_TYPE_1:
1395 sha_zero = ccp_sha1_zero;
1396 break;
1397 case CCP_SHA_TYPE_224:
1398 sha_zero = ccp_sha224_zero;
1399 break;
1400 case CCP_SHA_TYPE_256:
1401 sha_zero = ccp_sha256_zero;
1402 break;
1403 default:
1404 return -EINVAL;
1405 }
1406
1407 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1408 sha->ctx_len, 1);
1409
1410 return 0;
1411 }
1412
1413 if (!sha->src)
1414 return -EINVAL;
1415
1416 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1417
1418 memset(&op, 0, sizeof(op));
1419 op.cmd_q = cmd_q;
1420 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1421 op.ksb_ctx = cmd_q->ksb_ctx;
1422 op.u.sha.type = sha->type;
1423 op.u.sha.msg_bits = sha->msg_bits;
1424
1425 /* The SHA context fits in a single (32-byte) KSB entry and
1426 * must be in little endian format. Use the 256-bit byte swap
1427 * passthru option to convert from big endian to little endian.
1428 */
1429 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1430 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1431 DMA_BIDIRECTIONAL);
1432 if (ret)
1433 return ret;
1434
1435 if (sha->first) {
1436 const __be32 *init;
1437
1438 switch (sha->type) {
1439 case CCP_SHA_TYPE_1:
1440 init = ccp_sha1_init;
1441 break;
1442 case CCP_SHA_TYPE_224:
1443 init = ccp_sha224_init;
1444 break;
1445 case CCP_SHA_TYPE_256:
1446 init = ccp_sha256_init;
1447 break;
1448 default:
1449 ret = -EINVAL;
1450 goto e_ctx;
1451 }
1452 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
1453 } else {
1454 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1455 }
1456
1457 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1458 CCP_PASSTHRU_BYTESWAP_256BIT);
1459 if (ret) {
1460 cmd->engine_error = cmd_q->cmd_error;
1461 goto e_ctx;
1462 }
1463
1464 /* Send data to the CCP SHA engine */
1465 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1466 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1467 if (ret)
1468 goto e_ctx;
1469
1470 while (src.sg_wa.bytes_left) {
1471 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1472 if (sha->final && !src.sg_wa.bytes_left)
1473 op.eom = 1;
1474
1475 ret = ccp_perform_sha(&op);
1476 if (ret) {
1477 cmd->engine_error = cmd_q->cmd_error;
1478 goto e_data;
1479 }
1480
1481 ccp_process_data(&src, NULL, &op);
1482 }
1483
1484 /* Retrieve the SHA context - convert from LE to BE using
1485 * 32-byte (256-bit) byteswapping to BE
1486 */
1487 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1488 CCP_PASSTHRU_BYTESWAP_256BIT);
1489 if (ret) {
1490 cmd->engine_error = cmd_q->cmd_error;
1491 goto e_data;
1492 }
1493
1494 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1495
1496 if (sha->final && sha->opad) {
1497 /* HMAC operation, recursively perform final SHA */
1498 struct ccp_cmd hmac_cmd;
1499 struct scatterlist sg;
1500 u64 block_size, digest_size;
1501 u8 *hmac_buf;
1502
1503 switch (sha->type) {
1504 case CCP_SHA_TYPE_1:
1505 block_size = SHA1_BLOCK_SIZE;
1506 digest_size = SHA1_DIGEST_SIZE;
1507 break;
1508 case CCP_SHA_TYPE_224:
1509 block_size = SHA224_BLOCK_SIZE;
1510 digest_size = SHA224_DIGEST_SIZE;
1511 break;
1512 case CCP_SHA_TYPE_256:
1513 block_size = SHA256_BLOCK_SIZE;
1514 digest_size = SHA256_DIGEST_SIZE;
1515 break;
1516 default:
1517 ret = -EINVAL;
1518 goto e_data;
1519 }
1520
1521 if (sha->opad_len != block_size) {
1522 ret = -EINVAL;
1523 goto e_data;
1524 }
1525
1526 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1527 if (!hmac_buf) {
1528 ret = -ENOMEM;
1529 goto e_data;
1530 }
1531 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1532
1533 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1534 memcpy(hmac_buf + block_size, ctx.address, digest_size);
1535
1536 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1537 hmac_cmd.engine = CCP_ENGINE_SHA;
1538 hmac_cmd.u.sha.type = sha->type;
1539 hmac_cmd.u.sha.ctx = sha->ctx;
1540 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1541 hmac_cmd.u.sha.src = &sg;
1542 hmac_cmd.u.sha.src_len = block_size + digest_size;
1543 hmac_cmd.u.sha.opad = NULL;
1544 hmac_cmd.u.sha.opad_len = 0;
1545 hmac_cmd.u.sha.first = 1;
1546 hmac_cmd.u.sha.final = 1;
1547 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1548
1549 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1550 if (ret)
1551 cmd->engine_error = hmac_cmd.engine_error;
1552
1553 kfree(hmac_buf);
1554 }
1555
1556 e_data:
1557 ccp_free_data(&src, cmd_q);
1558
1559 e_ctx:
1560 ccp_dm_free(&ctx);
1561
1562 return ret;
1563 }
1564
1565 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1566 {
1567 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1568 struct ccp_dm_workarea exp, src;
1569 struct ccp_data dst;
1570 struct ccp_op op;
1571 unsigned int ksb_count, i_len, o_len;
1572 int ret;
1573
1574 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1575 return -EINVAL;
1576
1577 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1578 return -EINVAL;
1579
1580 /* The RSA modulus must precede the message being acted upon, so
1581 * it must be copied to a DMA area where the message and the
1582 * modulus can be concatenated. Therefore the input buffer
1583 * length required is twice the output buffer length (which
1584 * must be a multiple of 256-bits).
1585 */
1586 o_len = ((rsa->key_size + 255) / 256) * 32;
1587 i_len = o_len * 2;
1588
1589 ksb_count = o_len / CCP_KSB_BYTES;
1590
1591 memset(&op, 0, sizeof(op));
1592 op.cmd_q = cmd_q;
1593 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1594 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1595 if (!op.ksb_key)
1596 return -EIO;
1597
1598 /* The RSA exponent may span multiple (32-byte) KSB entries and must
1599 * be in little endian format. Reverse copy each 32-byte chunk
1600 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1601 * and each byte within that chunk and do not perform any byte swap
1602 * operations on the passthru operation.
1603 */
1604 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1605 if (ret)
1606 goto e_ksb;
1607
1608 ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES,
1609 false);
1610 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1611 CCP_PASSTHRU_BYTESWAP_NOOP);
1612 if (ret) {
1613 cmd->engine_error = cmd_q->cmd_error;
1614 goto e_exp;
1615 }
1616
1617 /* Concatenate the modulus and the message. Both the modulus and
1618 * the operands must be in little endian format. Since the input
1619 * is in big endian format it must be converted.
1620 */
1621 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1622 if (ret)
1623 goto e_exp;
1624
1625 ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES,
1626 false);
1627 src.address += o_len; /* Adjust the address for the copy operation */
1628 ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES,
1629 false);
1630 src.address -= o_len; /* Reset the address to original value */
1631
1632 /* Prepare the output area for the operation */
1633 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1634 o_len, DMA_FROM_DEVICE);
1635 if (ret)
1636 goto e_src;
1637
1638 op.soc = 1;
1639 op.src.u.dma.address = src.dma.address;
1640 op.src.u.dma.offset = 0;
1641 op.src.u.dma.length = i_len;
1642 op.dst.u.dma.address = dst.dm_wa.dma.address;
1643 op.dst.u.dma.offset = 0;
1644 op.dst.u.dma.length = o_len;
1645
1646 op.u.rsa.mod_size = rsa->key_size;
1647 op.u.rsa.input_len = i_len;
1648
1649 ret = ccp_perform_rsa(&op);
1650 if (ret) {
1651 cmd->engine_error = cmd_q->cmd_error;
1652 goto e_dst;
1653 }
1654
1655 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1656
1657 e_dst:
1658 ccp_free_data(&dst, cmd_q);
1659
1660 e_src:
1661 ccp_dm_free(&src);
1662
1663 e_exp:
1664 ccp_dm_free(&exp);
1665
1666 e_ksb:
1667 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1668
1669 return ret;
1670 }
1671
1672 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1673 struct ccp_cmd *cmd)
1674 {
1675 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1676 struct ccp_dm_workarea mask;
1677 struct ccp_data src, dst;
1678 struct ccp_op op;
1679 bool in_place = false;
1680 unsigned int i;
1681 int ret;
1682
1683 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1684 return -EINVAL;
1685
1686 if (!pt->src || !pt->dst)
1687 return -EINVAL;
1688
1689 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1690 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1691 return -EINVAL;
1692 if (!pt->mask)
1693 return -EINVAL;
1694 }
1695
1696 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1697
1698 memset(&op, 0, sizeof(op));
1699 op.cmd_q = cmd_q;
1700 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1701
1702 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1703 /* Load the mask */
1704 op.ksb_key = cmd_q->ksb_key;
1705
1706 ret = ccp_init_dm_workarea(&mask, cmd_q,
1707 CCP_PASSTHRU_KSB_COUNT *
1708 CCP_KSB_BYTES,
1709 DMA_TO_DEVICE);
1710 if (ret)
1711 return ret;
1712
1713 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1714 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1715 CCP_PASSTHRU_BYTESWAP_NOOP);
1716 if (ret) {
1717 cmd->engine_error = cmd_q->cmd_error;
1718 goto e_mask;
1719 }
1720 }
1721
1722 /* Prepare the input and output data workareas. For in-place
1723 * operations we need to set the dma direction to BIDIRECTIONAL
1724 * and copy the src workarea to the dst workarea.
1725 */
1726 if (sg_virt(pt->src) == sg_virt(pt->dst))
1727 in_place = true;
1728
1729 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1730 CCP_PASSTHRU_MASKSIZE,
1731 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1732 if (ret)
1733 goto e_mask;
1734
1735 if (in_place) {
1736 dst = src;
1737 } else {
1738 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1739 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1740 if (ret)
1741 goto e_src;
1742 }
1743
1744 /* Send data to the CCP Passthru engine
1745 * Because the CCP engine works on a single source and destination
1746 * dma address at a time, each entry in the source scatterlist
1747 * (after the dma_map_sg call) must be less than or equal to the
1748 * (remaining) length in the destination scatterlist entry and the
1749 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1750 */
1751 dst.sg_wa.sg_used = 0;
1752 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1753 if (!dst.sg_wa.sg ||
1754 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1755 ret = -EINVAL;
1756 goto e_dst;
1757 }
1758
1759 if (i == src.sg_wa.dma_count) {
1760 op.eom = 1;
1761 op.soc = 1;
1762 }
1763
1764 op.src.type = CCP_MEMTYPE_SYSTEM;
1765 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1766 op.src.u.dma.offset = 0;
1767 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1768
1769 op.dst.type = CCP_MEMTYPE_SYSTEM;
1770 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1771 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1772 op.dst.u.dma.length = op.src.u.dma.length;
1773
1774 ret = ccp_perform_passthru(&op);
1775 if (ret) {
1776 cmd->engine_error = cmd_q->cmd_error;
1777 goto e_dst;
1778 }
1779
1780 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1781 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1782 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1783 dst.sg_wa.sg_used = 0;
1784 }
1785 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1786 }
1787
1788 e_dst:
1789 if (!in_place)
1790 ccp_free_data(&dst, cmd_q);
1791
1792 e_src:
1793 ccp_free_data(&src, cmd_q);
1794
1795 e_mask:
1796 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1797 ccp_dm_free(&mask);
1798
1799 return ret;
1800 }
1801
1802 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1803 {
1804 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1805 struct ccp_dm_workarea src, dst;
1806 struct ccp_op op;
1807 int ret;
1808 u8 *save;
1809
1810 if (!ecc->u.mm.operand_1 ||
1811 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1812 return -EINVAL;
1813
1814 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1815 if (!ecc->u.mm.operand_2 ||
1816 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1817 return -EINVAL;
1818
1819 if (!ecc->u.mm.result ||
1820 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1821 return -EINVAL;
1822
1823 memset(&op, 0, sizeof(op));
1824 op.cmd_q = cmd_q;
1825 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1826
1827 /* Concatenate the modulus and the operands. Both the modulus and
1828 * the operands must be in little endian format. Since the input
1829 * is in big endian format it must be converted and placed in a
1830 * fixed length buffer.
1831 */
1832 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1833 DMA_TO_DEVICE);
1834 if (ret)
1835 return ret;
1836
1837 /* Save the workarea address since it is updated in order to perform
1838 * the concatenation
1839 */
1840 save = src.address;
1841
1842 /* Copy the ECC modulus */
1843 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1844 CCP_ECC_OPERAND_SIZE, false);
1845 src.address += CCP_ECC_OPERAND_SIZE;
1846
1847 /* Copy the first operand */
1848 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1849 ecc->u.mm.operand_1_len,
1850 CCP_ECC_OPERAND_SIZE, false);
1851 src.address += CCP_ECC_OPERAND_SIZE;
1852
1853 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1854 /* Copy the second operand */
1855 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1856 ecc->u.mm.operand_2_len,
1857 CCP_ECC_OPERAND_SIZE, false);
1858 src.address += CCP_ECC_OPERAND_SIZE;
1859 }
1860
1861 /* Restore the workarea address */
1862 src.address = save;
1863
1864 /* Prepare the output area for the operation */
1865 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1866 DMA_FROM_DEVICE);
1867 if (ret)
1868 goto e_src;
1869
1870 op.soc = 1;
1871 op.src.u.dma.address = src.dma.address;
1872 op.src.u.dma.offset = 0;
1873 op.src.u.dma.length = src.length;
1874 op.dst.u.dma.address = dst.dma.address;
1875 op.dst.u.dma.offset = 0;
1876 op.dst.u.dma.length = dst.length;
1877
1878 op.u.ecc.function = cmd->u.ecc.function;
1879
1880 ret = ccp_perform_ecc(&op);
1881 if (ret) {
1882 cmd->engine_error = cmd_q->cmd_error;
1883 goto e_dst;
1884 }
1885
1886 ecc->ecc_result = le16_to_cpup(
1887 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1888 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1889 ret = -EIO;
1890 goto e_dst;
1891 }
1892
1893 /* Save the ECC result */
1894 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1895
1896 e_dst:
1897 ccp_dm_free(&dst);
1898
1899 e_src:
1900 ccp_dm_free(&src);
1901
1902 return ret;
1903 }
1904
1905 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1906 {
1907 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1908 struct ccp_dm_workarea src, dst;
1909 struct ccp_op op;
1910 int ret;
1911 u8 *save;
1912
1913 if (!ecc->u.pm.point_1.x ||
1914 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1915 !ecc->u.pm.point_1.y ||
1916 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1917 return -EINVAL;
1918
1919 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1920 if (!ecc->u.pm.point_2.x ||
1921 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1922 !ecc->u.pm.point_2.y ||
1923 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1924 return -EINVAL;
1925 } else {
1926 if (!ecc->u.pm.domain_a ||
1927 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1928 return -EINVAL;
1929
1930 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1931 if (!ecc->u.pm.scalar ||
1932 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1933 return -EINVAL;
1934 }
1935
1936 if (!ecc->u.pm.result.x ||
1937 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1938 !ecc->u.pm.result.y ||
1939 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1940 return -EINVAL;
1941
1942 memset(&op, 0, sizeof(op));
1943 op.cmd_q = cmd_q;
1944 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1945
1946 /* Concatenate the modulus and the operands. Both the modulus and
1947 * the operands must be in little endian format. Since the input
1948 * is in big endian format it must be converted and placed in a
1949 * fixed length buffer.
1950 */
1951 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1952 DMA_TO_DEVICE);
1953 if (ret)
1954 return ret;
1955
1956 /* Save the workarea address since it is updated in order to perform
1957 * the concatenation
1958 */
1959 save = src.address;
1960
1961 /* Copy the ECC modulus */
1962 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1963 CCP_ECC_OPERAND_SIZE, false);
1964 src.address += CCP_ECC_OPERAND_SIZE;
1965
1966 /* Copy the first point X and Y coordinate */
1967 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1968 ecc->u.pm.point_1.x_len,
1969 CCP_ECC_OPERAND_SIZE, false);
1970 src.address += CCP_ECC_OPERAND_SIZE;
1971 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1972 ecc->u.pm.point_1.y_len,
1973 CCP_ECC_OPERAND_SIZE, false);
1974 src.address += CCP_ECC_OPERAND_SIZE;
1975
1976 /* Set the first point Z coordianate to 1 */
1977 *src.address = 0x01;
1978 src.address += CCP_ECC_OPERAND_SIZE;
1979
1980 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1981 /* Copy the second point X and Y coordinate */
1982 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1983 ecc->u.pm.point_2.x_len,
1984 CCP_ECC_OPERAND_SIZE, false);
1985 src.address += CCP_ECC_OPERAND_SIZE;
1986 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1987 ecc->u.pm.point_2.y_len,
1988 CCP_ECC_OPERAND_SIZE, false);
1989 src.address += CCP_ECC_OPERAND_SIZE;
1990
1991 /* Set the second point Z coordianate to 1 */
1992 *src.address = 0x01;
1993 src.address += CCP_ECC_OPERAND_SIZE;
1994 } else {
1995 /* Copy the Domain "a" parameter */
1996 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1997 ecc->u.pm.domain_a_len,
1998 CCP_ECC_OPERAND_SIZE, false);
1999 src.address += CCP_ECC_OPERAND_SIZE;
2000
2001 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2002 /* Copy the scalar value */
2003 ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
2004 ecc->u.pm.scalar_len,
2005 CCP_ECC_OPERAND_SIZE, false);
2006 src.address += CCP_ECC_OPERAND_SIZE;
2007 }
2008 }
2009
2010 /* Restore the workarea address */
2011 src.address = save;
2012
2013 /* Prepare the output area for the operation */
2014 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2015 DMA_FROM_DEVICE);
2016 if (ret)
2017 goto e_src;
2018
2019 op.soc = 1;
2020 op.src.u.dma.address = src.dma.address;
2021 op.src.u.dma.offset = 0;
2022 op.src.u.dma.length = src.length;
2023 op.dst.u.dma.address = dst.dma.address;
2024 op.dst.u.dma.offset = 0;
2025 op.dst.u.dma.length = dst.length;
2026
2027 op.u.ecc.function = cmd->u.ecc.function;
2028
2029 ret = ccp_perform_ecc(&op);
2030 if (ret) {
2031 cmd->engine_error = cmd_q->cmd_error;
2032 goto e_dst;
2033 }
2034
2035 ecc->ecc_result = le16_to_cpup(
2036 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2037 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2038 ret = -EIO;
2039 goto e_dst;
2040 }
2041
2042 /* Save the workarea address since it is updated as we walk through
2043 * to copy the point math result
2044 */
2045 save = dst.address;
2046
2047 /* Save the ECC result X and Y coordinates */
2048 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
2049 CCP_ECC_MODULUS_BYTES);
2050 dst.address += CCP_ECC_OUTPUT_SIZE;
2051 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
2052 CCP_ECC_MODULUS_BYTES);
2053 dst.address += CCP_ECC_OUTPUT_SIZE;
2054
2055 /* Restore the workarea address */
2056 dst.address = save;
2057
2058 e_dst:
2059 ccp_dm_free(&dst);
2060
2061 e_src:
2062 ccp_dm_free(&src);
2063
2064 return ret;
2065 }
2066
2067 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2068 {
2069 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2070
2071 ecc->ecc_result = 0;
2072
2073 if (!ecc->mod ||
2074 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2075 return -EINVAL;
2076
2077 switch (ecc->function) {
2078 case CCP_ECC_FUNCTION_MMUL_384BIT:
2079 case CCP_ECC_FUNCTION_MADD_384BIT:
2080 case CCP_ECC_FUNCTION_MINV_384BIT:
2081 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2082
2083 case CCP_ECC_FUNCTION_PADD_384BIT:
2084 case CCP_ECC_FUNCTION_PMUL_384BIT:
2085 case CCP_ECC_FUNCTION_PDBL_384BIT:
2086 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2087
2088 default:
2089 return -EINVAL;
2090 }
2091 }
2092
2093 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2094 {
2095 int ret;
2096
2097 cmd->engine_error = 0;
2098 cmd_q->cmd_error = 0;
2099 cmd_q->int_rcvd = 0;
2100 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
2101
2102 switch (cmd->engine) {
2103 case CCP_ENGINE_AES:
2104 ret = ccp_run_aes_cmd(cmd_q, cmd);
2105 break;
2106 case CCP_ENGINE_XTS_AES_128:
2107 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2108 break;
2109 case CCP_ENGINE_SHA:
2110 ret = ccp_run_sha_cmd(cmd_q, cmd);
2111 break;
2112 case CCP_ENGINE_RSA:
2113 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2114 break;
2115 case CCP_ENGINE_PASSTHRU:
2116 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2117 break;
2118 case CCP_ENGINE_ECC:
2119 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2120 break;
2121 default:
2122 ret = -EINVAL;
2123 }
2124
2125 return ret;
2126 }
Linux with added FPC-III support