?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ext4: reduce contention on s_orphan_lock
[linux/fpc-iii.git] / drivers / crypto / ccp / ccp-ops.c
blob9ae006d69df4d507cb74c446ca60bce35355a7ac
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 */
12
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/pci.h>
16 #include <linux/pci_ids.h>
17 #include <linux/kthread.h>
18 #include <linux/sched.h>
19 #include <linux/interrupt.h>
20 #include <linux/spinlock.h>
21 #include <linux/mutex.h>
22 #include <linux/delay.h>
23 #include <linux/ccp.h>
24 #include <linux/scatterlist.h>
25 #include <crypto/scatterwalk.h>
26 #include <crypto/sha.h>
27
28 #include "ccp-dev.h"
29
30
31 enum ccp_memtype {
32 CCP_MEMTYPE_SYSTEM = 0,
33 CCP_MEMTYPE_KSB,
34 CCP_MEMTYPE_LOCAL,
35 CCP_MEMTYPE__LAST,
36 };
37
38 struct ccp_dma_info {
39 dma_addr_t address;
40 unsigned int offset;
41 unsigned int length;
42 enum dma_data_direction dir;
43 };
44
45 struct ccp_dm_workarea {
46 struct device *dev;
47 struct dma_pool *dma_pool;
48 unsigned int length;
49
50 u8 *address;
51 struct ccp_dma_info dma;
52 };
53
54 struct ccp_sg_workarea {
55 struct scatterlist *sg;
56 unsigned int nents;
57 unsigned int length;
58
59 struct scatterlist *dma_sg;
60 struct device *dma_dev;
61 unsigned int dma_count;
62 enum dma_data_direction dma_dir;
63
64 unsigned int sg_used;
65
66 u64 bytes_left;
67 };
68
69 struct ccp_data {
70 struct ccp_sg_workarea sg_wa;
71 struct ccp_dm_workarea dm_wa;
72 };
73
74 struct ccp_mem {
75 enum ccp_memtype type;
76 union {
77 struct ccp_dma_info dma;
78 u32 ksb;
79 } u;
80 };
81
82 struct ccp_aes_op {
83 enum ccp_aes_type type;
84 enum ccp_aes_mode mode;
85 enum ccp_aes_action action;
86 };
87
88 struct ccp_xts_aes_op {
89 enum ccp_aes_action action;
90 enum ccp_xts_aes_unit_size unit_size;
91 };
92
93 struct ccp_sha_op {
94 enum ccp_sha_type type;
95 u64 msg_bits;
96 };
97
98 struct ccp_rsa_op {
99 u32 mod_size;
100 u32 input_len;
101 };
102
103 struct ccp_passthru_op {
104 enum ccp_passthru_bitwise bit_mod;
105 enum ccp_passthru_byteswap byte_swap;
106 };
107
108 struct ccp_ecc_op {
109 enum ccp_ecc_function function;
110 };
111
112 struct ccp_op {
113 struct ccp_cmd_queue *cmd_q;
114
115 u32 jobid;
116 u32 ioc;
117 u32 soc;
118 u32 ksb_key;
119 u32 ksb_ctx;
120 u32 init;
121 u32 eom;
122
123 struct ccp_mem src;
124 struct ccp_mem dst;
125
126 union {
127 struct ccp_aes_op aes;
128 struct ccp_xts_aes_op xts;
129 struct ccp_sha_op sha;
130 struct ccp_rsa_op rsa;
131 struct ccp_passthru_op passthru;
132 struct ccp_ecc_op ecc;
133 } u;
134 };
135
136 /* SHA initial context values */
137 static const __be32 ccp_sha1_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
138 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
139 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
140 cpu_to_be32(SHA1_H4), 0, 0, 0,
141 };
142
143 static const __be32 ccp_sha224_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
144 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
145 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
146 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
147 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
148 };
149
150 static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = {
151 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
152 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
153 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
154 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
155 };
156
157 /* The CCP cannot perform zero-length sha operations so the caller
158 * is required to buffer data for the final operation. However, a
159 * sha operation for a message with a total length of zero is valid
160 * so known values are required to supply the result.
161 */
162 static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = {
163 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d,
164 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90,
165 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00,
166 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
167 };
168
169 static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = {
170 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9,
171 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4,
172 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a,
173 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00,
174 };
175
176 static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = {
177 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14,
178 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24,
179 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c,
180 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55,
181 };
182
183 static u32 ccp_addr_lo(struct ccp_dma_info *info)
184 {
185 return lower_32_bits(info->address + info->offset);
186 }
187
188 static u32 ccp_addr_hi(struct ccp_dma_info *info)
189 {
190 return upper_32_bits(info->address + info->offset) & 0x0000ffff;
191 }
192
193 static int ccp_do_cmd(struct ccp_op *op, u32 *cr, unsigned int cr_count)
194 {
195 struct ccp_cmd_queue *cmd_q = op->cmd_q;
196 struct ccp_device *ccp = cmd_q->ccp;
197 void __iomem *cr_addr;
198 u32 cr0, cmd;
199 unsigned int i;
200 int ret = 0;
201
202 /* We could read a status register to see how many free slots
203 * are actually available, but reading that register resets it
204 * and you could lose some error information.
205 */
206 cmd_q->free_slots--;
207
208 cr0 = (cmd_q->id << REQ0_CMD_Q_SHIFT)
209 | (op->jobid << REQ0_JOBID_SHIFT)
210 | REQ0_WAIT_FOR_WRITE;
211
212 if (op->soc)
213 cr0 |= REQ0_STOP_ON_COMPLETE
214 | REQ0_INT_ON_COMPLETE;
215
216 if (op->ioc || !cmd_q->free_slots)
217 cr0 |= REQ0_INT_ON_COMPLETE;
218
219 /* Start at CMD_REQ1 */
220 cr_addr = ccp->io_regs + CMD_REQ0 + CMD_REQ_INCR;
221
222 mutex_lock(&ccp->req_mutex);
223
224 /* Write CMD_REQ1 through CMD_REQx first */
225 for (i = 0; i < cr_count; i++, cr_addr += CMD_REQ_INCR)
226 iowrite32(*(cr + i), cr_addr);
227
228 /* Tell the CCP to start */
229 wmb();
230 iowrite32(cr0, ccp->io_regs + CMD_REQ0);
231
232 mutex_unlock(&ccp->req_mutex);
233
234 if (cr0 & REQ0_INT_ON_COMPLETE) {
235 /* Wait for the job to complete */
236 ret = wait_event_interruptible(cmd_q->int_queue,
237 cmd_q->int_rcvd);
238 if (ret || cmd_q->cmd_error) {
239 /* On error delete all related jobs from the queue */
240 cmd = (cmd_q->id << DEL_Q_ID_SHIFT)
241 | op->jobid;
242
243 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
244
245 if (!ret)
246 ret = -EIO;
247 } else if (op->soc) {
248 /* Delete just head job from the queue on SoC */
249 cmd = DEL_Q_ACTIVE
250 | (cmd_q->id << DEL_Q_ID_SHIFT)
251 | op->jobid;
252
253 iowrite32(cmd, ccp->io_regs + DEL_CMD_Q_JOB);
254 }
255
256 cmd_q->free_slots = CMD_Q_DEPTH(cmd_q->q_status);
257
258 cmd_q->int_rcvd = 0;
259 }
260
261 return ret;
262 }
263
264 static int ccp_perform_aes(struct ccp_op *op)
265 {
266 u32 cr[6];
267
268 /* Fill out the register contents for REQ1 through REQ6 */
269 cr[0] = (CCP_ENGINE_AES << REQ1_ENGINE_SHIFT)
270 | (op->u.aes.type << REQ1_AES_TYPE_SHIFT)
271 | (op->u.aes.mode << REQ1_AES_MODE_SHIFT)
272 | (op->u.aes.action << REQ1_AES_ACTION_SHIFT)
273 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
274 cr[1] = op->src.u.dma.length - 1;
275 cr[2] = ccp_addr_lo(&op->src.u.dma);
276 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
277 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
278 | ccp_addr_hi(&op->src.u.dma);
279 cr[4] = ccp_addr_lo(&op->dst.u.dma);
280 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
281 | ccp_addr_hi(&op->dst.u.dma);
282
283 if (op->u.aes.mode == CCP_AES_MODE_CFB)
284 cr[0] |= ((0x7f) << REQ1_AES_CFB_SIZE_SHIFT);
285
286 if (op->eom)
287 cr[0] |= REQ1_EOM;
288
289 if (op->init)
290 cr[0] |= REQ1_INIT;
291
292 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
293 }
294
295 static int ccp_perform_xts_aes(struct ccp_op *op)
296 {
297 u32 cr[6];
298
299 /* Fill out the register contents for REQ1 through REQ6 */
300 cr[0] = (CCP_ENGINE_XTS_AES_128 << REQ1_ENGINE_SHIFT)
301 | (op->u.xts.action << REQ1_AES_ACTION_SHIFT)
302 | (op->u.xts.unit_size << REQ1_XTS_AES_SIZE_SHIFT)
303 | (op->ksb_key << REQ1_KEY_KSB_SHIFT);
304 cr[1] = op->src.u.dma.length - 1;
305 cr[2] = ccp_addr_lo(&op->src.u.dma);
306 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
307 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
308 | ccp_addr_hi(&op->src.u.dma);
309 cr[4] = ccp_addr_lo(&op->dst.u.dma);
310 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
311 | ccp_addr_hi(&op->dst.u.dma);
312
313 if (op->eom)
314 cr[0] |= REQ1_EOM;
315
316 if (op->init)
317 cr[0] |= REQ1_INIT;
318
319 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
320 }
321
322 static int ccp_perform_sha(struct ccp_op *op)
323 {
324 u32 cr[6];
325
326 /* Fill out the register contents for REQ1 through REQ6 */
327 cr[0] = (CCP_ENGINE_SHA << REQ1_ENGINE_SHIFT)
328 | (op->u.sha.type << REQ1_SHA_TYPE_SHIFT)
329 | REQ1_INIT;
330 cr[1] = op->src.u.dma.length - 1;
331 cr[2] = ccp_addr_lo(&op->src.u.dma);
332 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
333 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
334 | ccp_addr_hi(&op->src.u.dma);
335
336 if (op->eom) {
337 cr[0] |= REQ1_EOM;
338 cr[4] = lower_32_bits(op->u.sha.msg_bits);
339 cr[5] = upper_32_bits(op->u.sha.msg_bits);
340 } else {
341 cr[4] = 0;
342 cr[5] = 0;
343 }
344
345 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
346 }
347
348 static int ccp_perform_rsa(struct ccp_op *op)
349 {
350 u32 cr[6];
351
352 /* Fill out the register contents for REQ1 through REQ6 */
353 cr[0] = (CCP_ENGINE_RSA << REQ1_ENGINE_SHIFT)
354 | (op->u.rsa.mod_size << REQ1_RSA_MOD_SIZE_SHIFT)
355 | (op->ksb_key << REQ1_KEY_KSB_SHIFT)
356 | REQ1_EOM;
357 cr[1] = op->u.rsa.input_len - 1;
358 cr[2] = ccp_addr_lo(&op->src.u.dma);
359 cr[3] = (op->ksb_ctx << REQ4_KSB_SHIFT)
360 | (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
361 | ccp_addr_hi(&op->src.u.dma);
362 cr[4] = ccp_addr_lo(&op->dst.u.dma);
363 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
364 | ccp_addr_hi(&op->dst.u.dma);
365
366 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
367 }
368
369 static int ccp_perform_passthru(struct ccp_op *op)
370 {
371 u32 cr[6];
372
373 /* Fill out the register contents for REQ1 through REQ6 */
374 cr[0] = (CCP_ENGINE_PASSTHRU << REQ1_ENGINE_SHIFT)
375 | (op->u.passthru.bit_mod << REQ1_PT_BW_SHIFT)
376 | (op->u.passthru.byte_swap << REQ1_PT_BS_SHIFT);
377
378 if (op->src.type == CCP_MEMTYPE_SYSTEM)
379 cr[1] = op->src.u.dma.length - 1;
380 else
381 cr[1] = op->dst.u.dma.length - 1;
382
383 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
384 cr[2] = ccp_addr_lo(&op->src.u.dma);
385 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
386 | ccp_addr_hi(&op->src.u.dma);
387
388 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
389 cr[3] |= (op->ksb_key << REQ4_KSB_SHIFT);
390 } else {
391 cr[2] = op->src.u.ksb * CCP_KSB_BYTES;
392 cr[3] = (CCP_MEMTYPE_KSB << REQ4_MEMTYPE_SHIFT);
393 }
394
395 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
396 cr[4] = ccp_addr_lo(&op->dst.u.dma);
397 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
398 | ccp_addr_hi(&op->dst.u.dma);
399 } else {
400 cr[4] = op->dst.u.ksb * CCP_KSB_BYTES;
401 cr[5] = (CCP_MEMTYPE_KSB << REQ6_MEMTYPE_SHIFT);
402 }
403
404 if (op->eom)
405 cr[0] |= REQ1_EOM;
406
407 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
408 }
409
410 static int ccp_perform_ecc(struct ccp_op *op)
411 {
412 u32 cr[6];
413
414 /* Fill out the register contents for REQ1 through REQ6 */
415 cr[0] = REQ1_ECC_AFFINE_CONVERT
416 | (CCP_ENGINE_ECC << REQ1_ENGINE_SHIFT)
417 | (op->u.ecc.function << REQ1_ECC_FUNCTION_SHIFT)
418 | REQ1_EOM;
419 cr[1] = op->src.u.dma.length - 1;
420 cr[2] = ccp_addr_lo(&op->src.u.dma);
421 cr[3] = (CCP_MEMTYPE_SYSTEM << REQ4_MEMTYPE_SHIFT)
422 | ccp_addr_hi(&op->src.u.dma);
423 cr[4] = ccp_addr_lo(&op->dst.u.dma);
424 cr[5] = (CCP_MEMTYPE_SYSTEM << REQ6_MEMTYPE_SHIFT)
425 | ccp_addr_hi(&op->dst.u.dma);
426
427 return ccp_do_cmd(op, cr, ARRAY_SIZE(cr));
428 }
429
430 static u32 ccp_alloc_ksb(struct ccp_device *ccp, unsigned int count)
431 {
432 int start;
433
434 for (;;) {
435 mutex_lock(&ccp->ksb_mutex);
436
437 start = (u32)bitmap_find_next_zero_area(ccp->ksb,
438 ccp->ksb_count,
439 ccp->ksb_start,
440 count, 0);
441 if (start <= ccp->ksb_count) {
442 bitmap_set(ccp->ksb, start, count);
443
444 mutex_unlock(&ccp->ksb_mutex);
445 break;
446 }
447
448 ccp->ksb_avail = 0;
449
450 mutex_unlock(&ccp->ksb_mutex);
451
452 /* Wait for KSB entries to become available */
453 if (wait_event_interruptible(ccp->ksb_queue, ccp->ksb_avail))
454 return 0;
455 }
456
457 return KSB_START + start;
458 }
459
460 static void ccp_free_ksb(struct ccp_device *ccp, unsigned int start,
461 unsigned int count)
462 {
463 if (!start)
464 return;
465
466 mutex_lock(&ccp->ksb_mutex);
467
468 bitmap_clear(ccp->ksb, start - KSB_START, count);
469
470 ccp->ksb_avail = 1;
471
472 mutex_unlock(&ccp->ksb_mutex);
473
474 wake_up_interruptible_all(&ccp->ksb_queue);
475 }
476
477 static u32 ccp_gen_jobid(struct ccp_device *ccp)
478 {
479 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
480 }
481
482 static void ccp_sg_free(struct ccp_sg_workarea *wa)
483 {
484 if (wa->dma_count)
485 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
486
487 wa->dma_count = 0;
488 }
489
490 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
491 struct scatterlist *sg, u64 len,
492 enum dma_data_direction dma_dir)
493 {
494 memset(wa, 0, sizeof(*wa));
495
496 wa->sg = sg;
497 if (!sg)
498 return 0;
499
500 wa->nents = sg_nents(sg);
501 wa->length = sg->length;
502 wa->bytes_left = len;
503 wa->sg_used = 0;
504
505 if (len == 0)
506 return 0;
507
508 if (dma_dir == DMA_NONE)
509 return 0;
510
511 wa->dma_sg = sg;
512 wa->dma_dev = dev;
513 wa->dma_dir = dma_dir;
514 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
515 if (!wa->dma_count)
516 return -ENOMEM;
517
518
519 return 0;
520 }
521
522 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
523 {
524 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
525
526 if (!wa->sg)
527 return;
528
529 wa->sg_used += nbytes;
530 wa->bytes_left -= nbytes;
531 if (wa->sg_used == wa->sg->length) {
532 wa->sg = sg_next(wa->sg);
533 wa->sg_used = 0;
534 }
535 }
536
537 static void ccp_dm_free(struct ccp_dm_workarea *wa)
538 {
539 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
540 if (wa->address)
541 dma_pool_free(wa->dma_pool, wa->address,
542 wa->dma.address);
543 } else {
544 if (wa->dma.address)
545 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
546 wa->dma.dir);
547 kfree(wa->address);
548 }
549
550 wa->address = NULL;
551 wa->dma.address = 0;
552 }
553
554 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
555 struct ccp_cmd_queue *cmd_q,
556 unsigned int len,
557 enum dma_data_direction dir)
558 {
559 memset(wa, 0, sizeof(*wa));
560
561 if (!len)
562 return 0;
563
564 wa->dev = cmd_q->ccp->dev;
565 wa->length = len;
566
567 if (len <= CCP_DMAPOOL_MAX_SIZE) {
568 wa->dma_pool = cmd_q->dma_pool;
569
570 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
571 &wa->dma.address);
572 if (!wa->address)
573 return -ENOMEM;
574
575 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
576
577 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
578 } else {
579 wa->address = kzalloc(len, GFP_KERNEL);
580 if (!wa->address)
581 return -ENOMEM;
582
583 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
584 dir);
585 if (!wa->dma.address)
586 return -ENOMEM;
587
588 wa->dma.length = len;
589 }
590 wa->dma.dir = dir;
591
592 return 0;
593 }
594
595 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
596 struct scatterlist *sg, unsigned int sg_offset,
597 unsigned int len)
598 {
599 WARN_ON(!wa->address);
600
601 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
602 0);
603 }
604
605 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
606 struct scatterlist *sg, unsigned int sg_offset,
607 unsigned int len)
608 {
609 WARN_ON(!wa->address);
610
611 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
612 1);
613 }
614
615 static void ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
616 struct scatterlist *sg,
617 unsigned int len, unsigned int se_len,
618 bool sign_extend)
619 {
620 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
621 u8 buffer[CCP_REVERSE_BUF_SIZE];
622
623 BUG_ON(se_len > sizeof(buffer));
624
625 sg_offset = len;
626 dm_offset = 0;
627 nbytes = len;
628 while (nbytes) {
629 ksb_len = min_t(unsigned int, nbytes, se_len);
630 sg_offset -= ksb_len;
631
632 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 0);
633 for (i = 0; i < ksb_len; i++)
634 wa->address[dm_offset + i] = buffer[ksb_len - i - 1];
635
636 dm_offset += ksb_len;
637 nbytes -= ksb_len;
638
639 if ((ksb_len != se_len) && sign_extend) {
640 /* Must sign-extend to nearest sign-extend length */
641 if (wa->address[dm_offset - 1] & 0x80)
642 memset(wa->address + dm_offset, 0xff,
643 se_len - ksb_len);
644 }
645 }
646 }
647
648 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
649 struct scatterlist *sg,
650 unsigned int len)
651 {
652 unsigned int nbytes, sg_offset, dm_offset, ksb_len, i;
653 u8 buffer[CCP_REVERSE_BUF_SIZE];
654
655 sg_offset = 0;
656 dm_offset = len;
657 nbytes = len;
658 while (nbytes) {
659 ksb_len = min_t(unsigned int, nbytes, sizeof(buffer));
660 dm_offset -= ksb_len;
661
662 for (i = 0; i < ksb_len; i++)
663 buffer[ksb_len - i - 1] = wa->address[dm_offset + i];
664 scatterwalk_map_and_copy(buffer, sg, sg_offset, ksb_len, 1);
665
666 sg_offset += ksb_len;
667 nbytes -= ksb_len;
668 }
669 }
670
671 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
672 {
673 ccp_dm_free(&data->dm_wa);
674 ccp_sg_free(&data->sg_wa);
675 }
676
677 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
678 struct scatterlist *sg, u64 sg_len,
679 unsigned int dm_len,
680 enum dma_data_direction dir)
681 {
682 int ret;
683
684 memset(data, 0, sizeof(*data));
685
686 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
687 dir);
688 if (ret)
689 goto e_err;
690
691 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
692 if (ret)
693 goto e_err;
694
695 return 0;
696
697 e_err:
698 ccp_free_data(data, cmd_q);
699
700 return ret;
701 }
702
703 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
704 {
705 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
706 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
707 unsigned int buf_count, nbytes;
708
709 /* Clear the buffer if setting it */
710 if (!from)
711 memset(dm_wa->address, 0, dm_wa->length);
712
713 if (!sg_wa->sg)
714 return 0;
715
716 /* Perform the copy operation
717 * nbytes will always be <= UINT_MAX because dm_wa->length is
718 * an unsigned int
719 */
720 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
721 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
722 nbytes, from);
723
724 /* Update the structures and generate the count */
725 buf_count = 0;
726 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
727 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
728 dm_wa->length - buf_count);
729 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
730
731 buf_count += nbytes;
732 ccp_update_sg_workarea(sg_wa, nbytes);
733 }
734
735 return buf_count;
736 }
737
738 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
739 {
740 return ccp_queue_buf(data, 0);
741 }
742
743 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
744 {
745 return ccp_queue_buf(data, 1);
746 }
747
748 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
749 struct ccp_op *op, unsigned int block_size,
750 bool blocksize_op)
751 {
752 unsigned int sg_src_len, sg_dst_len, op_len;
753
754 /* The CCP can only DMA from/to one address each per operation. This
755 * requires that we find the smallest DMA area between the source
756 * and destination. The resulting len values will always be <= UINT_MAX
757 * because the dma length is an unsigned int.
758 */
759 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
760 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
761
762 if (dst) {
763 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
764 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
765 op_len = min(sg_src_len, sg_dst_len);
766 } else
767 op_len = sg_src_len;
768
769 /* The data operation length will be at least block_size in length
770 * or the smaller of available sg room remaining for the source or
771 * the destination
772 */
773 op_len = max(op_len, block_size);
774
775 /* Unless we have to buffer data, there's no reason to wait */
776 op->soc = 0;
777
778 if (sg_src_len < block_size) {
779 /* Not enough data in the sg element, so it
780 * needs to be buffered into a blocksize chunk
781 */
782 int cp_len = ccp_fill_queue_buf(src);
783
784 op->soc = 1;
785 op->src.u.dma.address = src->dm_wa.dma.address;
786 op->src.u.dma.offset = 0;
787 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
788 } else {
789 /* Enough data in the sg element, but we need to
790 * adjust for any previously copied data
791 */
792 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
793 op->src.u.dma.offset = src->sg_wa.sg_used;
794 op->src.u.dma.length = op_len & ~(block_size - 1);
795
796 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
797 }
798
799 if (dst) {
800 if (sg_dst_len < block_size) {
801 /* Not enough room in the sg element or we're on the
802 * last piece of data (when using padding), so the
803 * output needs to be buffered into a blocksize chunk
804 */
805 op->soc = 1;
806 op->dst.u.dma.address = dst->dm_wa.dma.address;
807 op->dst.u.dma.offset = 0;
808 op->dst.u.dma.length = op->src.u.dma.length;
809 } else {
810 /* Enough room in the sg element, but we need to
811 * adjust for any previously used area
812 */
813 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
814 op->dst.u.dma.offset = dst->sg_wa.sg_used;
815 op->dst.u.dma.length = op->src.u.dma.length;
816 }
817 }
818 }
819
820 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
821 struct ccp_op *op)
822 {
823 op->init = 0;
824
825 if (dst) {
826 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
827 ccp_empty_queue_buf(dst);
828 else
829 ccp_update_sg_workarea(&dst->sg_wa,
830 op->dst.u.dma.length);
831 }
832 }
833
834 static int ccp_copy_to_from_ksb(struct ccp_cmd_queue *cmd_q,
835 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
836 u32 byte_swap, bool from)
837 {
838 struct ccp_op op;
839
840 memset(&op, 0, sizeof(op));
841
842 op.cmd_q = cmd_q;
843 op.jobid = jobid;
844 op.eom = 1;
845
846 if (from) {
847 op.soc = 1;
848 op.src.type = CCP_MEMTYPE_KSB;
849 op.src.u.ksb = ksb;
850 op.dst.type = CCP_MEMTYPE_SYSTEM;
851 op.dst.u.dma.address = wa->dma.address;
852 op.dst.u.dma.length = wa->length;
853 } else {
854 op.src.type = CCP_MEMTYPE_SYSTEM;
855 op.src.u.dma.address = wa->dma.address;
856 op.src.u.dma.length = wa->length;
857 op.dst.type = CCP_MEMTYPE_KSB;
858 op.dst.u.ksb = ksb;
859 }
860
861 op.u.passthru.byte_swap = byte_swap;
862
863 return ccp_perform_passthru(&op);
864 }
865
866 static int ccp_copy_to_ksb(struct ccp_cmd_queue *cmd_q,
867 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
868 u32 byte_swap)
869 {
870 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, false);
871 }
872
873 static int ccp_copy_from_ksb(struct ccp_cmd_queue *cmd_q,
874 struct ccp_dm_workarea *wa, u32 jobid, u32 ksb,
875 u32 byte_swap)
876 {
877 return ccp_copy_to_from_ksb(cmd_q, wa, jobid, ksb, byte_swap, true);
878 }
879
880 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
881 struct ccp_cmd *cmd)
882 {
883 struct ccp_aes_engine *aes = &cmd->u.aes;
884 struct ccp_dm_workarea key, ctx;
885 struct ccp_data src;
886 struct ccp_op op;
887 unsigned int dm_offset;
888 int ret;
889
890 if (!((aes->key_len == AES_KEYSIZE_128) ||
891 (aes->key_len == AES_KEYSIZE_192) ||
892 (aes->key_len == AES_KEYSIZE_256)))
893 return -EINVAL;
894
895 if (aes->src_len & (AES_BLOCK_SIZE - 1))
896 return -EINVAL;
897
898 if (aes->iv_len != AES_BLOCK_SIZE)
899 return -EINVAL;
900
901 if (!aes->key || !aes->iv || !aes->src)
902 return -EINVAL;
903
904 if (aes->cmac_final) {
905 if (aes->cmac_key_len != AES_BLOCK_SIZE)
906 return -EINVAL;
907
908 if (!aes->cmac_key)
909 return -EINVAL;
910 }
911
912 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
913 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
914
915 ret = -EIO;
916 memset(&op, 0, sizeof(op));
917 op.cmd_q = cmd_q;
918 op.jobid = ccp_gen_jobid(cmd_q->ccp);
919 op.ksb_key = cmd_q->ksb_key;
920 op.ksb_ctx = cmd_q->ksb_ctx;
921 op.init = 1;
922 op.u.aes.type = aes->type;
923 op.u.aes.mode = aes->mode;
924 op.u.aes.action = aes->action;
925
926 /* All supported key sizes fit in a single (32-byte) KSB entry
927 * and must be in little endian format. Use the 256-bit byte
928 * swap passthru option to convert from big endian to little
929 * endian.
930 */
931 ret = ccp_init_dm_workarea(&key, cmd_q,
932 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
933 DMA_TO_DEVICE);
934 if (ret)
935 return ret;
936
937 dm_offset = CCP_KSB_BYTES - aes->key_len;
938 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
939 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
940 CCP_PASSTHRU_BYTESWAP_256BIT);
941 if (ret) {
942 cmd->engine_error = cmd_q->cmd_error;
943 goto e_key;
944 }
945
946 /* The AES context fits in a single (32-byte) KSB entry and
947 * must be in little endian format. Use the 256-bit byte swap
948 * passthru option to convert from big endian to little endian.
949 */
950 ret = ccp_init_dm_workarea(&ctx, cmd_q,
951 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
952 DMA_BIDIRECTIONAL);
953 if (ret)
954 goto e_key;
955
956 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
957 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
958 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
959 CCP_PASSTHRU_BYTESWAP_256BIT);
960 if (ret) {
961 cmd->engine_error = cmd_q->cmd_error;
962 goto e_ctx;
963 }
964
965 /* Send data to the CCP AES engine */
966 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
967 AES_BLOCK_SIZE, DMA_TO_DEVICE);
968 if (ret)
969 goto e_ctx;
970
971 while (src.sg_wa.bytes_left) {
972 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
973 if (aes->cmac_final && !src.sg_wa.bytes_left) {
974 op.eom = 1;
975
976 /* Push the K1/K2 key to the CCP now */
977 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid,
978 op.ksb_ctx,
979 CCP_PASSTHRU_BYTESWAP_256BIT);
980 if (ret) {
981 cmd->engine_error = cmd_q->cmd_error;
982 goto e_src;
983 }
984
985 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
986 aes->cmac_key_len);
987 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
988 CCP_PASSTHRU_BYTESWAP_256BIT);
989 if (ret) {
990 cmd->engine_error = cmd_q->cmd_error;
991 goto e_src;
992 }
993 }
994
995 ret = ccp_perform_aes(&op);
996 if (ret) {
997 cmd->engine_error = cmd_q->cmd_error;
998 goto e_src;
999 }
1000
1001 ccp_process_data(&src, NULL, &op);
1002 }
1003
1004 /* Retrieve the AES context - convert from LE to BE using
1005 * 32-byte (256-bit) byteswapping
1006 */
1007 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1008 CCP_PASSTHRU_BYTESWAP_256BIT);
1009 if (ret) {
1010 cmd->engine_error = cmd_q->cmd_error;
1011 goto e_src;
1012 }
1013
1014 /* ...but we only need AES_BLOCK_SIZE bytes */
1015 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1016 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1017
1018 e_src:
1019 ccp_free_data(&src, cmd_q);
1020
1021 e_ctx:
1022 ccp_dm_free(&ctx);
1023
1024 e_key:
1025 ccp_dm_free(&key);
1026
1027 return ret;
1028 }
1029
1030 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1031 {
1032 struct ccp_aes_engine *aes = &cmd->u.aes;
1033 struct ccp_dm_workarea key, ctx;
1034 struct ccp_data src, dst;
1035 struct ccp_op op;
1036 unsigned int dm_offset;
1037 bool in_place = false;
1038 int ret;
1039
1040 if (aes->mode == CCP_AES_MODE_CMAC)
1041 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
1042
1043 if (!((aes->key_len == AES_KEYSIZE_128) ||
1044 (aes->key_len == AES_KEYSIZE_192) ||
1045 (aes->key_len == AES_KEYSIZE_256)))
1046 return -EINVAL;
1047
1048 if (((aes->mode == CCP_AES_MODE_ECB) ||
1049 (aes->mode == CCP_AES_MODE_CBC) ||
1050 (aes->mode == CCP_AES_MODE_CFB)) &&
1051 (aes->src_len & (AES_BLOCK_SIZE - 1)))
1052 return -EINVAL;
1053
1054 if (!aes->key || !aes->src || !aes->dst)
1055 return -EINVAL;
1056
1057 if (aes->mode != CCP_AES_MODE_ECB) {
1058 if (aes->iv_len != AES_BLOCK_SIZE)
1059 return -EINVAL;
1060
1061 if (!aes->iv)
1062 return -EINVAL;
1063 }
1064
1065 BUILD_BUG_ON(CCP_AES_KEY_KSB_COUNT != 1);
1066 BUILD_BUG_ON(CCP_AES_CTX_KSB_COUNT != 1);
1067
1068 ret = -EIO;
1069 memset(&op, 0, sizeof(op));
1070 op.cmd_q = cmd_q;
1071 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1072 op.ksb_key = cmd_q->ksb_key;
1073 op.ksb_ctx = cmd_q->ksb_ctx;
1074 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
1075 op.u.aes.type = aes->type;
1076 op.u.aes.mode = aes->mode;
1077 op.u.aes.action = aes->action;
1078
1079 /* All supported key sizes fit in a single (32-byte) KSB entry
1080 * and must be in little endian format. Use the 256-bit byte
1081 * swap passthru option to convert from big endian to little
1082 * endian.
1083 */
1084 ret = ccp_init_dm_workarea(&key, cmd_q,
1085 CCP_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1086 DMA_TO_DEVICE);
1087 if (ret)
1088 return ret;
1089
1090 dm_offset = CCP_KSB_BYTES - aes->key_len;
1091 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
1092 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1093 CCP_PASSTHRU_BYTESWAP_256BIT);
1094 if (ret) {
1095 cmd->engine_error = cmd_q->cmd_error;
1096 goto e_key;
1097 }
1098
1099 /* The AES context fits in a single (32-byte) KSB entry and
1100 * must be in little endian format. Use the 256-bit byte swap
1101 * passthru option to convert from big endian to little endian.
1102 */
1103 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1104 CCP_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1105 DMA_BIDIRECTIONAL);
1106 if (ret)
1107 goto e_key;
1108
1109 if (aes->mode != CCP_AES_MODE_ECB) {
1110 /* Load the AES context - conver to LE */
1111 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1112 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1113 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1114 CCP_PASSTHRU_BYTESWAP_256BIT);
1115 if (ret) {
1116 cmd->engine_error = cmd_q->cmd_error;
1117 goto e_ctx;
1118 }
1119 }
1120
1121 /* Prepare the input and output data workareas. For in-place
1122 * operations we need to set the dma direction to BIDIRECTIONAL
1123 * and copy the src workarea to the dst workarea.
1124 */
1125 if (sg_virt(aes->src) == sg_virt(aes->dst))
1126 in_place = true;
1127
1128 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1129 AES_BLOCK_SIZE,
1130 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1131 if (ret)
1132 goto e_ctx;
1133
1134 if (in_place)
1135 dst = src;
1136 else {
1137 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1138 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1139 if (ret)
1140 goto e_src;
1141 }
1142
1143 /* Send data to the CCP AES engine */
1144 while (src.sg_wa.bytes_left) {
1145 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1146 if (!src.sg_wa.bytes_left) {
1147 op.eom = 1;
1148
1149 /* Since we don't retrieve the AES context in ECB
1150 * mode we have to wait for the operation to complete
1151 * on the last piece of data
1152 */
1153 if (aes->mode == CCP_AES_MODE_ECB)
1154 op.soc = 1;
1155 }
1156
1157 ret = ccp_perform_aes(&op);
1158 if (ret) {
1159 cmd->engine_error = cmd_q->cmd_error;
1160 goto e_dst;
1161 }
1162
1163 ccp_process_data(&src, &dst, &op);
1164 }
1165
1166 if (aes->mode != CCP_AES_MODE_ECB) {
1167 /* Retrieve the AES context - convert from LE to BE using
1168 * 32-byte (256-bit) byteswapping
1169 */
1170 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1171 CCP_PASSTHRU_BYTESWAP_256BIT);
1172 if (ret) {
1173 cmd->engine_error = cmd_q->cmd_error;
1174 goto e_dst;
1175 }
1176
1177 /* ...but we only need AES_BLOCK_SIZE bytes */
1178 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1179 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1180 }
1181
1182 e_dst:
1183 if (!in_place)
1184 ccp_free_data(&dst, cmd_q);
1185
1186 e_src:
1187 ccp_free_data(&src, cmd_q);
1188
1189 e_ctx:
1190 ccp_dm_free(&ctx);
1191
1192 e_key:
1193 ccp_dm_free(&key);
1194
1195 return ret;
1196 }
1197
1198 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1199 struct ccp_cmd *cmd)
1200 {
1201 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1202 struct ccp_dm_workarea key, ctx;
1203 struct ccp_data src, dst;
1204 struct ccp_op op;
1205 unsigned int unit_size, dm_offset;
1206 bool in_place = false;
1207 int ret;
1208
1209 switch (xts->unit_size) {
1210 case CCP_XTS_AES_UNIT_SIZE_16:
1211 unit_size = 16;
1212 break;
1213 case CCP_XTS_AES_UNIT_SIZE_512:
1214 unit_size = 512;
1215 break;
1216 case CCP_XTS_AES_UNIT_SIZE_1024:
1217 unit_size = 1024;
1218 break;
1219 case CCP_XTS_AES_UNIT_SIZE_2048:
1220 unit_size = 2048;
1221 break;
1222 case CCP_XTS_AES_UNIT_SIZE_4096:
1223 unit_size = 4096;
1224 break;
1225
1226 default:
1227 return -EINVAL;
1228 }
1229
1230 if (xts->key_len != AES_KEYSIZE_128)
1231 return -EINVAL;
1232
1233 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1234 return -EINVAL;
1235
1236 if (xts->iv_len != AES_BLOCK_SIZE)
1237 return -EINVAL;
1238
1239 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1240 return -EINVAL;
1241
1242 BUILD_BUG_ON(CCP_XTS_AES_KEY_KSB_COUNT != 1);
1243 BUILD_BUG_ON(CCP_XTS_AES_CTX_KSB_COUNT != 1);
1244
1245 ret = -EIO;
1246 memset(&op, 0, sizeof(op));
1247 op.cmd_q = cmd_q;
1248 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1249 op.ksb_key = cmd_q->ksb_key;
1250 op.ksb_ctx = cmd_q->ksb_ctx;
1251 op.init = 1;
1252 op.u.xts.action = xts->action;
1253 op.u.xts.unit_size = xts->unit_size;
1254
1255 /* All supported key sizes fit in a single (32-byte) KSB entry
1256 * and must be in little endian format. Use the 256-bit byte
1257 * swap passthru option to convert from big endian to little
1258 * endian.
1259 */
1260 ret = ccp_init_dm_workarea(&key, cmd_q,
1261 CCP_XTS_AES_KEY_KSB_COUNT * CCP_KSB_BYTES,
1262 DMA_TO_DEVICE);
1263 if (ret)
1264 return ret;
1265
1266 dm_offset = CCP_KSB_BYTES - AES_KEYSIZE_128;
1267 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1268 ccp_set_dm_area(&key, 0, xts->key, dm_offset, xts->key_len);
1269 ret = ccp_copy_to_ksb(cmd_q, &key, op.jobid, op.ksb_key,
1270 CCP_PASSTHRU_BYTESWAP_256BIT);
1271 if (ret) {
1272 cmd->engine_error = cmd_q->cmd_error;
1273 goto e_key;
1274 }
1275
1276 /* The AES context fits in a single (32-byte) KSB entry and
1277 * for XTS is already in little endian format so no byte swapping
1278 * is needed.
1279 */
1280 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1281 CCP_XTS_AES_CTX_KSB_COUNT * CCP_KSB_BYTES,
1282 DMA_BIDIRECTIONAL);
1283 if (ret)
1284 goto e_key;
1285
1286 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1287 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1288 CCP_PASSTHRU_BYTESWAP_NOOP);
1289 if (ret) {
1290 cmd->engine_error = cmd_q->cmd_error;
1291 goto e_ctx;
1292 }
1293
1294 /* Prepare the input and output data workareas. For in-place
1295 * operations we need to set the dma direction to BIDIRECTIONAL
1296 * and copy the src workarea to the dst workarea.
1297 */
1298 if (sg_virt(xts->src) == sg_virt(xts->dst))
1299 in_place = true;
1300
1301 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1302 unit_size,
1303 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1304 if (ret)
1305 goto e_ctx;
1306
1307 if (in_place)
1308 dst = src;
1309 else {
1310 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1311 unit_size, DMA_FROM_DEVICE);
1312 if (ret)
1313 goto e_src;
1314 }
1315
1316 /* Send data to the CCP AES engine */
1317 while (src.sg_wa.bytes_left) {
1318 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1319 if (!src.sg_wa.bytes_left)
1320 op.eom = 1;
1321
1322 ret = ccp_perform_xts_aes(&op);
1323 if (ret) {
1324 cmd->engine_error = cmd_q->cmd_error;
1325 goto e_dst;
1326 }
1327
1328 ccp_process_data(&src, &dst, &op);
1329 }
1330
1331 /* Retrieve the AES context - convert from LE to BE using
1332 * 32-byte (256-bit) byteswapping
1333 */
1334 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1335 CCP_PASSTHRU_BYTESWAP_256BIT);
1336 if (ret) {
1337 cmd->engine_error = cmd_q->cmd_error;
1338 goto e_dst;
1339 }
1340
1341 /* ...but we only need AES_BLOCK_SIZE bytes */
1342 dm_offset = CCP_KSB_BYTES - AES_BLOCK_SIZE;
1343 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1344
1345 e_dst:
1346 if (!in_place)
1347 ccp_free_data(&dst, cmd_q);
1348
1349 e_src:
1350 ccp_free_data(&src, cmd_q);
1351
1352 e_ctx:
1353 ccp_dm_free(&ctx);
1354
1355 e_key:
1356 ccp_dm_free(&key);
1357
1358 return ret;
1359 }
1360
1361 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1362 {
1363 struct ccp_sha_engine *sha = &cmd->u.sha;
1364 struct ccp_dm_workarea ctx;
1365 struct ccp_data src;
1366 struct ccp_op op;
1367 int ret;
1368
1369 if (sha->ctx_len != CCP_SHA_CTXSIZE)
1370 return -EINVAL;
1371
1372 if (!sha->ctx)
1373 return -EINVAL;
1374
1375 if (!sha->final && (sha->src_len & (CCP_SHA_BLOCKSIZE - 1)))
1376 return -EINVAL;
1377
1378 if (!sha->src_len) {
1379 const u8 *sha_zero;
1380
1381 /* Not final, just return */
1382 if (!sha->final)
1383 return 0;
1384
1385 /* CCP can't do a zero length sha operation so the caller
1386 * must buffer the data.
1387 */
1388 if (sha->msg_bits)
1389 return -EINVAL;
1390
1391 /* A sha operation for a message with a total length of zero,
1392 * return known result.
1393 */
1394 switch (sha->type) {
1395 case CCP_SHA_TYPE_1:
1396 sha_zero = ccp_sha1_zero;
1397 break;
1398 case CCP_SHA_TYPE_224:
1399 sha_zero = ccp_sha224_zero;
1400 break;
1401 case CCP_SHA_TYPE_256:
1402 sha_zero = ccp_sha256_zero;
1403 break;
1404 default:
1405 return -EINVAL;
1406 }
1407
1408 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1409 sha->ctx_len, 1);
1410
1411 return 0;
1412 }
1413
1414 if (!sha->src)
1415 return -EINVAL;
1416
1417 BUILD_BUG_ON(CCP_SHA_KSB_COUNT != 1);
1418
1419 memset(&op, 0, sizeof(op));
1420 op.cmd_q = cmd_q;
1421 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1422 op.ksb_ctx = cmd_q->ksb_ctx;
1423 op.u.sha.type = sha->type;
1424 op.u.sha.msg_bits = sha->msg_bits;
1425
1426 /* The SHA context fits in a single (32-byte) KSB entry and
1427 * must be in little endian format. Use the 256-bit byte swap
1428 * passthru option to convert from big endian to little endian.
1429 */
1430 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1431 CCP_SHA_KSB_COUNT * CCP_KSB_BYTES,
1432 DMA_BIDIRECTIONAL);
1433 if (ret)
1434 return ret;
1435
1436 if (sha->first) {
1437 const __be32 *init;
1438
1439 switch (sha->type) {
1440 case CCP_SHA_TYPE_1:
1441 init = ccp_sha1_init;
1442 break;
1443 case CCP_SHA_TYPE_224:
1444 init = ccp_sha224_init;
1445 break;
1446 case CCP_SHA_TYPE_256:
1447 init = ccp_sha256_init;
1448 break;
1449 default:
1450 ret = -EINVAL;
1451 goto e_ctx;
1452 }
1453 memcpy(ctx.address, init, CCP_SHA_CTXSIZE);
1454 } else
1455 ccp_set_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1456
1457 ret = ccp_copy_to_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1458 CCP_PASSTHRU_BYTESWAP_256BIT);
1459 if (ret) {
1460 cmd->engine_error = cmd_q->cmd_error;
1461 goto e_ctx;
1462 }
1463
1464 /* Send data to the CCP SHA engine */
1465 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1466 CCP_SHA_BLOCKSIZE, DMA_TO_DEVICE);
1467 if (ret)
1468 goto e_ctx;
1469
1470 while (src.sg_wa.bytes_left) {
1471 ccp_prepare_data(&src, NULL, &op, CCP_SHA_BLOCKSIZE, false);
1472 if (sha->final && !src.sg_wa.bytes_left)
1473 op.eom = 1;
1474
1475 ret = ccp_perform_sha(&op);
1476 if (ret) {
1477 cmd->engine_error = cmd_q->cmd_error;
1478 goto e_data;
1479 }
1480
1481 ccp_process_data(&src, NULL, &op);
1482 }
1483
1484 /* Retrieve the SHA context - convert from LE to BE using
1485 * 32-byte (256-bit) byteswapping to BE
1486 */
1487 ret = ccp_copy_from_ksb(cmd_q, &ctx, op.jobid, op.ksb_ctx,
1488 CCP_PASSTHRU_BYTESWAP_256BIT);
1489 if (ret) {
1490 cmd->engine_error = cmd_q->cmd_error;
1491 goto e_data;
1492 }
1493
1494 ccp_get_dm_area(&ctx, 0, sha->ctx, 0, sha->ctx_len);
1495
1496 if (sha->final && sha->opad) {
1497 /* HMAC operation, recursively perform final SHA */
1498 struct ccp_cmd hmac_cmd;
1499 struct scatterlist sg;
1500 u64 block_size, digest_size;
1501 u8 *hmac_buf;
1502
1503 switch (sha->type) {
1504 case CCP_SHA_TYPE_1:
1505 block_size = SHA1_BLOCK_SIZE;
1506 digest_size = SHA1_DIGEST_SIZE;
1507 break;
1508 case CCP_SHA_TYPE_224:
1509 block_size = SHA224_BLOCK_SIZE;
1510 digest_size = SHA224_DIGEST_SIZE;
1511 break;
1512 case CCP_SHA_TYPE_256:
1513 block_size = SHA256_BLOCK_SIZE;
1514 digest_size = SHA256_DIGEST_SIZE;
1515 break;
1516 default:
1517 ret = -EINVAL;
1518 goto e_data;
1519 }
1520
1521 if (sha->opad_len != block_size) {
1522 ret = -EINVAL;
1523 goto e_data;
1524 }
1525
1526 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1527 if (!hmac_buf) {
1528 ret = -ENOMEM;
1529 goto e_data;
1530 }
1531 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1532
1533 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1534 memcpy(hmac_buf + block_size, ctx.address, digest_size);
1535
1536 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1537 hmac_cmd.engine = CCP_ENGINE_SHA;
1538 hmac_cmd.u.sha.type = sha->type;
1539 hmac_cmd.u.sha.ctx = sha->ctx;
1540 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1541 hmac_cmd.u.sha.src = &sg;
1542 hmac_cmd.u.sha.src_len = block_size + digest_size;
1543 hmac_cmd.u.sha.opad = NULL;
1544 hmac_cmd.u.sha.opad_len = 0;
1545 hmac_cmd.u.sha.first = 1;
1546 hmac_cmd.u.sha.final = 1;
1547 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1548
1549 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1550 if (ret)
1551 cmd->engine_error = hmac_cmd.engine_error;
1552
1553 kfree(hmac_buf);
1554 }
1555
1556 e_data:
1557 ccp_free_data(&src, cmd_q);
1558
1559 e_ctx:
1560 ccp_dm_free(&ctx);
1561
1562 return ret;
1563 }
1564
1565 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1566 {
1567 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1568 struct ccp_dm_workarea exp, src;
1569 struct ccp_data dst;
1570 struct ccp_op op;
1571 unsigned int ksb_count, i_len, o_len;
1572 int ret;
1573
1574 if (rsa->key_size > CCP_RSA_MAX_WIDTH)
1575 return -EINVAL;
1576
1577 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1578 return -EINVAL;
1579
1580 /* The RSA modulus must precede the message being acted upon, so
1581 * it must be copied to a DMA area where the message and the
1582 * modulus can be concatenated. Therefore the input buffer
1583 * length required is twice the output buffer length (which
1584 * must be a multiple of 256-bits).
1585 */
1586 o_len = ((rsa->key_size + 255) / 256) * 32;
1587 i_len = o_len * 2;
1588
1589 ksb_count = o_len / CCP_KSB_BYTES;
1590
1591 memset(&op, 0, sizeof(op));
1592 op.cmd_q = cmd_q;
1593 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1594 op.ksb_key = ccp_alloc_ksb(cmd_q->ccp, ksb_count);
1595 if (!op.ksb_key)
1596 return -EIO;
1597
1598 /* The RSA exponent may span multiple (32-byte) KSB entries and must
1599 * be in little endian format. Reverse copy each 32-byte chunk
1600 * of the exponent (En chunk to E0 chunk, E(n-1) chunk to E1 chunk)
1601 * and each byte within that chunk and do not perform any byte swap
1602 * operations on the passthru operation.
1603 */
1604 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1605 if (ret)
1606 goto e_ksb;
1607
1608 ccp_reverse_set_dm_area(&exp, rsa->exp, rsa->exp_len, CCP_KSB_BYTES,
1609 true);
1610 ret = ccp_copy_to_ksb(cmd_q, &exp, op.jobid, op.ksb_key,
1611 CCP_PASSTHRU_BYTESWAP_NOOP);
1612 if (ret) {
1613 cmd->engine_error = cmd_q->cmd_error;
1614 goto e_exp;
1615 }
1616
1617 /* Concatenate the modulus and the message. Both the modulus and
1618 * the operands must be in little endian format. Since the input
1619 * is in big endian format it must be converted.
1620 */
1621 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1622 if (ret)
1623 goto e_exp;
1624
1625 ccp_reverse_set_dm_area(&src, rsa->mod, rsa->mod_len, CCP_KSB_BYTES,
1626 true);
1627 src.address += o_len; /* Adjust the address for the copy operation */
1628 ccp_reverse_set_dm_area(&src, rsa->src, rsa->src_len, CCP_KSB_BYTES,
1629 true);
1630 src.address -= o_len; /* Reset the address to original value */
1631
1632 /* Prepare the output area for the operation */
1633 ret = ccp_init_data(&dst, cmd_q, rsa->dst, rsa->mod_len,
1634 o_len, DMA_FROM_DEVICE);
1635 if (ret)
1636 goto e_src;
1637
1638 op.soc = 1;
1639 op.src.u.dma.address = src.dma.address;
1640 op.src.u.dma.offset = 0;
1641 op.src.u.dma.length = i_len;
1642 op.dst.u.dma.address = dst.dm_wa.dma.address;
1643 op.dst.u.dma.offset = 0;
1644 op.dst.u.dma.length = o_len;
1645
1646 op.u.rsa.mod_size = rsa->key_size;
1647 op.u.rsa.input_len = i_len;
1648
1649 ret = ccp_perform_rsa(&op);
1650 if (ret) {
1651 cmd->engine_error = cmd_q->cmd_error;
1652 goto e_dst;
1653 }
1654
1655 ccp_reverse_get_dm_area(&dst.dm_wa, rsa->dst, rsa->mod_len);
1656
1657 e_dst:
1658 ccp_free_data(&dst, cmd_q);
1659
1660 e_src:
1661 ccp_dm_free(&src);
1662
1663 e_exp:
1664 ccp_dm_free(&exp);
1665
1666 e_ksb:
1667 ccp_free_ksb(cmd_q->ccp, op.ksb_key, ksb_count);
1668
1669 return ret;
1670 }
1671
1672 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1673 struct ccp_cmd *cmd)
1674 {
1675 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1676 struct ccp_dm_workarea mask;
1677 struct ccp_data src, dst;
1678 struct ccp_op op;
1679 bool in_place = false;
1680 unsigned int i;
1681 int ret;
1682
1683 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1684 return -EINVAL;
1685
1686 if (!pt->src || !pt->dst)
1687 return -EINVAL;
1688
1689 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1690 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1691 return -EINVAL;
1692 if (!pt->mask)
1693 return -EINVAL;
1694 }
1695
1696 BUILD_BUG_ON(CCP_PASSTHRU_KSB_COUNT != 1);
1697
1698 memset(&op, 0, sizeof(op));
1699 op.cmd_q = cmd_q;
1700 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1701
1702 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1703 /* Load the mask */
1704 op.ksb_key = cmd_q->ksb_key;
1705
1706 ret = ccp_init_dm_workarea(&mask, cmd_q,
1707 CCP_PASSTHRU_KSB_COUNT *
1708 CCP_KSB_BYTES,
1709 DMA_TO_DEVICE);
1710 if (ret)
1711 return ret;
1712
1713 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1714 ret = ccp_copy_to_ksb(cmd_q, &mask, op.jobid, op.ksb_key,
1715 CCP_PASSTHRU_BYTESWAP_NOOP);
1716 if (ret) {
1717 cmd->engine_error = cmd_q->cmd_error;
1718 goto e_mask;
1719 }
1720 }
1721
1722 /* Prepare the input and output data workareas. For in-place
1723 * operations we need to set the dma direction to BIDIRECTIONAL
1724 * and copy the src workarea to the dst workarea.
1725 */
1726 if (sg_virt(pt->src) == sg_virt(pt->dst))
1727 in_place = true;
1728
1729 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1730 CCP_PASSTHRU_MASKSIZE,
1731 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1732 if (ret)
1733 goto e_mask;
1734
1735 if (in_place)
1736 dst = src;
1737 else {
1738 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1739 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1740 if (ret)
1741 goto e_src;
1742 }
1743
1744 /* Send data to the CCP Passthru engine
1745 * Because the CCP engine works on a single source and destination
1746 * dma address at a time, each entry in the source scatterlist
1747 * (after the dma_map_sg call) must be less than or equal to the
1748 * (remaining) length in the destination scatterlist entry and the
1749 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1750 */
1751 dst.sg_wa.sg_used = 0;
1752 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1753 if (!dst.sg_wa.sg ||
1754 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1755 ret = -EINVAL;
1756 goto e_dst;
1757 }
1758
1759 if (i == src.sg_wa.dma_count) {
1760 op.eom = 1;
1761 op.soc = 1;
1762 }
1763
1764 op.src.type = CCP_MEMTYPE_SYSTEM;
1765 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1766 op.src.u.dma.offset = 0;
1767 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1768
1769 op.dst.type = CCP_MEMTYPE_SYSTEM;
1770 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1771 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1772 op.dst.u.dma.length = op.src.u.dma.length;
1773
1774 ret = ccp_perform_passthru(&op);
1775 if (ret) {
1776 cmd->engine_error = cmd_q->cmd_error;
1777 goto e_dst;
1778 }
1779
1780 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1781 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1782 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
1783 dst.sg_wa.sg_used = 0;
1784 }
1785 src.sg_wa.sg = sg_next(src.sg_wa.sg);
1786 }
1787
1788 e_dst:
1789 if (!in_place)
1790 ccp_free_data(&dst, cmd_q);
1791
1792 e_src:
1793 ccp_free_data(&src, cmd_q);
1794
1795 e_mask:
1796 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
1797 ccp_dm_free(&mask);
1798
1799 return ret;
1800 }
1801
1802 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1803 {
1804 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1805 struct ccp_dm_workarea src, dst;
1806 struct ccp_op op;
1807 int ret;
1808 u8 *save;
1809
1810 if (!ecc->u.mm.operand_1 ||
1811 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
1812 return -EINVAL;
1813
1814 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
1815 if (!ecc->u.mm.operand_2 ||
1816 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
1817 return -EINVAL;
1818
1819 if (!ecc->u.mm.result ||
1820 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
1821 return -EINVAL;
1822
1823 memset(&op, 0, sizeof(op));
1824 op.cmd_q = cmd_q;
1825 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1826
1827 /* Concatenate the modulus and the operands. Both the modulus and
1828 * the operands must be in little endian format. Since the input
1829 * is in big endian format it must be converted and placed in a
1830 * fixed length buffer.
1831 */
1832 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1833 DMA_TO_DEVICE);
1834 if (ret)
1835 return ret;
1836
1837 /* Save the workarea address since it is updated in order to perform
1838 * the concatenation
1839 */
1840 save = src.address;
1841
1842 /* Copy the ECC modulus */
1843 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1844 CCP_ECC_OPERAND_SIZE, true);
1845 src.address += CCP_ECC_OPERAND_SIZE;
1846
1847 /* Copy the first operand */
1848 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_1,
1849 ecc->u.mm.operand_1_len,
1850 CCP_ECC_OPERAND_SIZE, true);
1851 src.address += CCP_ECC_OPERAND_SIZE;
1852
1853 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
1854 /* Copy the second operand */
1855 ccp_reverse_set_dm_area(&src, ecc->u.mm.operand_2,
1856 ecc->u.mm.operand_2_len,
1857 CCP_ECC_OPERAND_SIZE, true);
1858 src.address += CCP_ECC_OPERAND_SIZE;
1859 }
1860
1861 /* Restore the workarea address */
1862 src.address = save;
1863
1864 /* Prepare the output area for the operation */
1865 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
1866 DMA_FROM_DEVICE);
1867 if (ret)
1868 goto e_src;
1869
1870 op.soc = 1;
1871 op.src.u.dma.address = src.dma.address;
1872 op.src.u.dma.offset = 0;
1873 op.src.u.dma.length = src.length;
1874 op.dst.u.dma.address = dst.dma.address;
1875 op.dst.u.dma.offset = 0;
1876 op.dst.u.dma.length = dst.length;
1877
1878 op.u.ecc.function = cmd->u.ecc.function;
1879
1880 ret = ccp_perform_ecc(&op);
1881 if (ret) {
1882 cmd->engine_error = cmd_q->cmd_error;
1883 goto e_dst;
1884 }
1885
1886 ecc->ecc_result = le16_to_cpup(
1887 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
1888 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
1889 ret = -EIO;
1890 goto e_dst;
1891 }
1892
1893 /* Save the ECC result */
1894 ccp_reverse_get_dm_area(&dst, ecc->u.mm.result, CCP_ECC_MODULUS_BYTES);
1895
1896 e_dst:
1897 ccp_dm_free(&dst);
1898
1899 e_src:
1900 ccp_dm_free(&src);
1901
1902 return ret;
1903 }
1904
1905 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1906 {
1907 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
1908 struct ccp_dm_workarea src, dst;
1909 struct ccp_op op;
1910 int ret;
1911 u8 *save;
1912
1913 if (!ecc->u.pm.point_1.x ||
1914 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
1915 !ecc->u.pm.point_1.y ||
1916 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
1917 return -EINVAL;
1918
1919 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1920 if (!ecc->u.pm.point_2.x ||
1921 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
1922 !ecc->u.pm.point_2.y ||
1923 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
1924 return -EINVAL;
1925 } else {
1926 if (!ecc->u.pm.domain_a ||
1927 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
1928 return -EINVAL;
1929
1930 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
1931 if (!ecc->u.pm.scalar ||
1932 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
1933 return -EINVAL;
1934 }
1935
1936 if (!ecc->u.pm.result.x ||
1937 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
1938 !ecc->u.pm.result.y ||
1939 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
1940 return -EINVAL;
1941
1942 memset(&op, 0, sizeof(op));
1943 op.cmd_q = cmd_q;
1944 op.jobid = ccp_gen_jobid(cmd_q->ccp);
1945
1946 /* Concatenate the modulus and the operands. Both the modulus and
1947 * the operands must be in little endian format. Since the input
1948 * is in big endian format it must be converted and placed in a
1949 * fixed length buffer.
1950 */
1951 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
1952 DMA_TO_DEVICE);
1953 if (ret)
1954 return ret;
1955
1956 /* Save the workarea address since it is updated in order to perform
1957 * the concatenation
1958 */
1959 save = src.address;
1960
1961 /* Copy the ECC modulus */
1962 ccp_reverse_set_dm_area(&src, ecc->mod, ecc->mod_len,
1963 CCP_ECC_OPERAND_SIZE, true);
1964 src.address += CCP_ECC_OPERAND_SIZE;
1965
1966 /* Copy the first point X and Y coordinate */
1967 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.x,
1968 ecc->u.pm.point_1.x_len,
1969 CCP_ECC_OPERAND_SIZE, true);
1970 src.address += CCP_ECC_OPERAND_SIZE;
1971 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_1.y,
1972 ecc->u.pm.point_1.y_len,
1973 CCP_ECC_OPERAND_SIZE, true);
1974 src.address += CCP_ECC_OPERAND_SIZE;
1975
1976 /* Set the first point Z coordianate to 1 */
1977 *(src.address) = 0x01;
1978 src.address += CCP_ECC_OPERAND_SIZE;
1979
1980 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
1981 /* Copy the second point X and Y coordinate */
1982 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.x,
1983 ecc->u.pm.point_2.x_len,
1984 CCP_ECC_OPERAND_SIZE, true);
1985 src.address += CCP_ECC_OPERAND_SIZE;
1986 ccp_reverse_set_dm_area(&src, ecc->u.pm.point_2.y,
1987 ecc->u.pm.point_2.y_len,
1988 CCP_ECC_OPERAND_SIZE, true);
1989 src.address += CCP_ECC_OPERAND_SIZE;
1990
1991 /* Set the second point Z coordianate to 1 */
1992 *(src.address) = 0x01;
1993 src.address += CCP_ECC_OPERAND_SIZE;
1994 } else {
1995 /* Copy the Domain "a" parameter */
1996 ccp_reverse_set_dm_area(&src, ecc->u.pm.domain_a,
1997 ecc->u.pm.domain_a_len,
1998 CCP_ECC_OPERAND_SIZE, true);
1999 src.address += CCP_ECC_OPERAND_SIZE;
2000
2001 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2002 /* Copy the scalar value */
2003 ccp_reverse_set_dm_area(&src, ecc->u.pm.scalar,
2004 ecc->u.pm.scalar_len,
2005 CCP_ECC_OPERAND_SIZE, true);
2006 src.address += CCP_ECC_OPERAND_SIZE;
2007 }
2008 }
2009
2010 /* Restore the workarea address */
2011 src.address = save;
2012
2013 /* Prepare the output area for the operation */
2014 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2015 DMA_FROM_DEVICE);
2016 if (ret)
2017 goto e_src;
2018
2019 op.soc = 1;
2020 op.src.u.dma.address = src.dma.address;
2021 op.src.u.dma.offset = 0;
2022 op.src.u.dma.length = src.length;
2023 op.dst.u.dma.address = dst.dma.address;
2024 op.dst.u.dma.offset = 0;
2025 op.dst.u.dma.length = dst.length;
2026
2027 op.u.ecc.function = cmd->u.ecc.function;
2028
2029 ret = ccp_perform_ecc(&op);
2030 if (ret) {
2031 cmd->engine_error = cmd_q->cmd_error;
2032 goto e_dst;
2033 }
2034
2035 ecc->ecc_result = le16_to_cpup(
2036 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2037 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2038 ret = -EIO;
2039 goto e_dst;
2040 }
2041
2042 /* Save the workarea address since it is updated as we walk through
2043 * to copy the point math result
2044 */
2045 save = dst.address;
2046
2047 /* Save the ECC result X and Y coordinates */
2048 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.x,
2049 CCP_ECC_MODULUS_BYTES);
2050 dst.address += CCP_ECC_OUTPUT_SIZE;
2051 ccp_reverse_get_dm_area(&dst, ecc->u.pm.result.y,
2052 CCP_ECC_MODULUS_BYTES);
2053 dst.address += CCP_ECC_OUTPUT_SIZE;
2054
2055 /* Restore the workarea address */
2056 dst.address = save;
2057
2058 e_dst:
2059 ccp_dm_free(&dst);
2060
2061 e_src:
2062 ccp_dm_free(&src);
2063
2064 return ret;
2065 }
2066
2067 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2068 {
2069 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2070
2071 ecc->ecc_result = 0;
2072
2073 if (!ecc->mod ||
2074 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2075 return -EINVAL;
2076
2077 switch (ecc->function) {
2078 case CCP_ECC_FUNCTION_MMUL_384BIT:
2079 case CCP_ECC_FUNCTION_MADD_384BIT:
2080 case CCP_ECC_FUNCTION_MINV_384BIT:
2081 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2082
2083 case CCP_ECC_FUNCTION_PADD_384BIT:
2084 case CCP_ECC_FUNCTION_PMUL_384BIT:
2085 case CCP_ECC_FUNCTION_PDBL_384BIT:
2086 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2087
2088 default:
2089 return -EINVAL;
2090 }
2091 }
2092
2093 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2094 {
2095 int ret;
2096
2097 cmd->engine_error = 0;
2098 cmd_q->cmd_error = 0;
2099 cmd_q->int_rcvd = 0;
2100 cmd_q->free_slots = CMD_Q_DEPTH(ioread32(cmd_q->reg_status));
2101
2102 switch (cmd->engine) {
2103 case CCP_ENGINE_AES:
2104 ret = ccp_run_aes_cmd(cmd_q, cmd);
2105 break;
2106 case CCP_ENGINE_XTS_AES_128:
2107 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2108 break;
2109 case CCP_ENGINE_SHA:
2110 ret = ccp_run_sha_cmd(cmd_q, cmd);
2111 break;
2112 case CCP_ENGINE_RSA:
2113 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2114 break;
2115 case CCP_ENGINE_PASSTHRU:
2116 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2117 break;
2118 case CCP_ENGINE_ECC:
2119 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2120 break;
2121 default:
2122 ret = -EINVAL;
2123 }
2124
2125 return ret;
2126 }
Linux with added FPC-III support