?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
x86/topology: Fix function name in documentation
[cris-mirror.git] / drivers / crypto / ccp / ccp-ops.c
blob406b95329b3d27e45b94cc66d8d82a1d5bf6776b
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14 #include <linux/module.h>
15 #include <linux/kernel.h>
16 #include <linux/pci.h>
17 #include <linux/interrupt.h>
18 #include <crypto/scatterwalk.h>
19 #include <crypto/des.h>
20 #include <linux/ccp.h>
21
22 #include "ccp-dev.h"
23
24 /* SHA initial context values */
25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
28 cpu_to_be32(SHA1_H4),
29 };
30
31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
36 };
37
38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
43 };
44
45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
50 };
51
52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
57 };
58
59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
60 ccp_gen_jobid(ccp) : 0)
61
62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
63 {
64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
65 }
66
67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
68 {
69 if (wa->dma_count)
70 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
71
72 wa->dma_count = 0;
73 }
74
75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
76 struct scatterlist *sg, u64 len,
77 enum dma_data_direction dma_dir)
78 {
79 memset(wa, 0, sizeof(*wa));
80
81 wa->sg = sg;
82 if (!sg)
83 return 0;
84
85 wa->nents = sg_nents_for_len(sg, len);
86 if (wa->nents < 0)
87 return wa->nents;
88
89 wa->bytes_left = len;
90 wa->sg_used = 0;
91
92 if (len == 0)
93 return 0;
94
95 if (dma_dir == DMA_NONE)
96 return 0;
97
98 wa->dma_sg = sg;
99 wa->dma_dev = dev;
100 wa->dma_dir = dma_dir;
101 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
102 if (!wa->dma_count)
103 return -ENOMEM;
104
105 return 0;
106 }
107
108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
109 {
110 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
111
112 if (!wa->sg)
113 return;
114
115 wa->sg_used += nbytes;
116 wa->bytes_left -= nbytes;
117 if (wa->sg_used == wa->sg->length) {
118 wa->sg = sg_next(wa->sg);
119 wa->sg_used = 0;
120 }
121 }
122
123 static void ccp_dm_free(struct ccp_dm_workarea *wa)
124 {
125 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
126 if (wa->address)
127 dma_pool_free(wa->dma_pool, wa->address,
128 wa->dma.address);
129 } else {
130 if (wa->dma.address)
131 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
132 wa->dma.dir);
133 kfree(wa->address);
134 }
135
136 wa->address = NULL;
137 wa->dma.address = 0;
138 }
139
140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
141 struct ccp_cmd_queue *cmd_q,
142 unsigned int len,
143 enum dma_data_direction dir)
144 {
145 memset(wa, 0, sizeof(*wa));
146
147 if (!len)
148 return 0;
149
150 wa->dev = cmd_q->ccp->dev;
151 wa->length = len;
152
153 if (len <= CCP_DMAPOOL_MAX_SIZE) {
154 wa->dma_pool = cmd_q->dma_pool;
155
156 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
157 &wa->dma.address);
158 if (!wa->address)
159 return -ENOMEM;
160
161 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
162
163 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
164 } else {
165 wa->address = kzalloc(len, GFP_KERNEL);
166 if (!wa->address)
167 return -ENOMEM;
168
169 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
170 dir);
171 if (dma_mapping_error(wa->dev, wa->dma.address))
172 return -ENOMEM;
173
174 wa->dma.length = len;
175 }
176 wa->dma.dir = dir;
177
178 return 0;
179 }
180
181 static void ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
182 struct scatterlist *sg, unsigned int sg_offset,
183 unsigned int len)
184 {
185 WARN_ON(!wa->address);
186
187 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
188 0);
189 }
190
191 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
192 struct scatterlist *sg, unsigned int sg_offset,
193 unsigned int len)
194 {
195 WARN_ON(!wa->address);
196
197 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
198 1);
199 }
200
201 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
202 unsigned int wa_offset,
203 struct scatterlist *sg,
204 unsigned int sg_offset,
205 unsigned int len)
206 {
207 u8 *p, *q;
208
209 ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
210
211 p = wa->address + wa_offset;
212 q = p + len - 1;
213 while (p < q) {
214 *p = *p ^ *q;
215 *q = *p ^ *q;
216 *p = *p ^ *q;
217 p++;
218 q--;
219 }
220 return 0;
221 }
222
223 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
224 unsigned int wa_offset,
225 struct scatterlist *sg,
226 unsigned int sg_offset,
227 unsigned int len)
228 {
229 u8 *p, *q;
230
231 p = wa->address + wa_offset;
232 q = p + len - 1;
233 while (p < q) {
234 *p = *p ^ *q;
235 *q = *p ^ *q;
236 *p = *p ^ *q;
237 p++;
238 q--;
239 }
240
241 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
242 }
243
244 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
245 {
246 ccp_dm_free(&data->dm_wa);
247 ccp_sg_free(&data->sg_wa);
248 }
249
250 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
251 struct scatterlist *sg, u64 sg_len,
252 unsigned int dm_len,
253 enum dma_data_direction dir)
254 {
255 int ret;
256
257 memset(data, 0, sizeof(*data));
258
259 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
260 dir);
261 if (ret)
262 goto e_err;
263
264 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
265 if (ret)
266 goto e_err;
267
268 return 0;
269
270 e_err:
271 ccp_free_data(data, cmd_q);
272
273 return ret;
274 }
275
276 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
277 {
278 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
279 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
280 unsigned int buf_count, nbytes;
281
282 /* Clear the buffer if setting it */
283 if (!from)
284 memset(dm_wa->address, 0, dm_wa->length);
285
286 if (!sg_wa->sg)
287 return 0;
288
289 /* Perform the copy operation
290 * nbytes will always be <= UINT_MAX because dm_wa->length is
291 * an unsigned int
292 */
293 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
294 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
295 nbytes, from);
296
297 /* Update the structures and generate the count */
298 buf_count = 0;
299 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
300 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
301 dm_wa->length - buf_count);
302 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
303
304 buf_count += nbytes;
305 ccp_update_sg_workarea(sg_wa, nbytes);
306 }
307
308 return buf_count;
309 }
310
311 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
312 {
313 return ccp_queue_buf(data, 0);
314 }
315
316 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
317 {
318 return ccp_queue_buf(data, 1);
319 }
320
321 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
322 struct ccp_op *op, unsigned int block_size,
323 bool blocksize_op)
324 {
325 unsigned int sg_src_len, sg_dst_len, op_len;
326
327 /* The CCP can only DMA from/to one address each per operation. This
328 * requires that we find the smallest DMA area between the source
329 * and destination. The resulting len values will always be <= UINT_MAX
330 * because the dma length is an unsigned int.
331 */
332 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
333 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
334
335 if (dst) {
336 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
337 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
338 op_len = min(sg_src_len, sg_dst_len);
339 } else {
340 op_len = sg_src_len;
341 }
342
343 /* The data operation length will be at least block_size in length
344 * or the smaller of available sg room remaining for the source or
345 * the destination
346 */
347 op_len = max(op_len, block_size);
348
349 /* Unless we have to buffer data, there's no reason to wait */
350 op->soc = 0;
351
352 if (sg_src_len < block_size) {
353 /* Not enough data in the sg element, so it
354 * needs to be buffered into a blocksize chunk
355 */
356 int cp_len = ccp_fill_queue_buf(src);
357
358 op->soc = 1;
359 op->src.u.dma.address = src->dm_wa.dma.address;
360 op->src.u.dma.offset = 0;
361 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
362 } else {
363 /* Enough data in the sg element, but we need to
364 * adjust for any previously copied data
365 */
366 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
367 op->src.u.dma.offset = src->sg_wa.sg_used;
368 op->src.u.dma.length = op_len & ~(block_size - 1);
369
370 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
371 }
372
373 if (dst) {
374 if (sg_dst_len < block_size) {
375 /* Not enough room in the sg element or we're on the
376 * last piece of data (when using padding), so the
377 * output needs to be buffered into a blocksize chunk
378 */
379 op->soc = 1;
380 op->dst.u.dma.address = dst->dm_wa.dma.address;
381 op->dst.u.dma.offset = 0;
382 op->dst.u.dma.length = op->src.u.dma.length;
383 } else {
384 /* Enough room in the sg element, but we need to
385 * adjust for any previously used area
386 */
387 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
388 op->dst.u.dma.offset = dst->sg_wa.sg_used;
389 op->dst.u.dma.length = op->src.u.dma.length;
390 }
391 }
392 }
393
394 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
395 struct ccp_op *op)
396 {
397 op->init = 0;
398
399 if (dst) {
400 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
401 ccp_empty_queue_buf(dst);
402 else
403 ccp_update_sg_workarea(&dst->sg_wa,
404 op->dst.u.dma.length);
405 }
406 }
407
408 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
409 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
410 u32 byte_swap, bool from)
411 {
412 struct ccp_op op;
413
414 memset(&op, 0, sizeof(op));
415
416 op.cmd_q = cmd_q;
417 op.jobid = jobid;
418 op.eom = 1;
419
420 if (from) {
421 op.soc = 1;
422 op.src.type = CCP_MEMTYPE_SB;
423 op.src.u.sb = sb;
424 op.dst.type = CCP_MEMTYPE_SYSTEM;
425 op.dst.u.dma.address = wa->dma.address;
426 op.dst.u.dma.length = wa->length;
427 } else {
428 op.src.type = CCP_MEMTYPE_SYSTEM;
429 op.src.u.dma.address = wa->dma.address;
430 op.src.u.dma.length = wa->length;
431 op.dst.type = CCP_MEMTYPE_SB;
432 op.dst.u.sb = sb;
433 }
434
435 op.u.passthru.byte_swap = byte_swap;
436
437 return cmd_q->ccp->vdata->perform->passthru(&op);
438 }
439
440 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
441 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
442 u32 byte_swap)
443 {
444 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
445 }
446
447 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
448 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
449 u32 byte_swap)
450 {
451 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
452 }
453
454 static int ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q,
455 struct ccp_cmd *cmd)
456 {
457 struct ccp_aes_engine *aes = &cmd->u.aes;
458 struct ccp_dm_workarea key, ctx;
459 struct ccp_data src;
460 struct ccp_op op;
461 unsigned int dm_offset;
462 int ret;
463
464 if (!((aes->key_len == AES_KEYSIZE_128) ||
465 (aes->key_len == AES_KEYSIZE_192) ||
466 (aes->key_len == AES_KEYSIZE_256)))
467 return -EINVAL;
468
469 if (aes->src_len & (AES_BLOCK_SIZE - 1))
470 return -EINVAL;
471
472 if (aes->iv_len != AES_BLOCK_SIZE)
473 return -EINVAL;
474
475 if (!aes->key || !aes->iv || !aes->src)
476 return -EINVAL;
477
478 if (aes->cmac_final) {
479 if (aes->cmac_key_len != AES_BLOCK_SIZE)
480 return -EINVAL;
481
482 if (!aes->cmac_key)
483 return -EINVAL;
484 }
485
486 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
487 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
488
489 ret = -EIO;
490 memset(&op, 0, sizeof(op));
491 op.cmd_q = cmd_q;
492 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
493 op.sb_key = cmd_q->sb_key;
494 op.sb_ctx = cmd_q->sb_ctx;
495 op.init = 1;
496 op.u.aes.type = aes->type;
497 op.u.aes.mode = aes->mode;
498 op.u.aes.action = aes->action;
499
500 /* All supported key sizes fit in a single (32-byte) SB entry
501 * and must be in little endian format. Use the 256-bit byte
502 * swap passthru option to convert from big endian to little
503 * endian.
504 */
505 ret = ccp_init_dm_workarea(&key, cmd_q,
506 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
507 DMA_TO_DEVICE);
508 if (ret)
509 return ret;
510
511 dm_offset = CCP_SB_BYTES - aes->key_len;
512 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
513 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
514 CCP_PASSTHRU_BYTESWAP_256BIT);
515 if (ret) {
516 cmd->engine_error = cmd_q->cmd_error;
517 goto e_key;
518 }
519
520 /* The AES context fits in a single (32-byte) SB entry and
521 * must be in little endian format. Use the 256-bit byte swap
522 * passthru option to convert from big endian to little endian.
523 */
524 ret = ccp_init_dm_workarea(&ctx, cmd_q,
525 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
526 DMA_BIDIRECTIONAL);
527 if (ret)
528 goto e_key;
529
530 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
531 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
532 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
533 CCP_PASSTHRU_BYTESWAP_256BIT);
534 if (ret) {
535 cmd->engine_error = cmd_q->cmd_error;
536 goto e_ctx;
537 }
538
539 /* Send data to the CCP AES engine */
540 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
541 AES_BLOCK_SIZE, DMA_TO_DEVICE);
542 if (ret)
543 goto e_ctx;
544
545 while (src.sg_wa.bytes_left) {
546 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
547 if (aes->cmac_final && !src.sg_wa.bytes_left) {
548 op.eom = 1;
549
550 /* Push the K1/K2 key to the CCP now */
551 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
552 op.sb_ctx,
553 CCP_PASSTHRU_BYTESWAP_256BIT);
554 if (ret) {
555 cmd->engine_error = cmd_q->cmd_error;
556 goto e_src;
557 }
558
559 ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
560 aes->cmac_key_len);
561 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
562 CCP_PASSTHRU_BYTESWAP_256BIT);
563 if (ret) {
564 cmd->engine_error = cmd_q->cmd_error;
565 goto e_src;
566 }
567 }
568
569 ret = cmd_q->ccp->vdata->perform->aes(&op);
570 if (ret) {
571 cmd->engine_error = cmd_q->cmd_error;
572 goto e_src;
573 }
574
575 ccp_process_data(&src, NULL, &op);
576 }
577
578 /* Retrieve the AES context - convert from LE to BE using
579 * 32-byte (256-bit) byteswapping
580 */
581 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
582 CCP_PASSTHRU_BYTESWAP_256BIT);
583 if (ret) {
584 cmd->engine_error = cmd_q->cmd_error;
585 goto e_src;
586 }
587
588 /* ...but we only need AES_BLOCK_SIZE bytes */
589 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
590 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
591
592 e_src:
593 ccp_free_data(&src, cmd_q);
594
595 e_ctx:
596 ccp_dm_free(&ctx);
597
598 e_key:
599 ccp_dm_free(&key);
600
601 return ret;
602 }
603
604 static int ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q,
605 struct ccp_cmd *cmd)
606 {
607 struct ccp_aes_engine *aes = &cmd->u.aes;
608 struct ccp_dm_workarea key, ctx, final_wa, tag;
609 struct ccp_data src, dst;
610 struct ccp_data aad;
611 struct ccp_op op;
612
613 unsigned long long *final;
614 unsigned int dm_offset;
615 unsigned int ilen;
616 bool in_place = true; /* Default value */
617 int ret;
618
619 struct scatterlist *p_inp, sg_inp[2];
620 struct scatterlist *p_tag, sg_tag[2];
621 struct scatterlist *p_outp, sg_outp[2];
622 struct scatterlist *p_aad;
623
624 if (!aes->iv)
625 return -EINVAL;
626
627 if (!((aes->key_len == AES_KEYSIZE_128) ||
628 (aes->key_len == AES_KEYSIZE_192) ||
629 (aes->key_len == AES_KEYSIZE_256)))
630 return -EINVAL;
631
632 if (!aes->key) /* Gotta have a key SGL */
633 return -EINVAL;
634
635 /* First, decompose the source buffer into AAD & PT,
636 * and the destination buffer into AAD, CT & tag, or
637 * the input into CT & tag.
638 * It is expected that the input and output SGs will
639 * be valid, even if the AAD and input lengths are 0.
640 */
641 p_aad = aes->src;
642 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
643 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
644 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
645 ilen = aes->src_len;
646 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
647 } else {
648 /* Input length for decryption includes tag */
649 ilen = aes->src_len - AES_BLOCK_SIZE;
650 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
651 }
652
653 memset(&op, 0, sizeof(op));
654 op.cmd_q = cmd_q;
655 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
656 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
657 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
658 op.init = 1;
659 op.u.aes.type = aes->type;
660
661 /* Copy the key to the LSB */
662 ret = ccp_init_dm_workarea(&key, cmd_q,
663 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
664 DMA_TO_DEVICE);
665 if (ret)
666 return ret;
667
668 dm_offset = CCP_SB_BYTES - aes->key_len;
669 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
670 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
671 CCP_PASSTHRU_BYTESWAP_256BIT);
672 if (ret) {
673 cmd->engine_error = cmd_q->cmd_error;
674 goto e_key;
675 }
676
677 /* Copy the context (IV) to the LSB.
678 * There is an assumption here that the IV is 96 bits in length, plus
679 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
680 */
681 ret = ccp_init_dm_workarea(&ctx, cmd_q,
682 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
683 DMA_BIDIRECTIONAL);
684 if (ret)
685 goto e_key;
686
687 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
688 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
689
690 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
691 CCP_PASSTHRU_BYTESWAP_256BIT);
692 if (ret) {
693 cmd->engine_error = cmd_q->cmd_error;
694 goto e_ctx;
695 }
696
697 op.init = 1;
698 if (aes->aad_len > 0) {
699 /* Step 1: Run a GHASH over the Additional Authenticated Data */
700 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
701 AES_BLOCK_SIZE,
702 DMA_TO_DEVICE);
703 if (ret)
704 goto e_ctx;
705
706 op.u.aes.mode = CCP_AES_MODE_GHASH;
707 op.u.aes.action = CCP_AES_GHASHAAD;
708
709 while (aad.sg_wa.bytes_left) {
710 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
711
712 ret = cmd_q->ccp->vdata->perform->aes(&op);
713 if (ret) {
714 cmd->engine_error = cmd_q->cmd_error;
715 goto e_aad;
716 }
717
718 ccp_process_data(&aad, NULL, &op);
719 op.init = 0;
720 }
721 }
722
723 op.u.aes.mode = CCP_AES_MODE_GCTR;
724 op.u.aes.action = aes->action;
725
726 if (ilen > 0) {
727 /* Step 2: Run a GCTR over the plaintext */
728 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
729
730 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
731 AES_BLOCK_SIZE,
732 in_place ? DMA_BIDIRECTIONAL
733 : DMA_TO_DEVICE);
734 if (ret)
735 goto e_ctx;
736
737 if (in_place) {
738 dst = src;
739 } else {
740 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
741 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
742 if (ret)
743 goto e_src;
744 }
745
746 op.soc = 0;
747 op.eom = 0;
748 op.init = 1;
749 while (src.sg_wa.bytes_left) {
750 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
751 if (!src.sg_wa.bytes_left) {
752 unsigned int nbytes = aes->src_len
753 % AES_BLOCK_SIZE;
754
755 if (nbytes) {
756 op.eom = 1;
757 op.u.aes.size = (nbytes * 8) - 1;
758 }
759 }
760
761 ret = cmd_q->ccp->vdata->perform->aes(&op);
762 if (ret) {
763 cmd->engine_error = cmd_q->cmd_error;
764 goto e_dst;
765 }
766
767 ccp_process_data(&src, &dst, &op);
768 op.init = 0;
769 }
770 }
771
772 /* Step 3: Update the IV portion of the context with the original IV */
773 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
774 CCP_PASSTHRU_BYTESWAP_256BIT);
775 if (ret) {
776 cmd->engine_error = cmd_q->cmd_error;
777 goto e_dst;
778 }
779
780 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
781
782 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
783 CCP_PASSTHRU_BYTESWAP_256BIT);
784 if (ret) {
785 cmd->engine_error = cmd_q->cmd_error;
786 goto e_dst;
787 }
788
789 /* Step 4: Concatenate the lengths of the AAD and source, and
790 * hash that 16 byte buffer.
791 */
792 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
793 DMA_BIDIRECTIONAL);
794 if (ret)
795 goto e_dst;
796 final = (unsigned long long *) final_wa.address;
797 final[0] = cpu_to_be64(aes->aad_len * 8);
798 final[1] = cpu_to_be64(ilen * 8);
799
800 op.u.aes.mode = CCP_AES_MODE_GHASH;
801 op.u.aes.action = CCP_AES_GHASHFINAL;
802 op.src.type = CCP_MEMTYPE_SYSTEM;
803 op.src.u.dma.address = final_wa.dma.address;
804 op.src.u.dma.length = AES_BLOCK_SIZE;
805 op.dst.type = CCP_MEMTYPE_SYSTEM;
806 op.dst.u.dma.address = final_wa.dma.address;
807 op.dst.u.dma.length = AES_BLOCK_SIZE;
808 op.eom = 1;
809 op.u.aes.size = 0;
810 ret = cmd_q->ccp->vdata->perform->aes(&op);
811 if (ret)
812 goto e_dst;
813
814 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
815 /* Put the ciphered tag after the ciphertext. */
816 ccp_get_dm_area(&final_wa, 0, p_tag, 0, AES_BLOCK_SIZE);
817 } else {
818 /* Does this ciphered tag match the input? */
819 ret = ccp_init_dm_workarea(&tag, cmd_q, AES_BLOCK_SIZE,
820 DMA_BIDIRECTIONAL);
821 if (ret)
822 goto e_tag;
823 ccp_set_dm_area(&tag, 0, p_tag, 0, AES_BLOCK_SIZE);
824
825 ret = memcmp(tag.address, final_wa.address, AES_BLOCK_SIZE);
826 ccp_dm_free(&tag);
827 }
828
829 e_tag:
830 ccp_dm_free(&final_wa);
831
832 e_dst:
833 if (aes->src_len && !in_place)
834 ccp_free_data(&dst, cmd_q);
835
836 e_src:
837 if (aes->src_len)
838 ccp_free_data(&src, cmd_q);
839
840 e_aad:
841 if (aes->aad_len)
842 ccp_free_data(&aad, cmd_q);
843
844 e_ctx:
845 ccp_dm_free(&ctx);
846
847 e_key:
848 ccp_dm_free(&key);
849
850 return ret;
851 }
852
853 static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
854 {
855 struct ccp_aes_engine *aes = &cmd->u.aes;
856 struct ccp_dm_workarea key, ctx;
857 struct ccp_data src, dst;
858 struct ccp_op op;
859 unsigned int dm_offset;
860 bool in_place = false;
861 int ret;
862
863 if (aes->mode == CCP_AES_MODE_CMAC)
864 return ccp_run_aes_cmac_cmd(cmd_q, cmd);
865
866 if (aes->mode == CCP_AES_MODE_GCM)
867 return ccp_run_aes_gcm_cmd(cmd_q, cmd);
868
869 if (!((aes->key_len == AES_KEYSIZE_128) ||
870 (aes->key_len == AES_KEYSIZE_192) ||
871 (aes->key_len == AES_KEYSIZE_256)))
872 return -EINVAL;
873
874 if (((aes->mode == CCP_AES_MODE_ECB) ||
875 (aes->mode == CCP_AES_MODE_CBC) ||
876 (aes->mode == CCP_AES_MODE_CFB)) &&
877 (aes->src_len & (AES_BLOCK_SIZE - 1)))
878 return -EINVAL;
879
880 if (!aes->key || !aes->src || !aes->dst)
881 return -EINVAL;
882
883 if (aes->mode != CCP_AES_MODE_ECB) {
884 if (aes->iv_len != AES_BLOCK_SIZE)
885 return -EINVAL;
886
887 if (!aes->iv)
888 return -EINVAL;
889 }
890
891 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
892 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
893
894 ret = -EIO;
895 memset(&op, 0, sizeof(op));
896 op.cmd_q = cmd_q;
897 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
898 op.sb_key = cmd_q->sb_key;
899 op.sb_ctx = cmd_q->sb_ctx;
900 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
901 op.u.aes.type = aes->type;
902 op.u.aes.mode = aes->mode;
903 op.u.aes.action = aes->action;
904
905 /* All supported key sizes fit in a single (32-byte) SB entry
906 * and must be in little endian format. Use the 256-bit byte
907 * swap passthru option to convert from big endian to little
908 * endian.
909 */
910 ret = ccp_init_dm_workarea(&key, cmd_q,
911 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
912 DMA_TO_DEVICE);
913 if (ret)
914 return ret;
915
916 dm_offset = CCP_SB_BYTES - aes->key_len;
917 ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
918 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
919 CCP_PASSTHRU_BYTESWAP_256BIT);
920 if (ret) {
921 cmd->engine_error = cmd_q->cmd_error;
922 goto e_key;
923 }
924
925 /* The AES context fits in a single (32-byte) SB entry and
926 * must be in little endian format. Use the 256-bit byte swap
927 * passthru option to convert from big endian to little endian.
928 */
929 ret = ccp_init_dm_workarea(&ctx, cmd_q,
930 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
931 DMA_BIDIRECTIONAL);
932 if (ret)
933 goto e_key;
934
935 if (aes->mode != CCP_AES_MODE_ECB) {
936 /* Load the AES context - convert to LE */
937 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
938 ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
939 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
940 CCP_PASSTHRU_BYTESWAP_256BIT);
941 if (ret) {
942 cmd->engine_error = cmd_q->cmd_error;
943 goto e_ctx;
944 }
945 }
946 switch (aes->mode) {
947 case CCP_AES_MODE_CFB: /* CFB128 only */
948 case CCP_AES_MODE_CTR:
949 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
950 break;
951 default:
952 op.u.aes.size = 0;
953 }
954
955 /* Prepare the input and output data workareas. For in-place
956 * operations we need to set the dma direction to BIDIRECTIONAL
957 * and copy the src workarea to the dst workarea.
958 */
959 if (sg_virt(aes->src) == sg_virt(aes->dst))
960 in_place = true;
961
962 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
963 AES_BLOCK_SIZE,
964 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
965 if (ret)
966 goto e_ctx;
967
968 if (in_place) {
969 dst = src;
970 } else {
971 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
972 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
973 if (ret)
974 goto e_src;
975 }
976
977 /* Send data to the CCP AES engine */
978 while (src.sg_wa.bytes_left) {
979 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
980 if (!src.sg_wa.bytes_left) {
981 op.eom = 1;
982
983 /* Since we don't retrieve the AES context in ECB
984 * mode we have to wait for the operation to complete
985 * on the last piece of data
986 */
987 if (aes->mode == CCP_AES_MODE_ECB)
988 op.soc = 1;
989 }
990
991 ret = cmd_q->ccp->vdata->perform->aes(&op);
992 if (ret) {
993 cmd->engine_error = cmd_q->cmd_error;
994 goto e_dst;
995 }
996
997 ccp_process_data(&src, &dst, &op);
998 }
999
1000 if (aes->mode != CCP_AES_MODE_ECB) {
1001 /* Retrieve the AES context - convert from LE to BE using
1002 * 32-byte (256-bit) byteswapping
1003 */
1004 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1005 CCP_PASSTHRU_BYTESWAP_256BIT);
1006 if (ret) {
1007 cmd->engine_error = cmd_q->cmd_error;
1008 goto e_dst;
1009 }
1010
1011 /* ...but we only need AES_BLOCK_SIZE bytes */
1012 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1013 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1014 }
1015
1016 e_dst:
1017 if (!in_place)
1018 ccp_free_data(&dst, cmd_q);
1019
1020 e_src:
1021 ccp_free_data(&src, cmd_q);
1022
1023 e_ctx:
1024 ccp_dm_free(&ctx);
1025
1026 e_key:
1027 ccp_dm_free(&key);
1028
1029 return ret;
1030 }
1031
1032 static int ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q,
1033 struct ccp_cmd *cmd)
1034 {
1035 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1036 struct ccp_dm_workarea key, ctx;
1037 struct ccp_data src, dst;
1038 struct ccp_op op;
1039 unsigned int unit_size, dm_offset;
1040 bool in_place = false;
1041 unsigned int sb_count;
1042 enum ccp_aes_type aestype;
1043 int ret;
1044
1045 switch (xts->unit_size) {
1046 case CCP_XTS_AES_UNIT_SIZE_16:
1047 unit_size = 16;
1048 break;
1049 case CCP_XTS_AES_UNIT_SIZE_512:
1050 unit_size = 512;
1051 break;
1052 case CCP_XTS_AES_UNIT_SIZE_1024:
1053 unit_size = 1024;
1054 break;
1055 case CCP_XTS_AES_UNIT_SIZE_2048:
1056 unit_size = 2048;
1057 break;
1058 case CCP_XTS_AES_UNIT_SIZE_4096:
1059 unit_size = 4096;
1060 break;
1061
1062 default:
1063 return -EINVAL;
1064 }
1065
1066 if (xts->key_len == AES_KEYSIZE_128)
1067 aestype = CCP_AES_TYPE_128;
1068 else if (xts->key_len == AES_KEYSIZE_256)
1069 aestype = CCP_AES_TYPE_256;
1070 else
1071 return -EINVAL;
1072
1073 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1074 return -EINVAL;
1075
1076 if (xts->iv_len != AES_BLOCK_SIZE)
1077 return -EINVAL;
1078
1079 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1080 return -EINVAL;
1081
1082 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1083 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1084
1085 ret = -EIO;
1086 memset(&op, 0, sizeof(op));
1087 op.cmd_q = cmd_q;
1088 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1089 op.sb_key = cmd_q->sb_key;
1090 op.sb_ctx = cmd_q->sb_ctx;
1091 op.init = 1;
1092 op.u.xts.type = aestype;
1093 op.u.xts.action = xts->action;
1094 op.u.xts.unit_size = xts->unit_size;
1095
1096 /* A version 3 device only supports 128-bit keys, which fits into a
1097 * single SB entry. A version 5 device uses a 512-bit vector, so two
1098 * SB entries.
1099 */
1100 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1101 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1102 else
1103 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1104 ret = ccp_init_dm_workarea(&key, cmd_q,
1105 sb_count * CCP_SB_BYTES,
1106 DMA_TO_DEVICE);
1107 if (ret)
1108 return ret;
1109
1110 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1111 /* All supported key sizes must be in little endian format.
1112 * Use the 256-bit byte swap passthru option to convert from
1113 * big endian to little endian.
1114 */
1115 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1116 ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1117 ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1118 } else {
1119 /* Version 5 CCPs use a 512-bit space for the key: each portion
1120 * occupies 256 bits, or one entire slot, and is zero-padded.
1121 */
1122 unsigned int pad;
1123
1124 dm_offset = CCP_SB_BYTES;
1125 pad = dm_offset - xts->key_len;
1126 ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1127 ccp_set_dm_area(&key, dm_offset + pad, xts->key, xts->key_len,
1128 xts->key_len);
1129 }
1130 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1131 CCP_PASSTHRU_BYTESWAP_256BIT);
1132 if (ret) {
1133 cmd->engine_error = cmd_q->cmd_error;
1134 goto e_key;
1135 }
1136
1137 /* The AES context fits in a single (32-byte) SB entry and
1138 * for XTS is already in little endian format so no byte swapping
1139 * is needed.
1140 */
1141 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1142 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1143 DMA_BIDIRECTIONAL);
1144 if (ret)
1145 goto e_key;
1146
1147 ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1148 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1149 CCP_PASSTHRU_BYTESWAP_NOOP);
1150 if (ret) {
1151 cmd->engine_error = cmd_q->cmd_error;
1152 goto e_ctx;
1153 }
1154
1155 /* Prepare the input and output data workareas. For in-place
1156 * operations we need to set the dma direction to BIDIRECTIONAL
1157 * and copy the src workarea to the dst workarea.
1158 */
1159 if (sg_virt(xts->src) == sg_virt(xts->dst))
1160 in_place = true;
1161
1162 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1163 unit_size,
1164 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1165 if (ret)
1166 goto e_ctx;
1167
1168 if (in_place) {
1169 dst = src;
1170 } else {
1171 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1172 unit_size, DMA_FROM_DEVICE);
1173 if (ret)
1174 goto e_src;
1175 }
1176
1177 /* Send data to the CCP AES engine */
1178 while (src.sg_wa.bytes_left) {
1179 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1180 if (!src.sg_wa.bytes_left)
1181 op.eom = 1;
1182
1183 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1184 if (ret) {
1185 cmd->engine_error = cmd_q->cmd_error;
1186 goto e_dst;
1187 }
1188
1189 ccp_process_data(&src, &dst, &op);
1190 }
1191
1192 /* Retrieve the AES context - convert from LE to BE using
1193 * 32-byte (256-bit) byteswapping
1194 */
1195 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1196 CCP_PASSTHRU_BYTESWAP_256BIT);
1197 if (ret) {
1198 cmd->engine_error = cmd_q->cmd_error;
1199 goto e_dst;
1200 }
1201
1202 /* ...but we only need AES_BLOCK_SIZE bytes */
1203 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1204 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1205
1206 e_dst:
1207 if (!in_place)
1208 ccp_free_data(&dst, cmd_q);
1209
1210 e_src:
1211 ccp_free_data(&src, cmd_q);
1212
1213 e_ctx:
1214 ccp_dm_free(&ctx);
1215
1216 e_key:
1217 ccp_dm_free(&key);
1218
1219 return ret;
1220 }
1221
1222 static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1223 {
1224 struct ccp_des3_engine *des3 = &cmd->u.des3;
1225
1226 struct ccp_dm_workarea key, ctx;
1227 struct ccp_data src, dst;
1228 struct ccp_op op;
1229 unsigned int dm_offset;
1230 unsigned int len_singlekey;
1231 bool in_place = false;
1232 int ret;
1233
1234 /* Error checks */
1235 if (!cmd_q->ccp->vdata->perform->des3)
1236 return -EINVAL;
1237
1238 if (des3->key_len != DES3_EDE_KEY_SIZE)
1239 return -EINVAL;
1240
1241 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1242 (des3->mode == CCP_DES3_MODE_CBC)) &&
1243 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1244 return -EINVAL;
1245
1246 if (!des3->key || !des3->src || !des3->dst)
1247 return -EINVAL;
1248
1249 if (des3->mode != CCP_DES3_MODE_ECB) {
1250 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1251 return -EINVAL;
1252
1253 if (!des3->iv)
1254 return -EINVAL;
1255 }
1256
1257 ret = -EIO;
1258 /* Zero out all the fields of the command desc */
1259 memset(&op, 0, sizeof(op));
1260
1261 /* Set up the Function field */
1262 op.cmd_q = cmd_q;
1263 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1264 op.sb_key = cmd_q->sb_key;
1265
1266 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1267 op.u.des3.type = des3->type;
1268 op.u.des3.mode = des3->mode;
1269 op.u.des3.action = des3->action;
1270
1271 /*
1272 * All supported key sizes fit in a single (32-byte) KSB entry and
1273 * (like AES) must be in little endian format. Use the 256-bit byte
1274 * swap passthru option to convert from big endian to little endian.
1275 */
1276 ret = ccp_init_dm_workarea(&key, cmd_q,
1277 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1278 DMA_TO_DEVICE);
1279 if (ret)
1280 return ret;
1281
1282 /*
1283 * The contents of the key triplet are in the reverse order of what
1284 * is required by the engine. Copy the 3 pieces individually to put
1285 * them where they belong.
1286 */
1287 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1288
1289 len_singlekey = des3->key_len / 3;
1290 ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1291 des3->key, 0, len_singlekey);
1292 ccp_set_dm_area(&key, dm_offset + len_singlekey,
1293 des3->key, len_singlekey, len_singlekey);
1294 ccp_set_dm_area(&key, dm_offset,
1295 des3->key, 2 * len_singlekey, len_singlekey);
1296
1297 /* Copy the key to the SB */
1298 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1299 CCP_PASSTHRU_BYTESWAP_256BIT);
1300 if (ret) {
1301 cmd->engine_error = cmd_q->cmd_error;
1302 goto e_key;
1303 }
1304
1305 /*
1306 * The DES3 context fits in a single (32-byte) KSB entry and
1307 * must be in little endian format. Use the 256-bit byte swap
1308 * passthru option to convert from big endian to little endian.
1309 */
1310 if (des3->mode != CCP_DES3_MODE_ECB) {
1311 u32 load_mode;
1312
1313 op.sb_ctx = cmd_q->sb_ctx;
1314
1315 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1316 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1317 DMA_BIDIRECTIONAL);
1318 if (ret)
1319 goto e_key;
1320
1321 /* Load the context into the LSB */
1322 dm_offset = CCP_SB_BYTES - des3->iv_len;
1323 ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0, des3->iv_len);
1324
1325 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1326 load_mode = CCP_PASSTHRU_BYTESWAP_NOOP;
1327 else
1328 load_mode = CCP_PASSTHRU_BYTESWAP_256BIT;
1329 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1330 load_mode);
1331 if (ret) {
1332 cmd->engine_error = cmd_q->cmd_error;
1333 goto e_ctx;
1334 }
1335 }
1336
1337 /*
1338 * Prepare the input and output data workareas. For in-place
1339 * operations we need to set the dma direction to BIDIRECTIONAL
1340 * and copy the src workarea to the dst workarea.
1341 */
1342 if (sg_virt(des3->src) == sg_virt(des3->dst))
1343 in_place = true;
1344
1345 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1346 DES3_EDE_BLOCK_SIZE,
1347 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1348 if (ret)
1349 goto e_ctx;
1350
1351 if (in_place)
1352 dst = src;
1353 else {
1354 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1355 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1356 if (ret)
1357 goto e_src;
1358 }
1359
1360 /* Send data to the CCP DES3 engine */
1361 while (src.sg_wa.bytes_left) {
1362 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1363 if (!src.sg_wa.bytes_left) {
1364 op.eom = 1;
1365
1366 /* Since we don't retrieve the context in ECB mode
1367 * we have to wait for the operation to complete
1368 * on the last piece of data
1369 */
1370 op.soc = 0;
1371 }
1372
1373 ret = cmd_q->ccp->vdata->perform->des3(&op);
1374 if (ret) {
1375 cmd->engine_error = cmd_q->cmd_error;
1376 goto e_dst;
1377 }
1378
1379 ccp_process_data(&src, &dst, &op);
1380 }
1381
1382 if (des3->mode != CCP_DES3_MODE_ECB) {
1383 /* Retrieve the context and make BE */
1384 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1385 CCP_PASSTHRU_BYTESWAP_256BIT);
1386 if (ret) {
1387 cmd->engine_error = cmd_q->cmd_error;
1388 goto e_dst;
1389 }
1390
1391 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1392 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1393 dm_offset = CCP_SB_BYTES - des3->iv_len;
1394 else
1395 dm_offset = 0;
1396 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1397 DES3_EDE_BLOCK_SIZE);
1398 }
1399 e_dst:
1400 if (!in_place)
1401 ccp_free_data(&dst, cmd_q);
1402
1403 e_src:
1404 ccp_free_data(&src, cmd_q);
1405
1406 e_ctx:
1407 if (des3->mode != CCP_DES3_MODE_ECB)
1408 ccp_dm_free(&ctx);
1409
1410 e_key:
1411 ccp_dm_free(&key);
1412
1413 return ret;
1414 }
1415
1416 static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1417 {
1418 struct ccp_sha_engine *sha = &cmd->u.sha;
1419 struct ccp_dm_workarea ctx;
1420 struct ccp_data src;
1421 struct ccp_op op;
1422 unsigned int ioffset, ooffset;
1423 unsigned int digest_size;
1424 int sb_count;
1425 const void *init;
1426 u64 block_size;
1427 int ctx_size;
1428 int ret;
1429
1430 switch (sha->type) {
1431 case CCP_SHA_TYPE_1:
1432 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1433 return -EINVAL;
1434 block_size = SHA1_BLOCK_SIZE;
1435 break;
1436 case CCP_SHA_TYPE_224:
1437 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1438 return -EINVAL;
1439 block_size = SHA224_BLOCK_SIZE;
1440 break;
1441 case CCP_SHA_TYPE_256:
1442 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1443 return -EINVAL;
1444 block_size = SHA256_BLOCK_SIZE;
1445 break;
1446 case CCP_SHA_TYPE_384:
1447 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1448 || sha->ctx_len < SHA384_DIGEST_SIZE)
1449 return -EINVAL;
1450 block_size = SHA384_BLOCK_SIZE;
1451 break;
1452 case CCP_SHA_TYPE_512:
1453 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1454 || sha->ctx_len < SHA512_DIGEST_SIZE)
1455 return -EINVAL;
1456 block_size = SHA512_BLOCK_SIZE;
1457 break;
1458 default:
1459 return -EINVAL;
1460 }
1461
1462 if (!sha->ctx)
1463 return -EINVAL;
1464
1465 if (!sha->final && (sha->src_len & (block_size - 1)))
1466 return -EINVAL;
1467
1468 /* The version 3 device can't handle zero-length input */
1469 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1470
1471 if (!sha->src_len) {
1472 unsigned int digest_len;
1473 const u8 *sha_zero;
1474
1475 /* Not final, just return */
1476 if (!sha->final)
1477 return 0;
1478
1479 /* CCP can't do a zero length sha operation so the
1480 * caller must buffer the data.
1481 */
1482 if (sha->msg_bits)
1483 return -EINVAL;
1484
1485 /* The CCP cannot perform zero-length sha operations
1486 * so the caller is required to buffer data for the
1487 * final operation. However, a sha operation for a
1488 * message with a total length of zero is valid so
1489 * known values are required to supply the result.
1490 */
1491 switch (sha->type) {
1492 case CCP_SHA_TYPE_1:
1493 sha_zero = sha1_zero_message_hash;
1494 digest_len = SHA1_DIGEST_SIZE;
1495 break;
1496 case CCP_SHA_TYPE_224:
1497 sha_zero = sha224_zero_message_hash;
1498 digest_len = SHA224_DIGEST_SIZE;
1499 break;
1500 case CCP_SHA_TYPE_256:
1501 sha_zero = sha256_zero_message_hash;
1502 digest_len = SHA256_DIGEST_SIZE;
1503 break;
1504 default:
1505 return -EINVAL;
1506 }
1507
1508 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1509 digest_len, 1);
1510
1511 return 0;
1512 }
1513 }
1514
1515 /* Set variables used throughout */
1516 switch (sha->type) {
1517 case CCP_SHA_TYPE_1:
1518 digest_size = SHA1_DIGEST_SIZE;
1519 init = (void *) ccp_sha1_init;
1520 ctx_size = SHA1_DIGEST_SIZE;
1521 sb_count = 1;
1522 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1523 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1524 else
1525 ooffset = ioffset = 0;
1526 break;
1527 case CCP_SHA_TYPE_224:
1528 digest_size = SHA224_DIGEST_SIZE;
1529 init = (void *) ccp_sha224_init;
1530 ctx_size = SHA256_DIGEST_SIZE;
1531 sb_count = 1;
1532 ioffset = 0;
1533 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1534 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1535 else
1536 ooffset = 0;
1537 break;
1538 case CCP_SHA_TYPE_256:
1539 digest_size = SHA256_DIGEST_SIZE;
1540 init = (void *) ccp_sha256_init;
1541 ctx_size = SHA256_DIGEST_SIZE;
1542 sb_count = 1;
1543 ooffset = ioffset = 0;
1544 break;
1545 case CCP_SHA_TYPE_384:
1546 digest_size = SHA384_DIGEST_SIZE;
1547 init = (void *) ccp_sha384_init;
1548 ctx_size = SHA512_DIGEST_SIZE;
1549 sb_count = 2;
1550 ioffset = 0;
1551 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1552 break;
1553 case CCP_SHA_TYPE_512:
1554 digest_size = SHA512_DIGEST_SIZE;
1555 init = (void *) ccp_sha512_init;
1556 ctx_size = SHA512_DIGEST_SIZE;
1557 sb_count = 2;
1558 ooffset = ioffset = 0;
1559 break;
1560 default:
1561 ret = -EINVAL;
1562 goto e_data;
1563 }
1564
1565 /* For zero-length plaintext the src pointer is ignored;
1566 * otherwise both parts must be valid
1567 */
1568 if (sha->src_len && !sha->src)
1569 return -EINVAL;
1570
1571 memset(&op, 0, sizeof(op));
1572 op.cmd_q = cmd_q;
1573 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1574 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1575 op.u.sha.type = sha->type;
1576 op.u.sha.msg_bits = sha->msg_bits;
1577
1578 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1579 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1580 * first slot, and the left half in the second. Each portion must then
1581 * be in little endian format: use the 256-bit byte swap option.
1582 */
1583 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1584 DMA_BIDIRECTIONAL);
1585 if (ret)
1586 return ret;
1587 if (sha->first) {
1588 switch (sha->type) {
1589 case CCP_SHA_TYPE_1:
1590 case CCP_SHA_TYPE_224:
1591 case CCP_SHA_TYPE_256:
1592 memcpy(ctx.address + ioffset, init, ctx_size);
1593 break;
1594 case CCP_SHA_TYPE_384:
1595 case CCP_SHA_TYPE_512:
1596 memcpy(ctx.address + ctx_size / 2, init,
1597 ctx_size / 2);
1598 memcpy(ctx.address, init + ctx_size / 2,
1599 ctx_size / 2);
1600 break;
1601 default:
1602 ret = -EINVAL;
1603 goto e_ctx;
1604 }
1605 } else {
1606 /* Restore the context */
1607 ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1608 sb_count * CCP_SB_BYTES);
1609 }
1610
1611 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1612 CCP_PASSTHRU_BYTESWAP_256BIT);
1613 if (ret) {
1614 cmd->engine_error = cmd_q->cmd_error;
1615 goto e_ctx;
1616 }
1617
1618 if (sha->src) {
1619 /* Send data to the CCP SHA engine; block_size is set above */
1620 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1621 block_size, DMA_TO_DEVICE);
1622 if (ret)
1623 goto e_ctx;
1624
1625 while (src.sg_wa.bytes_left) {
1626 ccp_prepare_data(&src, NULL, &op, block_size, false);
1627 if (sha->final && !src.sg_wa.bytes_left)
1628 op.eom = 1;
1629
1630 ret = cmd_q->ccp->vdata->perform->sha(&op);
1631 if (ret) {
1632 cmd->engine_error = cmd_q->cmd_error;
1633 goto e_data;
1634 }
1635
1636 ccp_process_data(&src, NULL, &op);
1637 }
1638 } else {
1639 op.eom = 1;
1640 ret = cmd_q->ccp->vdata->perform->sha(&op);
1641 if (ret) {
1642 cmd->engine_error = cmd_q->cmd_error;
1643 goto e_data;
1644 }
1645 }
1646
1647 /* Retrieve the SHA context - convert from LE to BE using
1648 * 32-byte (256-bit) byteswapping to BE
1649 */
1650 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1651 CCP_PASSTHRU_BYTESWAP_256BIT);
1652 if (ret) {
1653 cmd->engine_error = cmd_q->cmd_error;
1654 goto e_data;
1655 }
1656
1657 if (sha->final) {
1658 /* Finishing up, so get the digest */
1659 switch (sha->type) {
1660 case CCP_SHA_TYPE_1:
1661 case CCP_SHA_TYPE_224:
1662 case CCP_SHA_TYPE_256:
1663 ccp_get_dm_area(&ctx, ooffset,
1664 sha->ctx, 0,
1665 digest_size);
1666 break;
1667 case CCP_SHA_TYPE_384:
1668 case CCP_SHA_TYPE_512:
1669 ccp_get_dm_area(&ctx, 0,
1670 sha->ctx, LSB_ITEM_SIZE - ooffset,
1671 LSB_ITEM_SIZE);
1672 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1673 sha->ctx, 0,
1674 LSB_ITEM_SIZE - ooffset);
1675 break;
1676 default:
1677 ret = -EINVAL;
1678 goto e_ctx;
1679 }
1680 } else {
1681 /* Stash the context */
1682 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1683 sb_count * CCP_SB_BYTES);
1684 }
1685
1686 if (sha->final && sha->opad) {
1687 /* HMAC operation, recursively perform final SHA */
1688 struct ccp_cmd hmac_cmd;
1689 struct scatterlist sg;
1690 u8 *hmac_buf;
1691
1692 if (sha->opad_len != block_size) {
1693 ret = -EINVAL;
1694 goto e_data;
1695 }
1696
1697 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1698 if (!hmac_buf) {
1699 ret = -ENOMEM;
1700 goto e_data;
1701 }
1702 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1703
1704 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1705 switch (sha->type) {
1706 case CCP_SHA_TYPE_1:
1707 case CCP_SHA_TYPE_224:
1708 case CCP_SHA_TYPE_256:
1709 memcpy(hmac_buf + block_size,
1710 ctx.address + ooffset,
1711 digest_size);
1712 break;
1713 case CCP_SHA_TYPE_384:
1714 case CCP_SHA_TYPE_512:
1715 memcpy(hmac_buf + block_size,
1716 ctx.address + LSB_ITEM_SIZE + ooffset,
1717 LSB_ITEM_SIZE);
1718 memcpy(hmac_buf + block_size +
1719 (LSB_ITEM_SIZE - ooffset),
1720 ctx.address,
1721 LSB_ITEM_SIZE);
1722 break;
1723 default:
1724 ret = -EINVAL;
1725 goto e_ctx;
1726 }
1727
1728 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1729 hmac_cmd.engine = CCP_ENGINE_SHA;
1730 hmac_cmd.u.sha.type = sha->type;
1731 hmac_cmd.u.sha.ctx = sha->ctx;
1732 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1733 hmac_cmd.u.sha.src = &sg;
1734 hmac_cmd.u.sha.src_len = block_size + digest_size;
1735 hmac_cmd.u.sha.opad = NULL;
1736 hmac_cmd.u.sha.opad_len = 0;
1737 hmac_cmd.u.sha.first = 1;
1738 hmac_cmd.u.sha.final = 1;
1739 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1740
1741 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1742 if (ret)
1743 cmd->engine_error = hmac_cmd.engine_error;
1744
1745 kfree(hmac_buf);
1746 }
1747
1748 e_data:
1749 if (sha->src)
1750 ccp_free_data(&src, cmd_q);
1751
1752 e_ctx:
1753 ccp_dm_free(&ctx);
1754
1755 return ret;
1756 }
1757
1758 static int ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1759 {
1760 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1761 struct ccp_dm_workarea exp, src, dst;
1762 struct ccp_op op;
1763 unsigned int sb_count, i_len, o_len;
1764 int ret;
1765
1766 /* Check against the maximum allowable size, in bits */
1767 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1768 return -EINVAL;
1769
1770 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1771 return -EINVAL;
1772
1773 memset(&op, 0, sizeof(op));
1774 op.cmd_q = cmd_q;
1775 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1776
1777 /* The RSA modulus must precede the message being acted upon, so
1778 * it must be copied to a DMA area where the message and the
1779 * modulus can be concatenated. Therefore the input buffer
1780 * length required is twice the output buffer length (which
1781 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1782 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1783 * required.
1784 */
1785 o_len = 32 * ((rsa->key_size + 255) / 256);
1786 i_len = o_len * 2;
1787
1788 sb_count = 0;
1789 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1790 /* sb_count is the number of storage block slots required
1791 * for the modulus.
1792 */
1793 sb_count = o_len / CCP_SB_BYTES;
1794 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1795 sb_count);
1796 if (!op.sb_key)
1797 return -EIO;
1798 } else {
1799 /* A version 5 device allows a modulus size that will not fit
1800 * in the LSB, so the command will transfer it from memory.
1801 * Set the sb key to the default, even though it's not used.
1802 */
1803 op.sb_key = cmd_q->sb_key;
1804 }
1805
1806 /* The RSA exponent must be in little endian format. Reverse its
1807 * byte order.
1808 */
1809 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1810 if (ret)
1811 goto e_sb;
1812
1813 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1814 if (ret)
1815 goto e_exp;
1816
1817 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1818 /* Copy the exponent to the local storage block, using
1819 * as many 32-byte blocks as were allocated above. It's
1820 * already little endian, so no further change is required.
1821 */
1822 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1823 CCP_PASSTHRU_BYTESWAP_NOOP);
1824 if (ret) {
1825 cmd->engine_error = cmd_q->cmd_error;
1826 goto e_exp;
1827 }
1828 } else {
1829 /* The exponent can be retrieved from memory via DMA. */
1830 op.exp.u.dma.address = exp.dma.address;
1831 op.exp.u.dma.offset = 0;
1832 }
1833
1834 /* Concatenate the modulus and the message. Both the modulus and
1835 * the operands must be in little endian format. Since the input
1836 * is in big endian format it must be converted.
1837 */
1838 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1839 if (ret)
1840 goto e_exp;
1841
1842 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1843 if (ret)
1844 goto e_src;
1845 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1846 if (ret)
1847 goto e_src;
1848
1849 /* Prepare the output area for the operation */
1850 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1851 if (ret)
1852 goto e_src;
1853
1854 op.soc = 1;
1855 op.src.u.dma.address = src.dma.address;
1856 op.src.u.dma.offset = 0;
1857 op.src.u.dma.length = i_len;
1858 op.dst.u.dma.address = dst.dma.address;
1859 op.dst.u.dma.offset = 0;
1860 op.dst.u.dma.length = o_len;
1861
1862 op.u.rsa.mod_size = rsa->key_size;
1863 op.u.rsa.input_len = i_len;
1864
1865 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1866 if (ret) {
1867 cmd->engine_error = cmd_q->cmd_error;
1868 goto e_dst;
1869 }
1870
1871 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1872
1873 e_dst:
1874 ccp_dm_free(&dst);
1875
1876 e_src:
1877 ccp_dm_free(&src);
1878
1879 e_exp:
1880 ccp_dm_free(&exp);
1881
1882 e_sb:
1883 if (sb_count)
1884 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1885
1886 return ret;
1887 }
1888
1889 static int ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q,
1890 struct ccp_cmd *cmd)
1891 {
1892 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1893 struct ccp_dm_workarea mask;
1894 struct ccp_data src, dst;
1895 struct ccp_op op;
1896 bool in_place = false;
1897 unsigned int i;
1898 int ret = 0;
1899
1900 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1901 return -EINVAL;
1902
1903 if (!pt->src || !pt->dst)
1904 return -EINVAL;
1905
1906 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1907 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1908 return -EINVAL;
1909 if (!pt->mask)
1910 return -EINVAL;
1911 }
1912
1913 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1914
1915 memset(&op, 0, sizeof(op));
1916 op.cmd_q = cmd_q;
1917 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1918
1919 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1920 /* Load the mask */
1921 op.sb_key = cmd_q->sb_key;
1922
1923 ret = ccp_init_dm_workarea(&mask, cmd_q,
1924 CCP_PASSTHRU_SB_COUNT *
1925 CCP_SB_BYTES,
1926 DMA_TO_DEVICE);
1927 if (ret)
1928 return ret;
1929
1930 ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1931 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1932 CCP_PASSTHRU_BYTESWAP_NOOP);
1933 if (ret) {
1934 cmd->engine_error = cmd_q->cmd_error;
1935 goto e_mask;
1936 }
1937 }
1938
1939 /* Prepare the input and output data workareas. For in-place
1940 * operations we need to set the dma direction to BIDIRECTIONAL
1941 * and copy the src workarea to the dst workarea.
1942 */
1943 if (sg_virt(pt->src) == sg_virt(pt->dst))
1944 in_place = true;
1945
1946 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
1947 CCP_PASSTHRU_MASKSIZE,
1948 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1949 if (ret)
1950 goto e_mask;
1951
1952 if (in_place) {
1953 dst = src;
1954 } else {
1955 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
1956 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
1957 if (ret)
1958 goto e_src;
1959 }
1960
1961 /* Send data to the CCP Passthru engine
1962 * Because the CCP engine works on a single source and destination
1963 * dma address at a time, each entry in the source scatterlist
1964 * (after the dma_map_sg call) must be less than or equal to the
1965 * (remaining) length in the destination scatterlist entry and the
1966 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
1967 */
1968 dst.sg_wa.sg_used = 0;
1969 for (i = 1; i <= src.sg_wa.dma_count; i++) {
1970 if (!dst.sg_wa.sg ||
1971 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
1972 ret = -EINVAL;
1973 goto e_dst;
1974 }
1975
1976 if (i == src.sg_wa.dma_count) {
1977 op.eom = 1;
1978 op.soc = 1;
1979 }
1980
1981 op.src.type = CCP_MEMTYPE_SYSTEM;
1982 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
1983 op.src.u.dma.offset = 0;
1984 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
1985
1986 op.dst.type = CCP_MEMTYPE_SYSTEM;
1987 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
1988 op.dst.u.dma.offset = dst.sg_wa.sg_used;
1989 op.dst.u.dma.length = op.src.u.dma.length;
1990
1991 ret = cmd_q->ccp->vdata->perform->passthru(&op);
1992 if (ret) {
1993 cmd->engine_error = cmd_q->cmd_error;
1994 goto e_dst;
1995 }
1996
1997 dst.sg_wa.sg_used += src.sg_wa.sg->length;
1998 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
1999 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2000 dst.sg_wa.sg_used = 0;
2001 }
2002 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2003 }
2004
2005 e_dst:
2006 if (!in_place)
2007 ccp_free_data(&dst, cmd_q);
2008
2009 e_src:
2010 ccp_free_data(&src, cmd_q);
2011
2012 e_mask:
2013 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2014 ccp_dm_free(&mask);
2015
2016 return ret;
2017 }
2018
2019 static int ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2020 struct ccp_cmd *cmd)
2021 {
2022 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2023 struct ccp_dm_workarea mask;
2024 struct ccp_op op;
2025 int ret;
2026
2027 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2028 return -EINVAL;
2029
2030 if (!pt->src_dma || !pt->dst_dma)
2031 return -EINVAL;
2032
2033 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2034 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2035 return -EINVAL;
2036 if (!pt->mask)
2037 return -EINVAL;
2038 }
2039
2040 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2041
2042 memset(&op, 0, sizeof(op));
2043 op.cmd_q = cmd_q;
2044 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2045
2046 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2047 /* Load the mask */
2048 op.sb_key = cmd_q->sb_key;
2049
2050 mask.length = pt->mask_len;
2051 mask.dma.address = pt->mask;
2052 mask.dma.length = pt->mask_len;
2053
2054 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2055 CCP_PASSTHRU_BYTESWAP_NOOP);
2056 if (ret) {
2057 cmd->engine_error = cmd_q->cmd_error;
2058 return ret;
2059 }
2060 }
2061
2062 /* Send data to the CCP Passthru engine */
2063 op.eom = 1;
2064 op.soc = 1;
2065
2066 op.src.type = CCP_MEMTYPE_SYSTEM;
2067 op.src.u.dma.address = pt->src_dma;
2068 op.src.u.dma.offset = 0;
2069 op.src.u.dma.length = pt->src_len;
2070
2071 op.dst.type = CCP_MEMTYPE_SYSTEM;
2072 op.dst.u.dma.address = pt->dst_dma;
2073 op.dst.u.dma.offset = 0;
2074 op.dst.u.dma.length = pt->src_len;
2075
2076 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2077 if (ret)
2078 cmd->engine_error = cmd_q->cmd_error;
2079
2080 return ret;
2081 }
2082
2083 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2084 {
2085 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2086 struct ccp_dm_workarea src, dst;
2087 struct ccp_op op;
2088 int ret;
2089 u8 *save;
2090
2091 if (!ecc->u.mm.operand_1 ||
2092 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2093 return -EINVAL;
2094
2095 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2096 if (!ecc->u.mm.operand_2 ||
2097 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2098 return -EINVAL;
2099
2100 if (!ecc->u.mm.result ||
2101 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2102 return -EINVAL;
2103
2104 memset(&op, 0, sizeof(op));
2105 op.cmd_q = cmd_q;
2106 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2107
2108 /* Concatenate the modulus and the operands. Both the modulus and
2109 * the operands must be in little endian format. Since the input
2110 * is in big endian format it must be converted and placed in a
2111 * fixed length buffer.
2112 */
2113 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2114 DMA_TO_DEVICE);
2115 if (ret)
2116 return ret;
2117
2118 /* Save the workarea address since it is updated in order to perform
2119 * the concatenation
2120 */
2121 save = src.address;
2122
2123 /* Copy the ECC modulus */
2124 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2125 if (ret)
2126 goto e_src;
2127 src.address += CCP_ECC_OPERAND_SIZE;
2128
2129 /* Copy the first operand */
2130 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2131 ecc->u.mm.operand_1_len);
2132 if (ret)
2133 goto e_src;
2134 src.address += CCP_ECC_OPERAND_SIZE;
2135
2136 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2137 /* Copy the second operand */
2138 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2139 ecc->u.mm.operand_2_len);
2140 if (ret)
2141 goto e_src;
2142 src.address += CCP_ECC_OPERAND_SIZE;
2143 }
2144
2145 /* Restore the workarea address */
2146 src.address = save;
2147
2148 /* Prepare the output area for the operation */
2149 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2150 DMA_FROM_DEVICE);
2151 if (ret)
2152 goto e_src;
2153
2154 op.soc = 1;
2155 op.src.u.dma.address = src.dma.address;
2156 op.src.u.dma.offset = 0;
2157 op.src.u.dma.length = src.length;
2158 op.dst.u.dma.address = dst.dma.address;
2159 op.dst.u.dma.offset = 0;
2160 op.dst.u.dma.length = dst.length;
2161
2162 op.u.ecc.function = cmd->u.ecc.function;
2163
2164 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2165 if (ret) {
2166 cmd->engine_error = cmd_q->cmd_error;
2167 goto e_dst;
2168 }
2169
2170 ecc->ecc_result = le16_to_cpup(
2171 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2172 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2173 ret = -EIO;
2174 goto e_dst;
2175 }
2176
2177 /* Save the ECC result */
2178 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2179 CCP_ECC_MODULUS_BYTES);
2180
2181 e_dst:
2182 ccp_dm_free(&dst);
2183
2184 e_src:
2185 ccp_dm_free(&src);
2186
2187 return ret;
2188 }
2189
2190 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2191 {
2192 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2193 struct ccp_dm_workarea src, dst;
2194 struct ccp_op op;
2195 int ret;
2196 u8 *save;
2197
2198 if (!ecc->u.pm.point_1.x ||
2199 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2200 !ecc->u.pm.point_1.y ||
2201 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2202 return -EINVAL;
2203
2204 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2205 if (!ecc->u.pm.point_2.x ||
2206 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2207 !ecc->u.pm.point_2.y ||
2208 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2209 return -EINVAL;
2210 } else {
2211 if (!ecc->u.pm.domain_a ||
2212 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2213 return -EINVAL;
2214
2215 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2216 if (!ecc->u.pm.scalar ||
2217 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2218 return -EINVAL;
2219 }
2220
2221 if (!ecc->u.pm.result.x ||
2222 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2223 !ecc->u.pm.result.y ||
2224 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2225 return -EINVAL;
2226
2227 memset(&op, 0, sizeof(op));
2228 op.cmd_q = cmd_q;
2229 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2230
2231 /* Concatenate the modulus and the operands. Both the modulus and
2232 * the operands must be in little endian format. Since the input
2233 * is in big endian format it must be converted and placed in a
2234 * fixed length buffer.
2235 */
2236 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2237 DMA_TO_DEVICE);
2238 if (ret)
2239 return ret;
2240
2241 /* Save the workarea address since it is updated in order to perform
2242 * the concatenation
2243 */
2244 save = src.address;
2245
2246 /* Copy the ECC modulus */
2247 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2248 if (ret)
2249 goto e_src;
2250 src.address += CCP_ECC_OPERAND_SIZE;
2251
2252 /* Copy the first point X and Y coordinate */
2253 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2254 ecc->u.pm.point_1.x_len);
2255 if (ret)
2256 goto e_src;
2257 src.address += CCP_ECC_OPERAND_SIZE;
2258 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2259 ecc->u.pm.point_1.y_len);
2260 if (ret)
2261 goto e_src;
2262 src.address += CCP_ECC_OPERAND_SIZE;
2263
2264 /* Set the first point Z coordinate to 1 */
2265 *src.address = 0x01;
2266 src.address += CCP_ECC_OPERAND_SIZE;
2267
2268 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2269 /* Copy the second point X and Y coordinate */
2270 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2271 ecc->u.pm.point_2.x_len);
2272 if (ret)
2273 goto e_src;
2274 src.address += CCP_ECC_OPERAND_SIZE;
2275 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2276 ecc->u.pm.point_2.y_len);
2277 if (ret)
2278 goto e_src;
2279 src.address += CCP_ECC_OPERAND_SIZE;
2280
2281 /* Set the second point Z coordinate to 1 */
2282 *src.address = 0x01;
2283 src.address += CCP_ECC_OPERAND_SIZE;
2284 } else {
2285 /* Copy the Domain "a" parameter */
2286 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2287 ecc->u.pm.domain_a_len);
2288 if (ret)
2289 goto e_src;
2290 src.address += CCP_ECC_OPERAND_SIZE;
2291
2292 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2293 /* Copy the scalar value */
2294 ret = ccp_reverse_set_dm_area(&src, 0,
2295 ecc->u.pm.scalar, 0,
2296 ecc->u.pm.scalar_len);
2297 if (ret)
2298 goto e_src;
2299 src.address += CCP_ECC_OPERAND_SIZE;
2300 }
2301 }
2302
2303 /* Restore the workarea address */
2304 src.address = save;
2305
2306 /* Prepare the output area for the operation */
2307 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2308 DMA_FROM_DEVICE);
2309 if (ret)
2310 goto e_src;
2311
2312 op.soc = 1;
2313 op.src.u.dma.address = src.dma.address;
2314 op.src.u.dma.offset = 0;
2315 op.src.u.dma.length = src.length;
2316 op.dst.u.dma.address = dst.dma.address;
2317 op.dst.u.dma.offset = 0;
2318 op.dst.u.dma.length = dst.length;
2319
2320 op.u.ecc.function = cmd->u.ecc.function;
2321
2322 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2323 if (ret) {
2324 cmd->engine_error = cmd_q->cmd_error;
2325 goto e_dst;
2326 }
2327
2328 ecc->ecc_result = le16_to_cpup(
2329 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2330 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2331 ret = -EIO;
2332 goto e_dst;
2333 }
2334
2335 /* Save the workarea address since it is updated as we walk through
2336 * to copy the point math result
2337 */
2338 save = dst.address;
2339
2340 /* Save the ECC result X and Y coordinates */
2341 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2342 CCP_ECC_MODULUS_BYTES);
2343 dst.address += CCP_ECC_OUTPUT_SIZE;
2344 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2345 CCP_ECC_MODULUS_BYTES);
2346 dst.address += CCP_ECC_OUTPUT_SIZE;
2347
2348 /* Restore the workarea address */
2349 dst.address = save;
2350
2351 e_dst:
2352 ccp_dm_free(&dst);
2353
2354 e_src:
2355 ccp_dm_free(&src);
2356
2357 return ret;
2358 }
2359
2360 static int ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2361 {
2362 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2363
2364 ecc->ecc_result = 0;
2365
2366 if (!ecc->mod ||
2367 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2368 return -EINVAL;
2369
2370 switch (ecc->function) {
2371 case CCP_ECC_FUNCTION_MMUL_384BIT:
2372 case CCP_ECC_FUNCTION_MADD_384BIT:
2373 case CCP_ECC_FUNCTION_MINV_384BIT:
2374 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2375
2376 case CCP_ECC_FUNCTION_PADD_384BIT:
2377 case CCP_ECC_FUNCTION_PMUL_384BIT:
2378 case CCP_ECC_FUNCTION_PDBL_384BIT:
2379 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2380
2381 default:
2382 return -EINVAL;
2383 }
2384 }
2385
2386 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2387 {
2388 int ret;
2389
2390 cmd->engine_error = 0;
2391 cmd_q->cmd_error = 0;
2392 cmd_q->int_rcvd = 0;
2393 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2394
2395 switch (cmd->engine) {
2396 case CCP_ENGINE_AES:
2397 ret = ccp_run_aes_cmd(cmd_q, cmd);
2398 break;
2399 case CCP_ENGINE_XTS_AES_128:
2400 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2401 break;
2402 case CCP_ENGINE_DES3:
2403 ret = ccp_run_des3_cmd(cmd_q, cmd);
2404 break;
2405 case CCP_ENGINE_SHA:
2406 ret = ccp_run_sha_cmd(cmd_q, cmd);
2407 break;
2408 case CCP_ENGINE_RSA:
2409 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2410 break;
2411 case CCP_ENGINE_PASSTHRU:
2412 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2413 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2414 else
2415 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2416 break;
2417 case CCP_ENGINE_ECC:
2418 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2419 break;
2420 default:
2421 ret = -EINVAL;
2422 }
2423
2424 return ret;
2425 }
CRIS linux kernel tree