?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / crypto / ccp / ccp-ops.c
blob330853a2702f027ac82b5bab4a098ab4f2e8d0d7
1 /*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
4 * Copyright (C) 2013,2017 Advanced Micro Devices, Inc.
5 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
7 * Author: Gary R Hook <gary.hook@amd.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14 #include <linux/module.h>
15 #include <linux/kernel.h>
16 #include <linux/pci.h>
17 #include <linux/interrupt.h>
18 #include <crypto/scatterwalk.h>
19 #include <crypto/des.h>
20 #include <linux/ccp.h>
21
22 #include "ccp-dev.h"
23
24 /* SHA initial context values */
25 static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
26 cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
27 cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
28 cpu_to_be32(SHA1_H4),
29 };
30
31 static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
32 cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
33 cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
34 cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
35 cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
36 };
37
38 static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
39 cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
40 cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
41 cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
42 cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
43 };
44
45 static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
46 cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
47 cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
48 cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
49 cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
50 };
51
52 static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
53 cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
54 cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
55 cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
56 cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
57 };
58
59 #define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
60 ccp_gen_jobid(ccp) : 0)
61
62 static u32 ccp_gen_jobid(struct ccp_device *ccp)
63 {
64 return atomic_inc_return(&ccp->current_id) & CCP_JOBID_MASK;
65 }
66
67 static void ccp_sg_free(struct ccp_sg_workarea *wa)
68 {
69 if (wa->dma_count)
70 dma_unmap_sg(wa->dma_dev, wa->dma_sg, wa->nents, wa->dma_dir);
71
72 wa->dma_count = 0;
73 }
74
75 static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
76 struct scatterlist *sg, u64 len,
77 enum dma_data_direction dma_dir)
78 {
79 memset(wa, 0, sizeof(*wa));
80
81 wa->sg = sg;
82 if (!sg)
83 return 0;
84
85 wa->nents = sg_nents_for_len(sg, len);
86 if (wa->nents < 0)
87 return wa->nents;
88
89 wa->bytes_left = len;
90 wa->sg_used = 0;
91
92 if (len == 0)
93 return 0;
94
95 if (dma_dir == DMA_NONE)
96 return 0;
97
98 wa->dma_sg = sg;
99 wa->dma_dev = dev;
100 wa->dma_dir = dma_dir;
101 wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
102 if (!wa->dma_count)
103 return -ENOMEM;
104
105 return 0;
106 }
107
108 static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
109 {
110 unsigned int nbytes = min_t(u64, len, wa->bytes_left);
111
112 if (!wa->sg)
113 return;
114
115 wa->sg_used += nbytes;
116 wa->bytes_left -= nbytes;
117 if (wa->sg_used == wa->sg->length) {
118 wa->sg = sg_next(wa->sg);
119 wa->sg_used = 0;
120 }
121 }
122
123 static void ccp_dm_free(struct ccp_dm_workarea *wa)
124 {
125 if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
126 if (wa->address)
127 dma_pool_free(wa->dma_pool, wa->address,
128 wa->dma.address);
129 } else {
130 if (wa->dma.address)
131 dma_unmap_single(wa->dev, wa->dma.address, wa->length,
132 wa->dma.dir);
133 kfree(wa->address);
134 }
135
136 wa->address = NULL;
137 wa->dma.address = 0;
138 }
139
140 static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
141 struct ccp_cmd_queue *cmd_q,
142 unsigned int len,
143 enum dma_data_direction dir)
144 {
145 memset(wa, 0, sizeof(*wa));
146
147 if (!len)
148 return 0;
149
150 wa->dev = cmd_q->ccp->dev;
151 wa->length = len;
152
153 if (len <= CCP_DMAPOOL_MAX_SIZE) {
154 wa->dma_pool = cmd_q->dma_pool;
155
156 wa->address = dma_pool_alloc(wa->dma_pool, GFP_KERNEL,
157 &wa->dma.address);
158 if (!wa->address)
159 return -ENOMEM;
160
161 wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
162
163 memset(wa->address, 0, CCP_DMAPOOL_MAX_SIZE);
164 } else {
165 wa->address = kzalloc(len, GFP_KERNEL);
166 if (!wa->address)
167 return -ENOMEM;
168
169 wa->dma.address = dma_map_single(wa->dev, wa->address, len,
170 dir);
171 if (dma_mapping_error(wa->dev, wa->dma.address))
172 return -ENOMEM;
173
174 wa->dma.length = len;
175 }
176 wa->dma.dir = dir;
177
178 return 0;
179 }
180
181 static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
182 struct scatterlist *sg, unsigned int sg_offset,
183 unsigned int len)
184 {
185 WARN_ON(!wa->address);
186
187 if (len > (wa->length - wa_offset))
188 return -EINVAL;
189
190 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
191 0);
192 return 0;
193 }
194
195 static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
196 struct scatterlist *sg, unsigned int sg_offset,
197 unsigned int len)
198 {
199 WARN_ON(!wa->address);
200
201 scatterwalk_map_and_copy(wa->address + wa_offset, sg, sg_offset, len,
202 1);
203 }
204
205 static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
206 unsigned int wa_offset,
207 struct scatterlist *sg,
208 unsigned int sg_offset,
209 unsigned int len)
210 {
211 u8 *p, *q;
212 int rc;
213
214 rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
215 if (rc)
216 return rc;
217
218 p = wa->address + wa_offset;
219 q = p + len - 1;
220 while (p < q) {
221 *p = *p ^ *q;
222 *q = *p ^ *q;
223 *p = *p ^ *q;
224 p++;
225 q--;
226 }
227 return 0;
228 }
229
230 static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
231 unsigned int wa_offset,
232 struct scatterlist *sg,
233 unsigned int sg_offset,
234 unsigned int len)
235 {
236 u8 *p, *q;
237
238 p = wa->address + wa_offset;
239 q = p + len - 1;
240 while (p < q) {
241 *p = *p ^ *q;
242 *q = *p ^ *q;
243 *p = *p ^ *q;
244 p++;
245 q--;
246 }
247
248 ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
249 }
250
251 static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
252 {
253 ccp_dm_free(&data->dm_wa);
254 ccp_sg_free(&data->sg_wa);
255 }
256
257 static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
258 struct scatterlist *sg, u64 sg_len,
259 unsigned int dm_len,
260 enum dma_data_direction dir)
261 {
262 int ret;
263
264 memset(data, 0, sizeof(*data));
265
266 ret = ccp_init_sg_workarea(&data->sg_wa, cmd_q->ccp->dev, sg, sg_len,
267 dir);
268 if (ret)
269 goto e_err;
270
271 ret = ccp_init_dm_workarea(&data->dm_wa, cmd_q, dm_len, dir);
272 if (ret)
273 goto e_err;
274
275 return 0;
276
277 e_err:
278 ccp_free_data(data, cmd_q);
279
280 return ret;
281 }
282
283 static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
284 {
285 struct ccp_sg_workarea *sg_wa = &data->sg_wa;
286 struct ccp_dm_workarea *dm_wa = &data->dm_wa;
287 unsigned int buf_count, nbytes;
288
289 /* Clear the buffer if setting it */
290 if (!from)
291 memset(dm_wa->address, 0, dm_wa->length);
292
293 if (!sg_wa->sg)
294 return 0;
295
296 /* Perform the copy operation
297 * nbytes will always be <= UINT_MAX because dm_wa->length is
298 * an unsigned int
299 */
300 nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
301 scatterwalk_map_and_copy(dm_wa->address, sg_wa->sg, sg_wa->sg_used,
302 nbytes, from);
303
304 /* Update the structures and generate the count */
305 buf_count = 0;
306 while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
307 nbytes = min(sg_wa->sg->length - sg_wa->sg_used,
308 dm_wa->length - buf_count);
309 nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
310
311 buf_count += nbytes;
312 ccp_update_sg_workarea(sg_wa, nbytes);
313 }
314
315 return buf_count;
316 }
317
318 static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
319 {
320 return ccp_queue_buf(data, 0);
321 }
322
323 static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
324 {
325 return ccp_queue_buf(data, 1);
326 }
327
328 static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
329 struct ccp_op *op, unsigned int block_size,
330 bool blocksize_op)
331 {
332 unsigned int sg_src_len, sg_dst_len, op_len;
333
334 /* The CCP can only DMA from/to one address each per operation. This
335 * requires that we find the smallest DMA area between the source
336 * and destination. The resulting len values will always be <= UINT_MAX
337 * because the dma length is an unsigned int.
338 */
339 sg_src_len = sg_dma_len(src->sg_wa.sg) - src->sg_wa.sg_used;
340 sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
341
342 if (dst) {
343 sg_dst_len = sg_dma_len(dst->sg_wa.sg) - dst->sg_wa.sg_used;
344 sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
345 op_len = min(sg_src_len, sg_dst_len);
346 } else {
347 op_len = sg_src_len;
348 }
349
350 /* The data operation length will be at least block_size in length
351 * or the smaller of available sg room remaining for the source or
352 * the destination
353 */
354 op_len = max(op_len, block_size);
355
356 /* Unless we have to buffer data, there's no reason to wait */
357 op->soc = 0;
358
359 if (sg_src_len < block_size) {
360 /* Not enough data in the sg element, so it
361 * needs to be buffered into a blocksize chunk
362 */
363 int cp_len = ccp_fill_queue_buf(src);
364
365 op->soc = 1;
366 op->src.u.dma.address = src->dm_wa.dma.address;
367 op->src.u.dma.offset = 0;
368 op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
369 } else {
370 /* Enough data in the sg element, but we need to
371 * adjust for any previously copied data
372 */
373 op->src.u.dma.address = sg_dma_address(src->sg_wa.sg);
374 op->src.u.dma.offset = src->sg_wa.sg_used;
375 op->src.u.dma.length = op_len & ~(block_size - 1);
376
377 ccp_update_sg_workarea(&src->sg_wa, op->src.u.dma.length);
378 }
379
380 if (dst) {
381 if (sg_dst_len < block_size) {
382 /* Not enough room in the sg element or we're on the
383 * last piece of data (when using padding), so the
384 * output needs to be buffered into a blocksize chunk
385 */
386 op->soc = 1;
387 op->dst.u.dma.address = dst->dm_wa.dma.address;
388 op->dst.u.dma.offset = 0;
389 op->dst.u.dma.length = op->src.u.dma.length;
390 } else {
391 /* Enough room in the sg element, but we need to
392 * adjust for any previously used area
393 */
394 op->dst.u.dma.address = sg_dma_address(dst->sg_wa.sg);
395 op->dst.u.dma.offset = dst->sg_wa.sg_used;
396 op->dst.u.dma.length = op->src.u.dma.length;
397 }
398 }
399 }
400
401 static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
402 struct ccp_op *op)
403 {
404 op->init = 0;
405
406 if (dst) {
407 if (op->dst.u.dma.address == dst->dm_wa.dma.address)
408 ccp_empty_queue_buf(dst);
409 else
410 ccp_update_sg_workarea(&dst->sg_wa,
411 op->dst.u.dma.length);
412 }
413 }
414
415 static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
416 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
417 u32 byte_swap, bool from)
418 {
419 struct ccp_op op;
420
421 memset(&op, 0, sizeof(op));
422
423 op.cmd_q = cmd_q;
424 op.jobid = jobid;
425 op.eom = 1;
426
427 if (from) {
428 op.soc = 1;
429 op.src.type = CCP_MEMTYPE_SB;
430 op.src.u.sb = sb;
431 op.dst.type = CCP_MEMTYPE_SYSTEM;
432 op.dst.u.dma.address = wa->dma.address;
433 op.dst.u.dma.length = wa->length;
434 } else {
435 op.src.type = CCP_MEMTYPE_SYSTEM;
436 op.src.u.dma.address = wa->dma.address;
437 op.src.u.dma.length = wa->length;
438 op.dst.type = CCP_MEMTYPE_SB;
439 op.dst.u.sb = sb;
440 }
441
442 op.u.passthru.byte_swap = byte_swap;
443
444 return cmd_q->ccp->vdata->perform->passthru(&op);
445 }
446
447 static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
448 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
449 u32 byte_swap)
450 {
451 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, false);
452 }
453
454 static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
455 struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
456 u32 byte_swap)
457 {
458 return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, true);
459 }
460
461 static noinline_for_stack int
462 ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
463 {
464 struct ccp_aes_engine *aes = &cmd->u.aes;
465 struct ccp_dm_workarea key, ctx;
466 struct ccp_data src;
467 struct ccp_op op;
468 unsigned int dm_offset;
469 int ret;
470
471 if (!((aes->key_len == AES_KEYSIZE_128) ||
472 (aes->key_len == AES_KEYSIZE_192) ||
473 (aes->key_len == AES_KEYSIZE_256)))
474 return -EINVAL;
475
476 if (aes->src_len & (AES_BLOCK_SIZE - 1))
477 return -EINVAL;
478
479 if (aes->iv_len != AES_BLOCK_SIZE)
480 return -EINVAL;
481
482 if (!aes->key || !aes->iv || !aes->src)
483 return -EINVAL;
484
485 if (aes->cmac_final) {
486 if (aes->cmac_key_len != AES_BLOCK_SIZE)
487 return -EINVAL;
488
489 if (!aes->cmac_key)
490 return -EINVAL;
491 }
492
493 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
494 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
495
496 ret = -EIO;
497 memset(&op, 0, sizeof(op));
498 op.cmd_q = cmd_q;
499 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
500 op.sb_key = cmd_q->sb_key;
501 op.sb_ctx = cmd_q->sb_ctx;
502 op.init = 1;
503 op.u.aes.type = aes->type;
504 op.u.aes.mode = aes->mode;
505 op.u.aes.action = aes->action;
506
507 /* All supported key sizes fit in a single (32-byte) SB entry
508 * and must be in little endian format. Use the 256-bit byte
509 * swap passthru option to convert from big endian to little
510 * endian.
511 */
512 ret = ccp_init_dm_workarea(&key, cmd_q,
513 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
514 DMA_TO_DEVICE);
515 if (ret)
516 return ret;
517
518 dm_offset = CCP_SB_BYTES - aes->key_len;
519 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
520 if (ret)
521 goto e_key;
522 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
523 CCP_PASSTHRU_BYTESWAP_256BIT);
524 if (ret) {
525 cmd->engine_error = cmd_q->cmd_error;
526 goto e_key;
527 }
528
529 /* The AES context fits in a single (32-byte) SB entry and
530 * must be in little endian format. Use the 256-bit byte swap
531 * passthru option to convert from big endian to little endian.
532 */
533 ret = ccp_init_dm_workarea(&ctx, cmd_q,
534 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
535 DMA_BIDIRECTIONAL);
536 if (ret)
537 goto e_key;
538
539 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
540 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
541 if (ret)
542 goto e_ctx;
543 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
544 CCP_PASSTHRU_BYTESWAP_256BIT);
545 if (ret) {
546 cmd->engine_error = cmd_q->cmd_error;
547 goto e_ctx;
548 }
549
550 /* Send data to the CCP AES engine */
551 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
552 AES_BLOCK_SIZE, DMA_TO_DEVICE);
553 if (ret)
554 goto e_ctx;
555
556 while (src.sg_wa.bytes_left) {
557 ccp_prepare_data(&src, NULL, &op, AES_BLOCK_SIZE, true);
558 if (aes->cmac_final && !src.sg_wa.bytes_left) {
559 op.eom = 1;
560
561 /* Push the K1/K2 key to the CCP now */
562 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid,
563 op.sb_ctx,
564 CCP_PASSTHRU_BYTESWAP_256BIT);
565 if (ret) {
566 cmd->engine_error = cmd_q->cmd_error;
567 goto e_src;
568 }
569
570 ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,
571 aes->cmac_key_len);
572 if (ret)
573 goto e_src;
574 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
575 CCP_PASSTHRU_BYTESWAP_256BIT);
576 if (ret) {
577 cmd->engine_error = cmd_q->cmd_error;
578 goto e_src;
579 }
580 }
581
582 ret = cmd_q->ccp->vdata->perform->aes(&op);
583 if (ret) {
584 cmd->engine_error = cmd_q->cmd_error;
585 goto e_src;
586 }
587
588 ccp_process_data(&src, NULL, &op);
589 }
590
591 /* Retrieve the AES context - convert from LE to BE using
592 * 32-byte (256-bit) byteswapping
593 */
594 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
595 CCP_PASSTHRU_BYTESWAP_256BIT);
596 if (ret) {
597 cmd->engine_error = cmd_q->cmd_error;
598 goto e_src;
599 }
600
601 /* ...but we only need AES_BLOCK_SIZE bytes */
602 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
603 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
604
605 e_src:
606 ccp_free_data(&src, cmd_q);
607
608 e_ctx:
609 ccp_dm_free(&ctx);
610
611 e_key:
612 ccp_dm_free(&key);
613
614 return ret;
615 }
616
617 static noinline_for_stack int
618 ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
619 {
620 struct ccp_aes_engine *aes = &cmd->u.aes;
621 struct ccp_dm_workarea key, ctx, final_wa, tag;
622 struct ccp_data src, dst;
623 struct ccp_data aad;
624 struct ccp_op op;
625
626 unsigned long long *final;
627 unsigned int dm_offset;
628 unsigned int authsize;
629 unsigned int jobid;
630 unsigned int ilen;
631 bool in_place = true; /* Default value */
632 int ret;
633
634 struct scatterlist *p_inp, sg_inp[2];
635 struct scatterlist *p_tag, sg_tag[2];
636 struct scatterlist *p_outp, sg_outp[2];
637 struct scatterlist *p_aad;
638
639 if (!aes->iv)
640 return -EINVAL;
641
642 if (!((aes->key_len == AES_KEYSIZE_128) ||
643 (aes->key_len == AES_KEYSIZE_192) ||
644 (aes->key_len == AES_KEYSIZE_256)))
645 return -EINVAL;
646
647 if (!aes->key) /* Gotta have a key SGL */
648 return -EINVAL;
649
650 /* Zero defaults to 16 bytes, the maximum size */
651 authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
652 switch (authsize) {
653 case 16:
654 case 15:
655 case 14:
656 case 13:
657 case 12:
658 case 8:
659 case 4:
660 break;
661 default:
662 return -EINVAL;
663 }
664
665 /* First, decompose the source buffer into AAD & PT,
666 * and the destination buffer into AAD, CT & tag, or
667 * the input into CT & tag.
668 * It is expected that the input and output SGs will
669 * be valid, even if the AAD and input lengths are 0.
670 */
671 p_aad = aes->src;
672 p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);
673 p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);
674 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
675 ilen = aes->src_len;
676 p_tag = scatterwalk_ffwd(sg_tag, p_outp, ilen);
677 } else {
678 /* Input length for decryption includes tag */
679 ilen = aes->src_len - authsize;
680 p_tag = scatterwalk_ffwd(sg_tag, p_inp, ilen);
681 }
682
683 jobid = CCP_NEW_JOBID(cmd_q->ccp);
684
685 memset(&op, 0, sizeof(op));
686 op.cmd_q = cmd_q;
687 op.jobid = jobid;
688 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
689 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
690 op.init = 1;
691 op.u.aes.type = aes->type;
692
693 /* Copy the key to the LSB */
694 ret = ccp_init_dm_workarea(&key, cmd_q,
695 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
696 DMA_TO_DEVICE);
697 if (ret)
698 return ret;
699
700 dm_offset = CCP_SB_BYTES - aes->key_len;
701 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
702 if (ret)
703 goto e_key;
704 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
705 CCP_PASSTHRU_BYTESWAP_256BIT);
706 if (ret) {
707 cmd->engine_error = cmd_q->cmd_error;
708 goto e_key;
709 }
710
711 /* Copy the context (IV) to the LSB.
712 * There is an assumption here that the IV is 96 bits in length, plus
713 * a nonce of 32 bits. If no IV is present, use a zeroed buffer.
714 */
715 ret = ccp_init_dm_workarea(&ctx, cmd_q,
716 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
717 DMA_BIDIRECTIONAL);
718 if (ret)
719 goto e_key;
720
721 dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
722 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
723 if (ret)
724 goto e_ctx;
725
726 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
727 CCP_PASSTHRU_BYTESWAP_256BIT);
728 if (ret) {
729 cmd->engine_error = cmd_q->cmd_error;
730 goto e_ctx;
731 }
732
733 op.init = 1;
734 if (aes->aad_len > 0) {
735 /* Step 1: Run a GHASH over the Additional Authenticated Data */
736 ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,
737 AES_BLOCK_SIZE,
738 DMA_TO_DEVICE);
739 if (ret)
740 goto e_ctx;
741
742 op.u.aes.mode = CCP_AES_MODE_GHASH;
743 op.u.aes.action = CCP_AES_GHASHAAD;
744
745 while (aad.sg_wa.bytes_left) {
746 ccp_prepare_data(&aad, NULL, &op, AES_BLOCK_SIZE, true);
747
748 ret = cmd_q->ccp->vdata->perform->aes(&op);
749 if (ret) {
750 cmd->engine_error = cmd_q->cmd_error;
751 goto e_aad;
752 }
753
754 ccp_process_data(&aad, NULL, &op);
755 op.init = 0;
756 }
757 }
758
759 op.u.aes.mode = CCP_AES_MODE_GCTR;
760 op.u.aes.action = aes->action;
761
762 if (ilen > 0) {
763 /* Step 2: Run a GCTR over the plaintext */
764 in_place = (sg_virt(p_inp) == sg_virt(p_outp)) ? true : false;
765
766 ret = ccp_init_data(&src, cmd_q, p_inp, ilen,
767 AES_BLOCK_SIZE,
768 in_place ? DMA_BIDIRECTIONAL
769 : DMA_TO_DEVICE);
770 if (ret)
771 goto e_ctx;
772
773 if (in_place) {
774 dst = src;
775 } else {
776 ret = ccp_init_data(&dst, cmd_q, p_outp, ilen,
777 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
778 if (ret)
779 goto e_src;
780 }
781
782 op.soc = 0;
783 op.eom = 0;
784 op.init = 1;
785 while (src.sg_wa.bytes_left) {
786 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
787 if (!src.sg_wa.bytes_left) {
788 unsigned int nbytes = ilen % AES_BLOCK_SIZE;
789
790 if (nbytes) {
791 op.eom = 1;
792 op.u.aes.size = (nbytes * 8) - 1;
793 }
794 }
795
796 ret = cmd_q->ccp->vdata->perform->aes(&op);
797 if (ret) {
798 cmd->engine_error = cmd_q->cmd_error;
799 goto e_dst;
800 }
801
802 ccp_process_data(&src, &dst, &op);
803 op.init = 0;
804 }
805 }
806
807 /* Step 3: Update the IV portion of the context with the original IV */
808 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
809 CCP_PASSTHRU_BYTESWAP_256BIT);
810 if (ret) {
811 cmd->engine_error = cmd_q->cmd_error;
812 goto e_dst;
813 }
814
815 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
816 if (ret)
817 goto e_dst;
818
819 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
820 CCP_PASSTHRU_BYTESWAP_256BIT);
821 if (ret) {
822 cmd->engine_error = cmd_q->cmd_error;
823 goto e_dst;
824 }
825
826 /* Step 4: Concatenate the lengths of the AAD and source, and
827 * hash that 16 byte buffer.
828 */
829 ret = ccp_init_dm_workarea(&final_wa, cmd_q, AES_BLOCK_SIZE,
830 DMA_BIDIRECTIONAL);
831 if (ret)
832 goto e_dst;
833 final = (unsigned long long *) final_wa.address;
834 final[0] = cpu_to_be64(aes->aad_len * 8);
835 final[1] = cpu_to_be64(ilen * 8);
836
837 memset(&op, 0, sizeof(op));
838 op.cmd_q = cmd_q;
839 op.jobid = jobid;
840 op.sb_key = cmd_q->sb_key; /* Pre-allocated */
841 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
842 op.init = 1;
843 op.u.aes.type = aes->type;
844 op.u.aes.mode = CCP_AES_MODE_GHASH;
845 op.u.aes.action = CCP_AES_GHASHFINAL;
846 op.src.type = CCP_MEMTYPE_SYSTEM;
847 op.src.u.dma.address = final_wa.dma.address;
848 op.src.u.dma.length = AES_BLOCK_SIZE;
849 op.dst.type = CCP_MEMTYPE_SYSTEM;
850 op.dst.u.dma.address = final_wa.dma.address;
851 op.dst.u.dma.length = AES_BLOCK_SIZE;
852 op.eom = 1;
853 op.u.aes.size = 0;
854 ret = cmd_q->ccp->vdata->perform->aes(&op);
855 if (ret)
856 goto e_dst;
857
858 if (aes->action == CCP_AES_ACTION_ENCRYPT) {
859 /* Put the ciphered tag after the ciphertext. */
860 ccp_get_dm_area(&final_wa, 0, p_tag, 0, authsize);
861 } else {
862 /* Does this ciphered tag match the input? */
863 ret = ccp_init_dm_workarea(&tag, cmd_q, authsize,
864 DMA_BIDIRECTIONAL);
865 if (ret)
866 goto e_tag;
867 ret = ccp_set_dm_area(&tag, 0, p_tag, 0, authsize);
868 if (ret)
869 goto e_tag;
870
871 ret = crypto_memneq(tag.address, final_wa.address,
872 authsize) ? -EBADMSG : 0;
873 ccp_dm_free(&tag);
874 }
875
876 e_tag:
877 ccp_dm_free(&final_wa);
878
879 e_dst:
880 if (ilen > 0 && !in_place)
881 ccp_free_data(&dst, cmd_q);
882
883 e_src:
884 if (ilen > 0)
885 ccp_free_data(&src, cmd_q);
886
887 e_aad:
888 if (aes->aad_len)
889 ccp_free_data(&aad, cmd_q);
890
891 e_ctx:
892 ccp_dm_free(&ctx);
893
894 e_key:
895 ccp_dm_free(&key);
896
897 return ret;
898 }
899
900 static noinline_for_stack int
901 ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
902 {
903 struct ccp_aes_engine *aes = &cmd->u.aes;
904 struct ccp_dm_workarea key, ctx;
905 struct ccp_data src, dst;
906 struct ccp_op op;
907 unsigned int dm_offset;
908 bool in_place = false;
909 int ret;
910
911 if (!((aes->key_len == AES_KEYSIZE_128) ||
912 (aes->key_len == AES_KEYSIZE_192) ||
913 (aes->key_len == AES_KEYSIZE_256)))
914 return -EINVAL;
915
916 if (((aes->mode == CCP_AES_MODE_ECB) ||
917 (aes->mode == CCP_AES_MODE_CBC) ||
918 (aes->mode == CCP_AES_MODE_CFB)) &&
919 (aes->src_len & (AES_BLOCK_SIZE - 1)))
920 return -EINVAL;
921
922 if (!aes->key || !aes->src || !aes->dst)
923 return -EINVAL;
924
925 if (aes->mode != CCP_AES_MODE_ECB) {
926 if (aes->iv_len != AES_BLOCK_SIZE)
927 return -EINVAL;
928
929 if (!aes->iv)
930 return -EINVAL;
931 }
932
933 BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
934 BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
935
936 ret = -EIO;
937 memset(&op, 0, sizeof(op));
938 op.cmd_q = cmd_q;
939 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
940 op.sb_key = cmd_q->sb_key;
941 op.sb_ctx = cmd_q->sb_ctx;
942 op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
943 op.u.aes.type = aes->type;
944 op.u.aes.mode = aes->mode;
945 op.u.aes.action = aes->action;
946
947 /* All supported key sizes fit in a single (32-byte) SB entry
948 * and must be in little endian format. Use the 256-bit byte
949 * swap passthru option to convert from big endian to little
950 * endian.
951 */
952 ret = ccp_init_dm_workarea(&key, cmd_q,
953 CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
954 DMA_TO_DEVICE);
955 if (ret)
956 return ret;
957
958 dm_offset = CCP_SB_BYTES - aes->key_len;
959 ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);
960 if (ret)
961 goto e_key;
962 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
963 CCP_PASSTHRU_BYTESWAP_256BIT);
964 if (ret) {
965 cmd->engine_error = cmd_q->cmd_error;
966 goto e_key;
967 }
968
969 /* The AES context fits in a single (32-byte) SB entry and
970 * must be in little endian format. Use the 256-bit byte swap
971 * passthru option to convert from big endian to little endian.
972 */
973 ret = ccp_init_dm_workarea(&ctx, cmd_q,
974 CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
975 DMA_BIDIRECTIONAL);
976 if (ret)
977 goto e_key;
978
979 if (aes->mode != CCP_AES_MODE_ECB) {
980 /* Load the AES context - convert to LE */
981 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
982 ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
983 if (ret)
984 goto e_ctx;
985 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
986 CCP_PASSTHRU_BYTESWAP_256BIT);
987 if (ret) {
988 cmd->engine_error = cmd_q->cmd_error;
989 goto e_ctx;
990 }
991 }
992 switch (aes->mode) {
993 case CCP_AES_MODE_CFB: /* CFB128 only */
994 case CCP_AES_MODE_CTR:
995 op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
996 break;
997 default:
998 op.u.aes.size = 0;
999 }
1000
1001 /* Prepare the input and output data workareas. For in-place
1002 * operations we need to set the dma direction to BIDIRECTIONAL
1003 * and copy the src workarea to the dst workarea.
1004 */
1005 if (sg_virt(aes->src) == sg_virt(aes->dst))
1006 in_place = true;
1007
1008 ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,
1009 AES_BLOCK_SIZE,
1010 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1011 if (ret)
1012 goto e_ctx;
1013
1014 if (in_place) {
1015 dst = src;
1016 } else {
1017 ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,
1018 AES_BLOCK_SIZE, DMA_FROM_DEVICE);
1019 if (ret)
1020 goto e_src;
1021 }
1022
1023 /* Send data to the CCP AES engine */
1024 while (src.sg_wa.bytes_left) {
1025 ccp_prepare_data(&src, &dst, &op, AES_BLOCK_SIZE, true);
1026 if (!src.sg_wa.bytes_left) {
1027 op.eom = 1;
1028
1029 /* Since we don't retrieve the AES context in ECB
1030 * mode we have to wait for the operation to complete
1031 * on the last piece of data
1032 */
1033 if (aes->mode == CCP_AES_MODE_ECB)
1034 op.soc = 1;
1035 }
1036
1037 ret = cmd_q->ccp->vdata->perform->aes(&op);
1038 if (ret) {
1039 cmd->engine_error = cmd_q->cmd_error;
1040 goto e_dst;
1041 }
1042
1043 ccp_process_data(&src, &dst, &op);
1044 }
1045
1046 if (aes->mode != CCP_AES_MODE_ECB) {
1047 /* Retrieve the AES context - convert from LE to BE using
1048 * 32-byte (256-bit) byteswapping
1049 */
1050 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1051 CCP_PASSTHRU_BYTESWAP_256BIT);
1052 if (ret) {
1053 cmd->engine_error = cmd_q->cmd_error;
1054 goto e_dst;
1055 }
1056
1057 /* ...but we only need AES_BLOCK_SIZE bytes */
1058 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1059 ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);
1060 }
1061
1062 e_dst:
1063 if (!in_place)
1064 ccp_free_data(&dst, cmd_q);
1065
1066 e_src:
1067 ccp_free_data(&src, cmd_q);
1068
1069 e_ctx:
1070 ccp_dm_free(&ctx);
1071
1072 e_key:
1073 ccp_dm_free(&key);
1074
1075 return ret;
1076 }
1077
1078 static noinline_for_stack int
1079 ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1080 {
1081 struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1082 struct ccp_dm_workarea key, ctx;
1083 struct ccp_data src, dst;
1084 struct ccp_op op;
1085 unsigned int unit_size, dm_offset;
1086 bool in_place = false;
1087 unsigned int sb_count;
1088 enum ccp_aes_type aestype;
1089 int ret;
1090
1091 switch (xts->unit_size) {
1092 case CCP_XTS_AES_UNIT_SIZE_16:
1093 unit_size = 16;
1094 break;
1095 case CCP_XTS_AES_UNIT_SIZE_512:
1096 unit_size = 512;
1097 break;
1098 case CCP_XTS_AES_UNIT_SIZE_1024:
1099 unit_size = 1024;
1100 break;
1101 case CCP_XTS_AES_UNIT_SIZE_2048:
1102 unit_size = 2048;
1103 break;
1104 case CCP_XTS_AES_UNIT_SIZE_4096:
1105 unit_size = 4096;
1106 break;
1107
1108 default:
1109 return -EINVAL;
1110 }
1111
1112 if (xts->key_len == AES_KEYSIZE_128)
1113 aestype = CCP_AES_TYPE_128;
1114 else if (xts->key_len == AES_KEYSIZE_256)
1115 aestype = CCP_AES_TYPE_256;
1116 else
1117 return -EINVAL;
1118
1119 if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1120 return -EINVAL;
1121
1122 if (xts->iv_len != AES_BLOCK_SIZE)
1123 return -EINVAL;
1124
1125 if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1126 return -EINVAL;
1127
1128 BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1129 BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1130
1131 ret = -EIO;
1132 memset(&op, 0, sizeof(op));
1133 op.cmd_q = cmd_q;
1134 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1135 op.sb_key = cmd_q->sb_key;
1136 op.sb_ctx = cmd_q->sb_ctx;
1137 op.init = 1;
1138 op.u.xts.type = aestype;
1139 op.u.xts.action = xts->action;
1140 op.u.xts.unit_size = xts->unit_size;
1141
1142 /* A version 3 device only supports 128-bit keys, which fits into a
1143 * single SB entry. A version 5 device uses a 512-bit vector, so two
1144 * SB entries.
1145 */
1146 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1147 sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1148 else
1149 sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1150 ret = ccp_init_dm_workarea(&key, cmd_q,
1151 sb_count * CCP_SB_BYTES,
1152 DMA_TO_DEVICE);
1153 if (ret)
1154 return ret;
1155
1156 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1157 /* All supported key sizes must be in little endian format.
1158 * Use the 256-bit byte swap passthru option to convert from
1159 * big endian to little endian.
1160 */
1161 dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1162 ret = ccp_set_dm_area(&key, dm_offset, xts->key, 0, xts->key_len);
1163 if (ret)
1164 goto e_key;
1165 ret = ccp_set_dm_area(&key, 0, xts->key, xts->key_len, xts->key_len);
1166 if (ret)
1167 goto e_key;
1168 } else {
1169 /* Version 5 CCPs use a 512-bit space for the key: each portion
1170 * occupies 256 bits, or one entire slot, and is zero-padded.
1171 */
1172 unsigned int pad;
1173
1174 dm_offset = CCP_SB_BYTES;
1175 pad = dm_offset - xts->key_len;
1176 ret = ccp_set_dm_area(&key, pad, xts->key, 0, xts->key_len);
1177 if (ret)
1178 goto e_key;
1179 ret = ccp_set_dm_area(&key, dm_offset + pad, xts->key,
1180 xts->key_len, xts->key_len);
1181 if (ret)
1182 goto e_key;
1183 }
1184 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1185 CCP_PASSTHRU_BYTESWAP_256BIT);
1186 if (ret) {
1187 cmd->engine_error = cmd_q->cmd_error;
1188 goto e_key;
1189 }
1190
1191 /* The AES context fits in a single (32-byte) SB entry and
1192 * for XTS is already in little endian format so no byte swapping
1193 * is needed.
1194 */
1195 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1196 CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1197 DMA_BIDIRECTIONAL);
1198 if (ret)
1199 goto e_key;
1200
1201 ret = ccp_set_dm_area(&ctx, 0, xts->iv, 0, xts->iv_len);
1202 if (ret)
1203 goto e_ctx;
1204 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1205 CCP_PASSTHRU_BYTESWAP_NOOP);
1206 if (ret) {
1207 cmd->engine_error = cmd_q->cmd_error;
1208 goto e_ctx;
1209 }
1210
1211 /* Prepare the input and output data workareas. For in-place
1212 * operations we need to set the dma direction to BIDIRECTIONAL
1213 * and copy the src workarea to the dst workarea.
1214 */
1215 if (sg_virt(xts->src) == sg_virt(xts->dst))
1216 in_place = true;
1217
1218 ret = ccp_init_data(&src, cmd_q, xts->src, xts->src_len,
1219 unit_size,
1220 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1221 if (ret)
1222 goto e_ctx;
1223
1224 if (in_place) {
1225 dst = src;
1226 } else {
1227 ret = ccp_init_data(&dst, cmd_q, xts->dst, xts->src_len,
1228 unit_size, DMA_FROM_DEVICE);
1229 if (ret)
1230 goto e_src;
1231 }
1232
1233 /* Send data to the CCP AES engine */
1234 while (src.sg_wa.bytes_left) {
1235 ccp_prepare_data(&src, &dst, &op, unit_size, true);
1236 if (!src.sg_wa.bytes_left)
1237 op.eom = 1;
1238
1239 ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1240 if (ret) {
1241 cmd->engine_error = cmd_q->cmd_error;
1242 goto e_dst;
1243 }
1244
1245 ccp_process_data(&src, &dst, &op);
1246 }
1247
1248 /* Retrieve the AES context - convert from LE to BE using
1249 * 32-byte (256-bit) byteswapping
1250 */
1251 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1252 CCP_PASSTHRU_BYTESWAP_256BIT);
1253 if (ret) {
1254 cmd->engine_error = cmd_q->cmd_error;
1255 goto e_dst;
1256 }
1257
1258 /* ...but we only need AES_BLOCK_SIZE bytes */
1259 dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1260 ccp_get_dm_area(&ctx, dm_offset, xts->iv, 0, xts->iv_len);
1261
1262 e_dst:
1263 if (!in_place)
1264 ccp_free_data(&dst, cmd_q);
1265
1266 e_src:
1267 ccp_free_data(&src, cmd_q);
1268
1269 e_ctx:
1270 ccp_dm_free(&ctx);
1271
1272 e_key:
1273 ccp_dm_free(&key);
1274
1275 return ret;
1276 }
1277
1278 static noinline_for_stack int
1279 ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1280 {
1281 struct ccp_des3_engine *des3 = &cmd->u.des3;
1282
1283 struct ccp_dm_workarea key, ctx;
1284 struct ccp_data src, dst;
1285 struct ccp_op op;
1286 unsigned int dm_offset;
1287 unsigned int len_singlekey;
1288 bool in_place = false;
1289 int ret;
1290
1291 /* Error checks */
1292 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1293 return -EINVAL;
1294
1295 if (!cmd_q->ccp->vdata->perform->des3)
1296 return -EINVAL;
1297
1298 if (des3->key_len != DES3_EDE_KEY_SIZE)
1299 return -EINVAL;
1300
1301 if (((des3->mode == CCP_DES3_MODE_ECB) ||
1302 (des3->mode == CCP_DES3_MODE_CBC)) &&
1303 (des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1304 return -EINVAL;
1305
1306 if (!des3->key || !des3->src || !des3->dst)
1307 return -EINVAL;
1308
1309 if (des3->mode != CCP_DES3_MODE_ECB) {
1310 if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1311 return -EINVAL;
1312
1313 if (!des3->iv)
1314 return -EINVAL;
1315 }
1316
1317 ret = -EIO;
1318 /* Zero out all the fields of the command desc */
1319 memset(&op, 0, sizeof(op));
1320
1321 /* Set up the Function field */
1322 op.cmd_q = cmd_q;
1323 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1324 op.sb_key = cmd_q->sb_key;
1325
1326 op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1327 op.u.des3.type = des3->type;
1328 op.u.des3.mode = des3->mode;
1329 op.u.des3.action = des3->action;
1330
1331 /*
1332 * All supported key sizes fit in a single (32-byte) KSB entry and
1333 * (like AES) must be in little endian format. Use the 256-bit byte
1334 * swap passthru option to convert from big endian to little endian.
1335 */
1336 ret = ccp_init_dm_workarea(&key, cmd_q,
1337 CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1338 DMA_TO_DEVICE);
1339 if (ret)
1340 return ret;
1341
1342 /*
1343 * The contents of the key triplet are in the reverse order of what
1344 * is required by the engine. Copy the 3 pieces individually to put
1345 * them where they belong.
1346 */
1347 dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1348
1349 len_singlekey = des3->key_len / 3;
1350 ret = ccp_set_dm_area(&key, dm_offset + 2 * len_singlekey,
1351 des3->key, 0, len_singlekey);
1352 if (ret)
1353 goto e_key;
1354 ret = ccp_set_dm_area(&key, dm_offset + len_singlekey,
1355 des3->key, len_singlekey, len_singlekey);
1356 if (ret)
1357 goto e_key;
1358 ret = ccp_set_dm_area(&key, dm_offset,
1359 des3->key, 2 * len_singlekey, len_singlekey);
1360 if (ret)
1361 goto e_key;
1362
1363 /* Copy the key to the SB */
1364 ret = ccp_copy_to_sb(cmd_q, &key, op.jobid, op.sb_key,
1365 CCP_PASSTHRU_BYTESWAP_256BIT);
1366 if (ret) {
1367 cmd->engine_error = cmd_q->cmd_error;
1368 goto e_key;
1369 }
1370
1371 /*
1372 * The DES3 context fits in a single (32-byte) KSB entry and
1373 * must be in little endian format. Use the 256-bit byte swap
1374 * passthru option to convert from big endian to little endian.
1375 */
1376 if (des3->mode != CCP_DES3_MODE_ECB) {
1377 op.sb_ctx = cmd_q->sb_ctx;
1378
1379 ret = ccp_init_dm_workarea(&ctx, cmd_q,
1380 CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1381 DMA_BIDIRECTIONAL);
1382 if (ret)
1383 goto e_key;
1384
1385 /* Load the context into the LSB */
1386 dm_offset = CCP_SB_BYTES - des3->iv_len;
1387 ret = ccp_set_dm_area(&ctx, dm_offset, des3->iv, 0,
1388 des3->iv_len);
1389 if (ret)
1390 goto e_ctx;
1391
1392 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1393 CCP_PASSTHRU_BYTESWAP_256BIT);
1394 if (ret) {
1395 cmd->engine_error = cmd_q->cmd_error;
1396 goto e_ctx;
1397 }
1398 }
1399
1400 /*
1401 * Prepare the input and output data workareas. For in-place
1402 * operations we need to set the dma direction to BIDIRECTIONAL
1403 * and copy the src workarea to the dst workarea.
1404 */
1405 if (sg_virt(des3->src) == sg_virt(des3->dst))
1406 in_place = true;
1407
1408 ret = ccp_init_data(&src, cmd_q, des3->src, des3->src_len,
1409 DES3_EDE_BLOCK_SIZE,
1410 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1411 if (ret)
1412 goto e_ctx;
1413
1414 if (in_place)
1415 dst = src;
1416 else {
1417 ret = ccp_init_data(&dst, cmd_q, des3->dst, des3->src_len,
1418 DES3_EDE_BLOCK_SIZE, DMA_FROM_DEVICE);
1419 if (ret)
1420 goto e_src;
1421 }
1422
1423 /* Send data to the CCP DES3 engine */
1424 while (src.sg_wa.bytes_left) {
1425 ccp_prepare_data(&src, &dst, &op, DES3_EDE_BLOCK_SIZE, true);
1426 if (!src.sg_wa.bytes_left) {
1427 op.eom = 1;
1428
1429 /* Since we don't retrieve the context in ECB mode
1430 * we have to wait for the operation to complete
1431 * on the last piece of data
1432 */
1433 op.soc = 0;
1434 }
1435
1436 ret = cmd_q->ccp->vdata->perform->des3(&op);
1437 if (ret) {
1438 cmd->engine_error = cmd_q->cmd_error;
1439 goto e_dst;
1440 }
1441
1442 ccp_process_data(&src, &dst, &op);
1443 }
1444
1445 if (des3->mode != CCP_DES3_MODE_ECB) {
1446 /* Retrieve the context and make BE */
1447 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1448 CCP_PASSTHRU_BYTESWAP_256BIT);
1449 if (ret) {
1450 cmd->engine_error = cmd_q->cmd_error;
1451 goto e_dst;
1452 }
1453
1454 /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1455 ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0,
1456 DES3_EDE_BLOCK_SIZE);
1457 }
1458 e_dst:
1459 if (!in_place)
1460 ccp_free_data(&dst, cmd_q);
1461
1462 e_src:
1463 ccp_free_data(&src, cmd_q);
1464
1465 e_ctx:
1466 if (des3->mode != CCP_DES3_MODE_ECB)
1467 ccp_dm_free(&ctx);
1468
1469 e_key:
1470 ccp_dm_free(&key);
1471
1472 return ret;
1473 }
1474
1475 static noinline_for_stack int
1476 ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1477 {
1478 struct ccp_sha_engine *sha = &cmd->u.sha;
1479 struct ccp_dm_workarea ctx;
1480 struct ccp_data src;
1481 struct ccp_op op;
1482 unsigned int ioffset, ooffset;
1483 unsigned int digest_size;
1484 int sb_count;
1485 const void *init;
1486 u64 block_size;
1487 int ctx_size;
1488 int ret;
1489
1490 switch (sha->type) {
1491 case CCP_SHA_TYPE_1:
1492 if (sha->ctx_len < SHA1_DIGEST_SIZE)
1493 return -EINVAL;
1494 block_size = SHA1_BLOCK_SIZE;
1495 break;
1496 case CCP_SHA_TYPE_224:
1497 if (sha->ctx_len < SHA224_DIGEST_SIZE)
1498 return -EINVAL;
1499 block_size = SHA224_BLOCK_SIZE;
1500 break;
1501 case CCP_SHA_TYPE_256:
1502 if (sha->ctx_len < SHA256_DIGEST_SIZE)
1503 return -EINVAL;
1504 block_size = SHA256_BLOCK_SIZE;
1505 break;
1506 case CCP_SHA_TYPE_384:
1507 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1508 || sha->ctx_len < SHA384_DIGEST_SIZE)
1509 return -EINVAL;
1510 block_size = SHA384_BLOCK_SIZE;
1511 break;
1512 case CCP_SHA_TYPE_512:
1513 if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1514 || sha->ctx_len < SHA512_DIGEST_SIZE)
1515 return -EINVAL;
1516 block_size = SHA512_BLOCK_SIZE;
1517 break;
1518 default:
1519 return -EINVAL;
1520 }
1521
1522 if (!sha->ctx)
1523 return -EINVAL;
1524
1525 if (!sha->final && (sha->src_len & (block_size - 1)))
1526 return -EINVAL;
1527
1528 /* The version 3 device can't handle zero-length input */
1529 if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1530
1531 if (!sha->src_len) {
1532 unsigned int digest_len;
1533 const u8 *sha_zero;
1534
1535 /* Not final, just return */
1536 if (!sha->final)
1537 return 0;
1538
1539 /* CCP can't do a zero length sha operation so the
1540 * caller must buffer the data.
1541 */
1542 if (sha->msg_bits)
1543 return -EINVAL;
1544
1545 /* The CCP cannot perform zero-length sha operations
1546 * so the caller is required to buffer data for the
1547 * final operation. However, a sha operation for a
1548 * message with a total length of zero is valid so
1549 * known values are required to supply the result.
1550 */
1551 switch (sha->type) {
1552 case CCP_SHA_TYPE_1:
1553 sha_zero = sha1_zero_message_hash;
1554 digest_len = SHA1_DIGEST_SIZE;
1555 break;
1556 case CCP_SHA_TYPE_224:
1557 sha_zero = sha224_zero_message_hash;
1558 digest_len = SHA224_DIGEST_SIZE;
1559 break;
1560 case CCP_SHA_TYPE_256:
1561 sha_zero = sha256_zero_message_hash;
1562 digest_len = SHA256_DIGEST_SIZE;
1563 break;
1564 default:
1565 return -EINVAL;
1566 }
1567
1568 scatterwalk_map_and_copy((void *)sha_zero, sha->ctx, 0,
1569 digest_len, 1);
1570
1571 return 0;
1572 }
1573 }
1574
1575 /* Set variables used throughout */
1576 switch (sha->type) {
1577 case CCP_SHA_TYPE_1:
1578 digest_size = SHA1_DIGEST_SIZE;
1579 init = (void *) ccp_sha1_init;
1580 ctx_size = SHA1_DIGEST_SIZE;
1581 sb_count = 1;
1582 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1583 ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1584 else
1585 ooffset = ioffset = 0;
1586 break;
1587 case CCP_SHA_TYPE_224:
1588 digest_size = SHA224_DIGEST_SIZE;
1589 init = (void *) ccp_sha224_init;
1590 ctx_size = SHA256_DIGEST_SIZE;
1591 sb_count = 1;
1592 ioffset = 0;
1593 if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1594 ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1595 else
1596 ooffset = 0;
1597 break;
1598 case CCP_SHA_TYPE_256:
1599 digest_size = SHA256_DIGEST_SIZE;
1600 init = (void *) ccp_sha256_init;
1601 ctx_size = SHA256_DIGEST_SIZE;
1602 sb_count = 1;
1603 ooffset = ioffset = 0;
1604 break;
1605 case CCP_SHA_TYPE_384:
1606 digest_size = SHA384_DIGEST_SIZE;
1607 init = (void *) ccp_sha384_init;
1608 ctx_size = SHA512_DIGEST_SIZE;
1609 sb_count = 2;
1610 ioffset = 0;
1611 ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1612 break;
1613 case CCP_SHA_TYPE_512:
1614 digest_size = SHA512_DIGEST_SIZE;
1615 init = (void *) ccp_sha512_init;
1616 ctx_size = SHA512_DIGEST_SIZE;
1617 sb_count = 2;
1618 ooffset = ioffset = 0;
1619 break;
1620 default:
1621 ret = -EINVAL;
1622 goto e_data;
1623 }
1624
1625 /* For zero-length plaintext the src pointer is ignored;
1626 * otherwise both parts must be valid
1627 */
1628 if (sha->src_len && !sha->src)
1629 return -EINVAL;
1630
1631 memset(&op, 0, sizeof(op));
1632 op.cmd_q = cmd_q;
1633 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1634 op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1635 op.u.sha.type = sha->type;
1636 op.u.sha.msg_bits = sha->msg_bits;
1637
1638 /* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1639 * SHA384/512 require 2 adjacent SB slots, with the right half in the
1640 * first slot, and the left half in the second. Each portion must then
1641 * be in little endian format: use the 256-bit byte swap option.
1642 */
1643 ret = ccp_init_dm_workarea(&ctx, cmd_q, sb_count * CCP_SB_BYTES,
1644 DMA_BIDIRECTIONAL);
1645 if (ret)
1646 return ret;
1647 if (sha->first) {
1648 switch (sha->type) {
1649 case CCP_SHA_TYPE_1:
1650 case CCP_SHA_TYPE_224:
1651 case CCP_SHA_TYPE_256:
1652 memcpy(ctx.address + ioffset, init, ctx_size);
1653 break;
1654 case CCP_SHA_TYPE_384:
1655 case CCP_SHA_TYPE_512:
1656 memcpy(ctx.address + ctx_size / 2, init,
1657 ctx_size / 2);
1658 memcpy(ctx.address, init + ctx_size / 2,
1659 ctx_size / 2);
1660 break;
1661 default:
1662 ret = -EINVAL;
1663 goto e_ctx;
1664 }
1665 } else {
1666 /* Restore the context */
1667 ret = ccp_set_dm_area(&ctx, 0, sha->ctx, 0,
1668 sb_count * CCP_SB_BYTES);
1669 if (ret)
1670 goto e_ctx;
1671 }
1672
1673 ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1674 CCP_PASSTHRU_BYTESWAP_256BIT);
1675 if (ret) {
1676 cmd->engine_error = cmd_q->cmd_error;
1677 goto e_ctx;
1678 }
1679
1680 if (sha->src) {
1681 /* Send data to the CCP SHA engine; block_size is set above */
1682 ret = ccp_init_data(&src, cmd_q, sha->src, sha->src_len,
1683 block_size, DMA_TO_DEVICE);
1684 if (ret)
1685 goto e_ctx;
1686
1687 while (src.sg_wa.bytes_left) {
1688 ccp_prepare_data(&src, NULL, &op, block_size, false);
1689 if (sha->final && !src.sg_wa.bytes_left)
1690 op.eom = 1;
1691
1692 ret = cmd_q->ccp->vdata->perform->sha(&op);
1693 if (ret) {
1694 cmd->engine_error = cmd_q->cmd_error;
1695 goto e_data;
1696 }
1697
1698 ccp_process_data(&src, NULL, &op);
1699 }
1700 } else {
1701 op.eom = 1;
1702 ret = cmd_q->ccp->vdata->perform->sha(&op);
1703 if (ret) {
1704 cmd->engine_error = cmd_q->cmd_error;
1705 goto e_data;
1706 }
1707 }
1708
1709 /* Retrieve the SHA context - convert from LE to BE using
1710 * 32-byte (256-bit) byteswapping to BE
1711 */
1712 ret = ccp_copy_from_sb(cmd_q, &ctx, op.jobid, op.sb_ctx,
1713 CCP_PASSTHRU_BYTESWAP_256BIT);
1714 if (ret) {
1715 cmd->engine_error = cmd_q->cmd_error;
1716 goto e_data;
1717 }
1718
1719 if (sha->final) {
1720 /* Finishing up, so get the digest */
1721 switch (sha->type) {
1722 case CCP_SHA_TYPE_1:
1723 case CCP_SHA_TYPE_224:
1724 case CCP_SHA_TYPE_256:
1725 ccp_get_dm_area(&ctx, ooffset,
1726 sha->ctx, 0,
1727 digest_size);
1728 break;
1729 case CCP_SHA_TYPE_384:
1730 case CCP_SHA_TYPE_512:
1731 ccp_get_dm_area(&ctx, 0,
1732 sha->ctx, LSB_ITEM_SIZE - ooffset,
1733 LSB_ITEM_SIZE);
1734 ccp_get_dm_area(&ctx, LSB_ITEM_SIZE + ooffset,
1735 sha->ctx, 0,
1736 LSB_ITEM_SIZE - ooffset);
1737 break;
1738 default:
1739 ret = -EINVAL;
1740 goto e_ctx;
1741 }
1742 } else {
1743 /* Stash the context */
1744 ccp_get_dm_area(&ctx, 0, sha->ctx, 0,
1745 sb_count * CCP_SB_BYTES);
1746 }
1747
1748 if (sha->final && sha->opad) {
1749 /* HMAC operation, recursively perform final SHA */
1750 struct ccp_cmd hmac_cmd;
1751 struct scatterlist sg;
1752 u8 *hmac_buf;
1753
1754 if (sha->opad_len != block_size) {
1755 ret = -EINVAL;
1756 goto e_data;
1757 }
1758
1759 hmac_buf = kmalloc(block_size + digest_size, GFP_KERNEL);
1760 if (!hmac_buf) {
1761 ret = -ENOMEM;
1762 goto e_data;
1763 }
1764 sg_init_one(&sg, hmac_buf, block_size + digest_size);
1765
1766 scatterwalk_map_and_copy(hmac_buf, sha->opad, 0, block_size, 0);
1767 switch (sha->type) {
1768 case CCP_SHA_TYPE_1:
1769 case CCP_SHA_TYPE_224:
1770 case CCP_SHA_TYPE_256:
1771 memcpy(hmac_buf + block_size,
1772 ctx.address + ooffset,
1773 digest_size);
1774 break;
1775 case CCP_SHA_TYPE_384:
1776 case CCP_SHA_TYPE_512:
1777 memcpy(hmac_buf + block_size,
1778 ctx.address + LSB_ITEM_SIZE + ooffset,
1779 LSB_ITEM_SIZE);
1780 memcpy(hmac_buf + block_size +
1781 (LSB_ITEM_SIZE - ooffset),
1782 ctx.address,
1783 LSB_ITEM_SIZE);
1784 break;
1785 default:
1786 ret = -EINVAL;
1787 goto e_ctx;
1788 }
1789
1790 memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1791 hmac_cmd.engine = CCP_ENGINE_SHA;
1792 hmac_cmd.u.sha.type = sha->type;
1793 hmac_cmd.u.sha.ctx = sha->ctx;
1794 hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1795 hmac_cmd.u.sha.src = &sg;
1796 hmac_cmd.u.sha.src_len = block_size + digest_size;
1797 hmac_cmd.u.sha.opad = NULL;
1798 hmac_cmd.u.sha.opad_len = 0;
1799 hmac_cmd.u.sha.first = 1;
1800 hmac_cmd.u.sha.final = 1;
1801 hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1802
1803 ret = ccp_run_sha_cmd(cmd_q, &hmac_cmd);
1804 if (ret)
1805 cmd->engine_error = hmac_cmd.engine_error;
1806
1807 kfree(hmac_buf);
1808 }
1809
1810 e_data:
1811 if (sha->src)
1812 ccp_free_data(&src, cmd_q);
1813
1814 e_ctx:
1815 ccp_dm_free(&ctx);
1816
1817 return ret;
1818 }
1819
1820 static noinline_for_stack int
1821 ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1822 {
1823 struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1824 struct ccp_dm_workarea exp, src, dst;
1825 struct ccp_op op;
1826 unsigned int sb_count, i_len, o_len;
1827 int ret;
1828
1829 /* Check against the maximum allowable size, in bits */
1830 if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1831 return -EINVAL;
1832
1833 if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1834 return -EINVAL;
1835
1836 memset(&op, 0, sizeof(op));
1837 op.cmd_q = cmd_q;
1838 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1839
1840 /* The RSA modulus must precede the message being acted upon, so
1841 * it must be copied to a DMA area where the message and the
1842 * modulus can be concatenated. Therefore the input buffer
1843 * length required is twice the output buffer length (which
1844 * must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1845 * Buffer sizes must be a multiple of 32 bytes; rounding up may be
1846 * required.
1847 */
1848 o_len = 32 * ((rsa->key_size + 255) / 256);
1849 i_len = o_len * 2;
1850
1851 sb_count = 0;
1852 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1853 /* sb_count is the number of storage block slots required
1854 * for the modulus.
1855 */
1856 sb_count = o_len / CCP_SB_BYTES;
1857 op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1858 sb_count);
1859 if (!op.sb_key)
1860 return -EIO;
1861 } else {
1862 /* A version 5 device allows a modulus size that will not fit
1863 * in the LSB, so the command will transfer it from memory.
1864 * Set the sb key to the default, even though it's not used.
1865 */
1866 op.sb_key = cmd_q->sb_key;
1867 }
1868
1869 /* The RSA exponent must be in little endian format. Reverse its
1870 * byte order.
1871 */
1872 ret = ccp_init_dm_workarea(&exp, cmd_q, o_len, DMA_TO_DEVICE);
1873 if (ret)
1874 goto e_sb;
1875
1876 ret = ccp_reverse_set_dm_area(&exp, 0, rsa->exp, 0, rsa->exp_len);
1877 if (ret)
1878 goto e_exp;
1879
1880 if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1881 /* Copy the exponent to the local storage block, using
1882 * as many 32-byte blocks as were allocated above. It's
1883 * already little endian, so no further change is required.
1884 */
1885 ret = ccp_copy_to_sb(cmd_q, &exp, op.jobid, op.sb_key,
1886 CCP_PASSTHRU_BYTESWAP_NOOP);
1887 if (ret) {
1888 cmd->engine_error = cmd_q->cmd_error;
1889 goto e_exp;
1890 }
1891 } else {
1892 /* The exponent can be retrieved from memory via DMA. */
1893 op.exp.u.dma.address = exp.dma.address;
1894 op.exp.u.dma.offset = 0;
1895 }
1896
1897 /* Concatenate the modulus and the message. Both the modulus and
1898 * the operands must be in little endian format. Since the input
1899 * is in big endian format it must be converted.
1900 */
1901 ret = ccp_init_dm_workarea(&src, cmd_q, i_len, DMA_TO_DEVICE);
1902 if (ret)
1903 goto e_exp;
1904
1905 ret = ccp_reverse_set_dm_area(&src, 0, rsa->mod, 0, rsa->mod_len);
1906 if (ret)
1907 goto e_src;
1908 ret = ccp_reverse_set_dm_area(&src, o_len, rsa->src, 0, rsa->src_len);
1909 if (ret)
1910 goto e_src;
1911
1912 /* Prepare the output area for the operation */
1913 ret = ccp_init_dm_workarea(&dst, cmd_q, o_len, DMA_FROM_DEVICE);
1914 if (ret)
1915 goto e_src;
1916
1917 op.soc = 1;
1918 op.src.u.dma.address = src.dma.address;
1919 op.src.u.dma.offset = 0;
1920 op.src.u.dma.length = i_len;
1921 op.dst.u.dma.address = dst.dma.address;
1922 op.dst.u.dma.offset = 0;
1923 op.dst.u.dma.length = o_len;
1924
1925 op.u.rsa.mod_size = rsa->key_size;
1926 op.u.rsa.input_len = i_len;
1927
1928 ret = cmd_q->ccp->vdata->perform->rsa(&op);
1929 if (ret) {
1930 cmd->engine_error = cmd_q->cmd_error;
1931 goto e_dst;
1932 }
1933
1934 ccp_reverse_get_dm_area(&dst, 0, rsa->dst, 0, rsa->mod_len);
1935
1936 e_dst:
1937 ccp_dm_free(&dst);
1938
1939 e_src:
1940 ccp_dm_free(&src);
1941
1942 e_exp:
1943 ccp_dm_free(&exp);
1944
1945 e_sb:
1946 if (sb_count)
1947 cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1948
1949 return ret;
1950 }
1951
1952 static noinline_for_stack int
1953 ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1954 {
1955 struct ccp_passthru_engine *pt = &cmd->u.passthru;
1956 struct ccp_dm_workarea mask;
1957 struct ccp_data src, dst;
1958 struct ccp_op op;
1959 bool in_place = false;
1960 unsigned int i;
1961 int ret = 0;
1962
1963 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1964 return -EINVAL;
1965
1966 if (!pt->src || !pt->dst)
1967 return -EINVAL;
1968
1969 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1970 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1971 return -EINVAL;
1972 if (!pt->mask)
1973 return -EINVAL;
1974 }
1975
1976 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1977
1978 memset(&op, 0, sizeof(op));
1979 op.cmd_q = cmd_q;
1980 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1981
1982 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1983 /* Load the mask */
1984 op.sb_key = cmd_q->sb_key;
1985
1986 ret = ccp_init_dm_workarea(&mask, cmd_q,
1987 CCP_PASSTHRU_SB_COUNT *
1988 CCP_SB_BYTES,
1989 DMA_TO_DEVICE);
1990 if (ret)
1991 return ret;
1992
1993 ret = ccp_set_dm_area(&mask, 0, pt->mask, 0, pt->mask_len);
1994 if (ret)
1995 goto e_mask;
1996 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
1997 CCP_PASSTHRU_BYTESWAP_NOOP);
1998 if (ret) {
1999 cmd->engine_error = cmd_q->cmd_error;
2000 goto e_mask;
2001 }
2002 }
2003
2004 /* Prepare the input and output data workareas. For in-place
2005 * operations we need to set the dma direction to BIDIRECTIONAL
2006 * and copy the src workarea to the dst workarea.
2007 */
2008 if (sg_virt(pt->src) == sg_virt(pt->dst))
2009 in_place = true;
2010
2011 ret = ccp_init_data(&src, cmd_q, pt->src, pt->src_len,
2012 CCP_PASSTHRU_MASKSIZE,
2013 in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2014 if (ret)
2015 goto e_mask;
2016
2017 if (in_place) {
2018 dst = src;
2019 } else {
2020 ret = ccp_init_data(&dst, cmd_q, pt->dst, pt->src_len,
2021 CCP_PASSTHRU_MASKSIZE, DMA_FROM_DEVICE);
2022 if (ret)
2023 goto e_src;
2024 }
2025
2026 /* Send data to the CCP Passthru engine
2027 * Because the CCP engine works on a single source and destination
2028 * dma address at a time, each entry in the source scatterlist
2029 * (after the dma_map_sg call) must be less than or equal to the
2030 * (remaining) length in the destination scatterlist entry and the
2031 * length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2032 */
2033 dst.sg_wa.sg_used = 0;
2034 for (i = 1; i <= src.sg_wa.dma_count; i++) {
2035 if (!dst.sg_wa.sg ||
2036 (dst.sg_wa.sg->length < src.sg_wa.sg->length)) {
2037 ret = -EINVAL;
2038 goto e_dst;
2039 }
2040
2041 if (i == src.sg_wa.dma_count) {
2042 op.eom = 1;
2043 op.soc = 1;
2044 }
2045
2046 op.src.type = CCP_MEMTYPE_SYSTEM;
2047 op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2048 op.src.u.dma.offset = 0;
2049 op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2050
2051 op.dst.type = CCP_MEMTYPE_SYSTEM;
2052 op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2053 op.dst.u.dma.offset = dst.sg_wa.sg_used;
2054 op.dst.u.dma.length = op.src.u.dma.length;
2055
2056 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2057 if (ret) {
2058 cmd->engine_error = cmd_q->cmd_error;
2059 goto e_dst;
2060 }
2061
2062 dst.sg_wa.sg_used += src.sg_wa.sg->length;
2063 if (dst.sg_wa.sg_used == dst.sg_wa.sg->length) {
2064 dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2065 dst.sg_wa.sg_used = 0;
2066 }
2067 src.sg_wa.sg = sg_next(src.sg_wa.sg);
2068 }
2069
2070 e_dst:
2071 if (!in_place)
2072 ccp_free_data(&dst, cmd_q);
2073
2074 e_src:
2075 ccp_free_data(&src, cmd_q);
2076
2077 e_mask:
2078 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2079 ccp_dm_free(&mask);
2080
2081 return ret;
2082 }
2083
2084 static noinline_for_stack int
2085 ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2086 struct ccp_cmd *cmd)
2087 {
2088 struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2089 struct ccp_dm_workarea mask;
2090 struct ccp_op op;
2091 int ret;
2092
2093 if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2094 return -EINVAL;
2095
2096 if (!pt->src_dma || !pt->dst_dma)
2097 return -EINVAL;
2098
2099 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2100 if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2101 return -EINVAL;
2102 if (!pt->mask)
2103 return -EINVAL;
2104 }
2105
2106 BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2107
2108 memset(&op, 0, sizeof(op));
2109 op.cmd_q = cmd_q;
2110 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2111
2112 if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2113 /* Load the mask */
2114 op.sb_key = cmd_q->sb_key;
2115
2116 mask.length = pt->mask_len;
2117 mask.dma.address = pt->mask;
2118 mask.dma.length = pt->mask_len;
2119
2120 ret = ccp_copy_to_sb(cmd_q, &mask, op.jobid, op.sb_key,
2121 CCP_PASSTHRU_BYTESWAP_NOOP);
2122 if (ret) {
2123 cmd->engine_error = cmd_q->cmd_error;
2124 return ret;
2125 }
2126 }
2127
2128 /* Send data to the CCP Passthru engine */
2129 op.eom = 1;
2130 op.soc = 1;
2131
2132 op.src.type = CCP_MEMTYPE_SYSTEM;
2133 op.src.u.dma.address = pt->src_dma;
2134 op.src.u.dma.offset = 0;
2135 op.src.u.dma.length = pt->src_len;
2136
2137 op.dst.type = CCP_MEMTYPE_SYSTEM;
2138 op.dst.u.dma.address = pt->dst_dma;
2139 op.dst.u.dma.offset = 0;
2140 op.dst.u.dma.length = pt->src_len;
2141
2142 ret = cmd_q->ccp->vdata->perform->passthru(&op);
2143 if (ret)
2144 cmd->engine_error = cmd_q->cmd_error;
2145
2146 return ret;
2147 }
2148
2149 static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2150 {
2151 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2152 struct ccp_dm_workarea src, dst;
2153 struct ccp_op op;
2154 int ret;
2155 u8 *save;
2156
2157 if (!ecc->u.mm.operand_1 ||
2158 (ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2159 return -EINVAL;
2160
2161 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2162 if (!ecc->u.mm.operand_2 ||
2163 (ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2164 return -EINVAL;
2165
2166 if (!ecc->u.mm.result ||
2167 (ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2168 return -EINVAL;
2169
2170 memset(&op, 0, sizeof(op));
2171 op.cmd_q = cmd_q;
2172 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2173
2174 /* Concatenate the modulus and the operands. Both the modulus and
2175 * the operands must be in little endian format. Since the input
2176 * is in big endian format it must be converted and placed in a
2177 * fixed length buffer.
2178 */
2179 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2180 DMA_TO_DEVICE);
2181 if (ret)
2182 return ret;
2183
2184 /* Save the workarea address since it is updated in order to perform
2185 * the concatenation
2186 */
2187 save = src.address;
2188
2189 /* Copy the ECC modulus */
2190 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2191 if (ret)
2192 goto e_src;
2193 src.address += CCP_ECC_OPERAND_SIZE;
2194
2195 /* Copy the first operand */
2196 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_1, 0,
2197 ecc->u.mm.operand_1_len);
2198 if (ret)
2199 goto e_src;
2200 src.address += CCP_ECC_OPERAND_SIZE;
2201
2202 if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2203 /* Copy the second operand */
2204 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.mm.operand_2, 0,
2205 ecc->u.mm.operand_2_len);
2206 if (ret)
2207 goto e_src;
2208 src.address += CCP_ECC_OPERAND_SIZE;
2209 }
2210
2211 /* Restore the workarea address */
2212 src.address = save;
2213
2214 /* Prepare the output area for the operation */
2215 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2216 DMA_FROM_DEVICE);
2217 if (ret)
2218 goto e_src;
2219
2220 op.soc = 1;
2221 op.src.u.dma.address = src.dma.address;
2222 op.src.u.dma.offset = 0;
2223 op.src.u.dma.length = src.length;
2224 op.dst.u.dma.address = dst.dma.address;
2225 op.dst.u.dma.offset = 0;
2226 op.dst.u.dma.length = dst.length;
2227
2228 op.u.ecc.function = cmd->u.ecc.function;
2229
2230 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2231 if (ret) {
2232 cmd->engine_error = cmd_q->cmd_error;
2233 goto e_dst;
2234 }
2235
2236 ecc->ecc_result = le16_to_cpup(
2237 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2238 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2239 ret = -EIO;
2240 goto e_dst;
2241 }
2242
2243 /* Save the ECC result */
2244 ccp_reverse_get_dm_area(&dst, 0, ecc->u.mm.result, 0,
2245 CCP_ECC_MODULUS_BYTES);
2246
2247 e_dst:
2248 ccp_dm_free(&dst);
2249
2250 e_src:
2251 ccp_dm_free(&src);
2252
2253 return ret;
2254 }
2255
2256 static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2257 {
2258 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2259 struct ccp_dm_workarea src, dst;
2260 struct ccp_op op;
2261 int ret;
2262 u8 *save;
2263
2264 if (!ecc->u.pm.point_1.x ||
2265 (ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2266 !ecc->u.pm.point_1.y ||
2267 (ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2268 return -EINVAL;
2269
2270 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2271 if (!ecc->u.pm.point_2.x ||
2272 (ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2273 !ecc->u.pm.point_2.y ||
2274 (ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2275 return -EINVAL;
2276 } else {
2277 if (!ecc->u.pm.domain_a ||
2278 (ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2279 return -EINVAL;
2280
2281 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2282 if (!ecc->u.pm.scalar ||
2283 (ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2284 return -EINVAL;
2285 }
2286
2287 if (!ecc->u.pm.result.x ||
2288 (ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2289 !ecc->u.pm.result.y ||
2290 (ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2291 return -EINVAL;
2292
2293 memset(&op, 0, sizeof(op));
2294 op.cmd_q = cmd_q;
2295 op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2296
2297 /* Concatenate the modulus and the operands. Both the modulus and
2298 * the operands must be in little endian format. Since the input
2299 * is in big endian format it must be converted and placed in a
2300 * fixed length buffer.
2301 */
2302 ret = ccp_init_dm_workarea(&src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2303 DMA_TO_DEVICE);
2304 if (ret)
2305 return ret;
2306
2307 /* Save the workarea address since it is updated in order to perform
2308 * the concatenation
2309 */
2310 save = src.address;
2311
2312 /* Copy the ECC modulus */
2313 ret = ccp_reverse_set_dm_area(&src, 0, ecc->mod, 0, ecc->mod_len);
2314 if (ret)
2315 goto e_src;
2316 src.address += CCP_ECC_OPERAND_SIZE;
2317
2318 /* Copy the first point X and Y coordinate */
2319 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.x, 0,
2320 ecc->u.pm.point_1.x_len);
2321 if (ret)
2322 goto e_src;
2323 src.address += CCP_ECC_OPERAND_SIZE;
2324 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_1.y, 0,
2325 ecc->u.pm.point_1.y_len);
2326 if (ret)
2327 goto e_src;
2328 src.address += CCP_ECC_OPERAND_SIZE;
2329
2330 /* Set the first point Z coordinate to 1 */
2331 *src.address = 0x01;
2332 src.address += CCP_ECC_OPERAND_SIZE;
2333
2334 if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2335 /* Copy the second point X and Y coordinate */
2336 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.x, 0,
2337 ecc->u.pm.point_2.x_len);
2338 if (ret)
2339 goto e_src;
2340 src.address += CCP_ECC_OPERAND_SIZE;
2341 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.point_2.y, 0,
2342 ecc->u.pm.point_2.y_len);
2343 if (ret)
2344 goto e_src;
2345 src.address += CCP_ECC_OPERAND_SIZE;
2346
2347 /* Set the second point Z coordinate to 1 */
2348 *src.address = 0x01;
2349 src.address += CCP_ECC_OPERAND_SIZE;
2350 } else {
2351 /* Copy the Domain "a" parameter */
2352 ret = ccp_reverse_set_dm_area(&src, 0, ecc->u.pm.domain_a, 0,
2353 ecc->u.pm.domain_a_len);
2354 if (ret)
2355 goto e_src;
2356 src.address += CCP_ECC_OPERAND_SIZE;
2357
2358 if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2359 /* Copy the scalar value */
2360 ret = ccp_reverse_set_dm_area(&src, 0,
2361 ecc->u.pm.scalar, 0,
2362 ecc->u.pm.scalar_len);
2363 if (ret)
2364 goto e_src;
2365 src.address += CCP_ECC_OPERAND_SIZE;
2366 }
2367 }
2368
2369 /* Restore the workarea address */
2370 src.address = save;
2371
2372 /* Prepare the output area for the operation */
2373 ret = ccp_init_dm_workarea(&dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2374 DMA_FROM_DEVICE);
2375 if (ret)
2376 goto e_src;
2377
2378 op.soc = 1;
2379 op.src.u.dma.address = src.dma.address;
2380 op.src.u.dma.offset = 0;
2381 op.src.u.dma.length = src.length;
2382 op.dst.u.dma.address = dst.dma.address;
2383 op.dst.u.dma.offset = 0;
2384 op.dst.u.dma.length = dst.length;
2385
2386 op.u.ecc.function = cmd->u.ecc.function;
2387
2388 ret = cmd_q->ccp->vdata->perform->ecc(&op);
2389 if (ret) {
2390 cmd->engine_error = cmd_q->cmd_error;
2391 goto e_dst;
2392 }
2393
2394 ecc->ecc_result = le16_to_cpup(
2395 (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2396 if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2397 ret = -EIO;
2398 goto e_dst;
2399 }
2400
2401 /* Save the workarea address since it is updated as we walk through
2402 * to copy the point math result
2403 */
2404 save = dst.address;
2405
2406 /* Save the ECC result X and Y coordinates */
2407 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.x, 0,
2408 CCP_ECC_MODULUS_BYTES);
2409 dst.address += CCP_ECC_OUTPUT_SIZE;
2410 ccp_reverse_get_dm_area(&dst, 0, ecc->u.pm.result.y, 0,
2411 CCP_ECC_MODULUS_BYTES);
2412 dst.address += CCP_ECC_OUTPUT_SIZE;
2413
2414 /* Restore the workarea address */
2415 dst.address = save;
2416
2417 e_dst:
2418 ccp_dm_free(&dst);
2419
2420 e_src:
2421 ccp_dm_free(&src);
2422
2423 return ret;
2424 }
2425
2426 static noinline_for_stack int
2427 ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2428 {
2429 struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2430
2431 ecc->ecc_result = 0;
2432
2433 if (!ecc->mod ||
2434 (ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2435 return -EINVAL;
2436
2437 switch (ecc->function) {
2438 case CCP_ECC_FUNCTION_MMUL_384BIT:
2439 case CCP_ECC_FUNCTION_MADD_384BIT:
2440 case CCP_ECC_FUNCTION_MINV_384BIT:
2441 return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2442
2443 case CCP_ECC_FUNCTION_PADD_384BIT:
2444 case CCP_ECC_FUNCTION_PMUL_384BIT:
2445 case CCP_ECC_FUNCTION_PDBL_384BIT:
2446 return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2447
2448 default:
2449 return -EINVAL;
2450 }
2451 }
2452
2453 int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2454 {
2455 int ret;
2456
2457 cmd->engine_error = 0;
2458 cmd_q->cmd_error = 0;
2459 cmd_q->int_rcvd = 0;
2460 cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2461
2462 switch (cmd->engine) {
2463 case CCP_ENGINE_AES:
2464 switch (cmd->u.aes.mode) {
2465 case CCP_AES_MODE_CMAC:
2466 ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2467 break;
2468 case CCP_AES_MODE_GCM:
2469 ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2470 break;
2471 default:
2472 ret = ccp_run_aes_cmd(cmd_q, cmd);
2473 break;
2474 }
2475 break;
2476 case CCP_ENGINE_XTS_AES_128:
2477 ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2478 break;
2479 case CCP_ENGINE_DES3:
2480 ret = ccp_run_des3_cmd(cmd_q, cmd);
2481 break;
2482 case CCP_ENGINE_SHA:
2483 ret = ccp_run_sha_cmd(cmd_q, cmd);
2484 break;
2485 case CCP_ENGINE_RSA:
2486 ret = ccp_run_rsa_cmd(cmd_q, cmd);
2487 break;
2488 case CCP_ENGINE_PASSTHRU:
2489 if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2490 ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2491 else
2492 ret = ccp_run_passthru_cmd(cmd_q, cmd);
2493 break;
2494 case CCP_ENGINE_ECC:
2495 ret = ccp_run_ecc_cmd(cmd_q, cmd);
2496 break;
2497 default:
2498 ret = -EINVAL;
2499 }
2500
2501 return ret;
2502 }
Linux with added FPC-III support