?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
watchdog: pic32-dmt: Remove .owner field for driver
[linux/fpc-iii.git] / drivers / crypto / caam / caamalg.c
blobea8189f4b0212cc038f5f4363cf10f5e8a54099a
1 /*
2 * caam - Freescale FSL CAAM support for crypto API
3 *
4 * Copyright 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Based on talitos crypto API driver.
7 *
8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
9 *
10 * --------------- ---------------
11 * | JobDesc #1 |-------------------->| ShareDesc |
12 * | *(packet 1) | | (PDB) |
13 * --------------- |------------->| (hashKey) |
14 * . | | (cipherKey) |
15 * . | |-------->| (operation) |
16 * --------------- | | ---------------
17 * | JobDesc #2 |------| |
18 * | *(packet 2) | |
19 * --------------- |
20 * . |
21 * . |
22 * --------------- |
23 * | JobDesc #3 |------------
24 * | *(packet 3) |
25 * ---------------
26 *
27 * The SharedDesc never changes for a connection unless rekeyed, but
28 * each packet will likely be in a different place. So all we need
29 * to know to process the packet is where the input is, where the
30 * output goes, and what context we want to process with. Context is
31 * in the SharedDesc, packet references in the JobDesc.
32 *
33 * So, a job desc looks like:
34 *
35 * ---------------------
36 * | Header |
37 * | ShareDesc Pointer |
38 * | SEQ_OUT_PTR |
39 * | (output buffer) |
40 * | (output length) |
41 * | SEQ_IN_PTR |
42 * | (input buffer) |
43 * | (input length) |
44 * ---------------------
45 */
46
47 #include "compat.h"
48
49 #include "regs.h"
50 #include "intern.h"
51 #include "desc_constr.h"
52 #include "jr.h"
53 #include "error.h"
54 #include "sg_sw_sec4.h"
55 #include "key_gen.h"
56
57 /*
58 * crypto alg
59 */
60 #define CAAM_CRA_PRIORITY 3000
61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
63 CTR_RFC3686_NONCE_SIZE + \
64 SHA512_DIGEST_SIZE * 2)
65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
66 #define CAAM_MAX_IV_LENGTH 16
67
68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
70 CAAM_CMD_SZ * 4)
71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
72 CAAM_CMD_SZ * 5)
73
74 /* length of descriptors text */
75 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
76 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
77 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
78 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ)
79
80 /* Note: Nonce is counted in enckeylen */
81 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ)
82
83 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ)
84 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ)
85 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ)
86
87 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ)
88 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
89 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
90
91 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
92 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
93 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
94
95 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ)
96 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
97 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
98
99 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
100 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
101 20 * CAAM_CMD_SZ)
102 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
103 15 * CAAM_CMD_SZ)
104
105 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
106 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
107
108 #ifdef DEBUG
109 /* for print_hex_dumps with line references */
110 #define debug(format, arg...) printk(format, arg)
111 #else
112 #define debug(format, arg...)
113 #endif
114 static struct list_head alg_list;
115
116 struct caam_alg_entry {
117 int class1_alg_type;
118 int class2_alg_type;
119 int alg_op;
120 bool rfc3686;
121 bool geniv;
122 };
123
124 struct caam_aead_alg {
125 struct aead_alg aead;
126 struct caam_alg_entry caam;
127 bool registered;
128 };
129
130 /* Set DK bit in class 1 operation if shared */
131 static inline void append_dec_op1(u32 *desc, u32 type)
132 {
133 u32 *jump_cmd, *uncond_jump_cmd;
134
135 /* DK bit is valid only for AES */
136 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
137 append_operation(desc, type | OP_ALG_AS_INITFINAL |
138 OP_ALG_DECRYPT);
139 return;
140 }
141
142 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
143 append_operation(desc, type | OP_ALG_AS_INITFINAL |
144 OP_ALG_DECRYPT);
145 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
146 set_jump_tgt_here(desc, jump_cmd);
147 append_operation(desc, type | OP_ALG_AS_INITFINAL |
148 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
149 set_jump_tgt_here(desc, uncond_jump_cmd);
150 }
151
152 /*
153 * For aead functions, read payload and write payload,
154 * both of which are specified in req->src and req->dst
155 */
156 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
157 {
158 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
159 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
160 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
161 }
162
163 /*
164 * For ablkcipher encrypt and decrypt, read from req->src and
165 * write to req->dst
166 */
167 static inline void ablkcipher_append_src_dst(u32 *desc)
168 {
169 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
170 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
171 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
172 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
173 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
174 }
175
176 /*
177 * per-session context
178 */
179 struct caam_ctx {
180 struct device *jrdev;
181 u32 sh_desc_enc[DESC_MAX_USED_LEN];
182 u32 sh_desc_dec[DESC_MAX_USED_LEN];
183 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
184 dma_addr_t sh_desc_enc_dma;
185 dma_addr_t sh_desc_dec_dma;
186 dma_addr_t sh_desc_givenc_dma;
187 u32 class1_alg_type;
188 u32 class2_alg_type;
189 u32 alg_op;
190 u8 key[CAAM_MAX_KEY_SIZE];
191 dma_addr_t key_dma;
192 unsigned int enckeylen;
193 unsigned int split_key_len;
194 unsigned int split_key_pad_len;
195 unsigned int authsize;
196 };
197
198 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
199 int keys_fit_inline, bool is_rfc3686)
200 {
201 u32 *nonce;
202 unsigned int enckeylen = ctx->enckeylen;
203
204 /*
205 * RFC3686 specific:
206 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
207 * | enckeylen = encryption key size + nonce size
208 */
209 if (is_rfc3686)
210 enckeylen -= CTR_RFC3686_NONCE_SIZE;
211
212 if (keys_fit_inline) {
213 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
214 ctx->split_key_len, CLASS_2 |
215 KEY_DEST_MDHA_SPLIT | KEY_ENC);
216 append_key_as_imm(desc, (void *)ctx->key +
217 ctx->split_key_pad_len, enckeylen,
218 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
219 } else {
220 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
221 KEY_DEST_MDHA_SPLIT | KEY_ENC);
222 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
223 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
224 }
225
226 /* Load Counter into CONTEXT1 reg */
227 if (is_rfc3686) {
228 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
229 enckeylen);
230 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
231 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
232 append_move(desc,
233 MOVE_SRC_OUTFIFO |
234 MOVE_DEST_CLASS1CTX |
235 (16 << MOVE_OFFSET_SHIFT) |
236 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
237 }
238 }
239
240 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
241 int keys_fit_inline, bool is_rfc3686)
242 {
243 u32 *key_jump_cmd;
244
245 /* Note: Context registers are saved. */
246 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
247
248 /* Skip if already shared */
249 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
250 JUMP_COND_SHRD);
251
252 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
253
254 set_jump_tgt_here(desc, key_jump_cmd);
255 }
256
257 static int aead_null_set_sh_desc(struct crypto_aead *aead)
258 {
259 struct caam_ctx *ctx = crypto_aead_ctx(aead);
260 struct device *jrdev = ctx->jrdev;
261 bool keys_fit_inline = false;
262 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
263 u32 *desc;
264
265 /*
266 * Job Descriptor and Shared Descriptors
267 * must all fit into the 64-word Descriptor h/w Buffer
268 */
269 if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN +
270 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
271 keys_fit_inline = true;
272
273 /* aead_encrypt shared descriptor */
274 desc = ctx->sh_desc_enc;
275
276 init_sh_desc(desc, HDR_SHARE_SERIAL);
277
278 /* Skip if already shared */
279 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
280 JUMP_COND_SHRD);
281 if (keys_fit_inline)
282 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
283 ctx->split_key_len, CLASS_2 |
284 KEY_DEST_MDHA_SPLIT | KEY_ENC);
285 else
286 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
287 KEY_DEST_MDHA_SPLIT | KEY_ENC);
288 set_jump_tgt_here(desc, key_jump_cmd);
289
290 /* assoclen + cryptlen = seqinlen */
291 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
292
293 /* Prepare to read and write cryptlen + assoclen bytes */
294 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
295 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
296
297 /*
298 * MOVE_LEN opcode is not available in all SEC HW revisions,
299 * thus need to do some magic, i.e. self-patch the descriptor
300 * buffer.
301 */
302 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
303 MOVE_DEST_MATH3 |
304 (0x6 << MOVE_LEN_SHIFT));
305 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
306 MOVE_DEST_DESCBUF |
307 MOVE_WAITCOMP |
308 (0x8 << MOVE_LEN_SHIFT));
309
310 /* Class 2 operation */
311 append_operation(desc, ctx->class2_alg_type |
312 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
313
314 /* Read and write cryptlen bytes */
315 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
316
317 set_move_tgt_here(desc, read_move_cmd);
318 set_move_tgt_here(desc, write_move_cmd);
319 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
320 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
321 MOVE_AUX_LS);
322
323 /* Write ICV */
324 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
325 LDST_SRCDST_BYTE_CONTEXT);
326
327 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
328 desc_bytes(desc),
329 DMA_TO_DEVICE);
330 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
331 dev_err(jrdev, "unable to map shared descriptor\n");
332 return -ENOMEM;
333 }
334 #ifdef DEBUG
335 print_hex_dump(KERN_ERR,
336 "aead null enc shdesc@"__stringify(__LINE__)": ",
337 DUMP_PREFIX_ADDRESS, 16, 4, desc,
338 desc_bytes(desc), 1);
339 #endif
340
341 /*
342 * Job Descriptor and Shared Descriptors
343 * must all fit into the 64-word Descriptor h/w Buffer
344 */
345 keys_fit_inline = false;
346 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
347 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
348 keys_fit_inline = true;
349
350 desc = ctx->sh_desc_dec;
351
352 /* aead_decrypt shared descriptor */
353 init_sh_desc(desc, HDR_SHARE_SERIAL);
354
355 /* Skip if already shared */
356 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
357 JUMP_COND_SHRD);
358 if (keys_fit_inline)
359 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
360 ctx->split_key_len, CLASS_2 |
361 KEY_DEST_MDHA_SPLIT | KEY_ENC);
362 else
363 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
364 KEY_DEST_MDHA_SPLIT | KEY_ENC);
365 set_jump_tgt_here(desc, key_jump_cmd);
366
367 /* Class 2 operation */
368 append_operation(desc, ctx->class2_alg_type |
369 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
370
371 /* assoclen + cryptlen = seqoutlen */
372 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
373
374 /* Prepare to read and write cryptlen + assoclen bytes */
375 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
376 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
377
378 /*
379 * MOVE_LEN opcode is not available in all SEC HW revisions,
380 * thus need to do some magic, i.e. self-patch the descriptor
381 * buffer.
382 */
383 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
384 MOVE_DEST_MATH2 |
385 (0x6 << MOVE_LEN_SHIFT));
386 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
387 MOVE_DEST_DESCBUF |
388 MOVE_WAITCOMP |
389 (0x8 << MOVE_LEN_SHIFT));
390
391 /* Read and write cryptlen bytes */
392 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
393
394 /*
395 * Insert a NOP here, since we need at least 4 instructions between
396 * code patching the descriptor buffer and the location being patched.
397 */
398 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
399 set_jump_tgt_here(desc, jump_cmd);
400
401 set_move_tgt_here(desc, read_move_cmd);
402 set_move_tgt_here(desc, write_move_cmd);
403 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
404 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
405 MOVE_AUX_LS);
406 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
407
408 /* Load ICV */
409 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
410 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
411
412 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
413 desc_bytes(desc),
414 DMA_TO_DEVICE);
415 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
416 dev_err(jrdev, "unable to map shared descriptor\n");
417 return -ENOMEM;
418 }
419 #ifdef DEBUG
420 print_hex_dump(KERN_ERR,
421 "aead null dec shdesc@"__stringify(__LINE__)": ",
422 DUMP_PREFIX_ADDRESS, 16, 4, desc,
423 desc_bytes(desc), 1);
424 #endif
425
426 return 0;
427 }
428
429 static int aead_set_sh_desc(struct crypto_aead *aead)
430 {
431 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
432 struct caam_aead_alg, aead);
433 unsigned int ivsize = crypto_aead_ivsize(aead);
434 struct caam_ctx *ctx = crypto_aead_ctx(aead);
435 struct device *jrdev = ctx->jrdev;
436 bool keys_fit_inline;
437 u32 geniv, moveiv;
438 u32 ctx1_iv_off = 0;
439 u32 *desc;
440 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
441 OP_ALG_AAI_CTR_MOD128);
442 const bool is_rfc3686 = alg->caam.rfc3686;
443
444 /* NULL encryption / decryption */
445 if (!ctx->enckeylen)
446 return aead_null_set_sh_desc(aead);
447
448 /*
449 * AES-CTR needs to load IV in CONTEXT1 reg
450 * at an offset of 128bits (16bytes)
451 * CONTEXT1[255:128] = IV
452 */
453 if (ctr_mode)
454 ctx1_iv_off = 16;
455
456 /*
457 * RFC3686 specific:
458 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
459 */
460 if (is_rfc3686)
461 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
462
463 if (alg->caam.geniv)
464 goto skip_enc;
465
466 /*
467 * Job Descriptor and Shared Descriptors
468 * must all fit into the 64-word Descriptor h/w Buffer
469 */
470 keys_fit_inline = false;
471 if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
472 ctx->split_key_pad_len + ctx->enckeylen +
473 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
474 CAAM_DESC_BYTES_MAX)
475 keys_fit_inline = true;
476
477 /* aead_encrypt shared descriptor */
478 desc = ctx->sh_desc_enc;
479
480 /* Note: Context registers are saved. */
481 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
482
483 /* Class 2 operation */
484 append_operation(desc, ctx->class2_alg_type |
485 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
486
487 /* Read and write assoclen bytes */
488 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
489 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
490
491 /* Skip assoc data */
492 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
493
494 /* read assoc before reading payload */
495 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
496 FIFOLDST_VLF);
497
498 /* Load Counter into CONTEXT1 reg */
499 if (is_rfc3686)
500 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
501 LDST_CLASS_1_CCB |
502 LDST_SRCDST_BYTE_CONTEXT |
503 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
504 LDST_OFFSET_SHIFT));
505
506 /* Class 1 operation */
507 append_operation(desc, ctx->class1_alg_type |
508 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
509
510 /* Read and write cryptlen bytes */
511 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
512 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
513 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
514
515 /* Write ICV */
516 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
517 LDST_SRCDST_BYTE_CONTEXT);
518
519 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
520 desc_bytes(desc),
521 DMA_TO_DEVICE);
522 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
523 dev_err(jrdev, "unable to map shared descriptor\n");
524 return -ENOMEM;
525 }
526 #ifdef DEBUG
527 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
528 DUMP_PREFIX_ADDRESS, 16, 4, desc,
529 desc_bytes(desc), 1);
530 #endif
531
532 skip_enc:
533 /*
534 * Job Descriptor and Shared Descriptors
535 * must all fit into the 64-word Descriptor h/w Buffer
536 */
537 keys_fit_inline = false;
538 if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN +
539 ctx->split_key_pad_len + ctx->enckeylen +
540 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
541 CAAM_DESC_BYTES_MAX)
542 keys_fit_inline = true;
543
544 /* aead_decrypt shared descriptor */
545 desc = ctx->sh_desc_dec;
546
547 /* Note: Context registers are saved. */
548 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
549
550 /* Class 2 operation */
551 append_operation(desc, ctx->class2_alg_type |
552 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
553
554 /* Read and write assoclen bytes */
555 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
556 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
557
558 /* Skip assoc data */
559 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
560
561 /* read assoc before reading payload */
562 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
563 KEY_VLF);
564
565 /* Load Counter into CONTEXT1 reg */
566 if (is_rfc3686)
567 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
568 LDST_CLASS_1_CCB |
569 LDST_SRCDST_BYTE_CONTEXT |
570 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
571 LDST_OFFSET_SHIFT));
572
573 /* Choose operation */
574 if (ctr_mode)
575 append_operation(desc, ctx->class1_alg_type |
576 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
577 else
578 append_dec_op1(desc, ctx->class1_alg_type);
579
580 /* Read and write cryptlen bytes */
581 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
582 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
583 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
584
585 /* Load ICV */
586 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
587 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
588
589 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
590 desc_bytes(desc),
591 DMA_TO_DEVICE);
592 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
593 dev_err(jrdev, "unable to map shared descriptor\n");
594 return -ENOMEM;
595 }
596 #ifdef DEBUG
597 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
598 DUMP_PREFIX_ADDRESS, 16, 4, desc,
599 desc_bytes(desc), 1);
600 #endif
601
602 if (!alg->caam.geniv)
603 goto skip_givenc;
604
605 /*
606 * Job Descriptor and Shared Descriptors
607 * must all fit into the 64-word Descriptor h/w Buffer
608 */
609 keys_fit_inline = false;
610 if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
611 ctx->split_key_pad_len + ctx->enckeylen +
612 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
613 CAAM_DESC_BYTES_MAX)
614 keys_fit_inline = true;
615
616 /* aead_givencrypt shared descriptor */
617 desc = ctx->sh_desc_givenc;
618
619 /* Note: Context registers are saved. */
620 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
621
622 if (is_rfc3686)
623 goto copy_iv;
624
625 /* Generate IV */
626 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
627 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
628 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
629 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
630 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
631 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
632 append_move(desc, MOVE_WAITCOMP |
633 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
634 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
635 (ivsize << MOVE_LEN_SHIFT));
636 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
637
638 copy_iv:
639 /* Copy IV to class 1 context */
640 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
641 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
642 (ivsize << MOVE_LEN_SHIFT));
643
644 /* Return to encryption */
645 append_operation(desc, ctx->class2_alg_type |
646 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
647
648 /* ivsize + cryptlen = seqoutlen - authsize */
649 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
650
651 /* Read and write assoclen bytes */
652 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
653 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
654
655 /* Skip assoc data */
656 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
657
658 /* read assoc before reading payload */
659 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
660 KEY_VLF);
661
662 /* Copy iv from outfifo to class 2 fifo */
663 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
664 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
665 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
666 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
667 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
668 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
669
670 /* Load Counter into CONTEXT1 reg */
671 if (is_rfc3686)
672 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
673 LDST_CLASS_1_CCB |
674 LDST_SRCDST_BYTE_CONTEXT |
675 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
676 LDST_OFFSET_SHIFT));
677
678 /* Class 1 operation */
679 append_operation(desc, ctx->class1_alg_type |
680 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
681
682 /* Will write ivsize + cryptlen */
683 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
684
685 /* Not need to reload iv */
686 append_seq_fifo_load(desc, ivsize,
687 FIFOLD_CLASS_SKIP);
688
689 /* Will read cryptlen */
690 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
691 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
692
693 /* Write ICV */
694 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
695 LDST_SRCDST_BYTE_CONTEXT);
696
697 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
698 desc_bytes(desc),
699 DMA_TO_DEVICE);
700 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
701 dev_err(jrdev, "unable to map shared descriptor\n");
702 return -ENOMEM;
703 }
704 #ifdef DEBUG
705 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
706 DUMP_PREFIX_ADDRESS, 16, 4, desc,
707 desc_bytes(desc), 1);
708 #endif
709
710 skip_givenc:
711 return 0;
712 }
713
714 static int aead_setauthsize(struct crypto_aead *authenc,
715 unsigned int authsize)
716 {
717 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
718
719 ctx->authsize = authsize;
720 aead_set_sh_desc(authenc);
721
722 return 0;
723 }
724
725 static int gcm_set_sh_desc(struct crypto_aead *aead)
726 {
727 struct caam_ctx *ctx = crypto_aead_ctx(aead);
728 struct device *jrdev = ctx->jrdev;
729 bool keys_fit_inline = false;
730 u32 *key_jump_cmd, *zero_payload_jump_cmd,
731 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
732 u32 *desc;
733
734 if (!ctx->enckeylen || !ctx->authsize)
735 return 0;
736
737 /*
738 * AES GCM encrypt shared descriptor
739 * Job Descriptor and Shared Descriptor
740 * must fit into the 64-word Descriptor h/w Buffer
741 */
742 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN +
743 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
744 keys_fit_inline = true;
745
746 desc = ctx->sh_desc_enc;
747
748 init_sh_desc(desc, HDR_SHARE_SERIAL);
749
750 /* skip key loading if they are loaded due to sharing */
751 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
752 JUMP_COND_SHRD | JUMP_COND_SELF);
753 if (keys_fit_inline)
754 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
755 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
756 else
757 append_key(desc, ctx->key_dma, ctx->enckeylen,
758 CLASS_1 | KEY_DEST_CLASS_REG);
759 set_jump_tgt_here(desc, key_jump_cmd);
760
761 /* class 1 operation */
762 append_operation(desc, ctx->class1_alg_type |
763 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
764
765 /* if assoclen + cryptlen is ZERO, skip to ICV write */
766 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
767 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
768 JUMP_COND_MATH_Z);
769
770 /* if assoclen is ZERO, skip reading the assoc data */
771 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
772 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
773 JUMP_COND_MATH_Z);
774
775 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
776
777 /* skip assoc data */
778 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
779
780 /* cryptlen = seqinlen - assoclen */
781 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
782
783 /* if cryptlen is ZERO jump to zero-payload commands */
784 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
785 JUMP_COND_MATH_Z);
786
787 /* read assoc data */
788 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
789 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
790 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
791
792 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
793
794 /* write encrypted data */
795 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
796
797 /* read payload data */
798 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
799 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
800
801 /* jump the zero-payload commands */
802 append_jump(desc, JUMP_TEST_ALL | 2);
803
804 /* zero-payload commands */
805 set_jump_tgt_here(desc, zero_payload_jump_cmd);
806
807 /* read assoc data */
808 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
809 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
810
811 /* There is no input data */
812 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
813
814 /* write ICV */
815 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
816 LDST_SRCDST_BYTE_CONTEXT);
817
818 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
819 desc_bytes(desc),
820 DMA_TO_DEVICE);
821 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
822 dev_err(jrdev, "unable to map shared descriptor\n");
823 return -ENOMEM;
824 }
825 #ifdef DEBUG
826 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
827 DUMP_PREFIX_ADDRESS, 16, 4, desc,
828 desc_bytes(desc), 1);
829 #endif
830
831 /*
832 * Job Descriptor and Shared Descriptors
833 * must all fit into the 64-word Descriptor h/w Buffer
834 */
835 keys_fit_inline = false;
836 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN +
837 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
838 keys_fit_inline = true;
839
840 desc = ctx->sh_desc_dec;
841
842 init_sh_desc(desc, HDR_SHARE_SERIAL);
843
844 /* skip key loading if they are loaded due to sharing */
845 key_jump_cmd = append_jump(desc, JUMP_JSL |
846 JUMP_TEST_ALL | JUMP_COND_SHRD |
847 JUMP_COND_SELF);
848 if (keys_fit_inline)
849 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
850 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
851 else
852 append_key(desc, ctx->key_dma, ctx->enckeylen,
853 CLASS_1 | KEY_DEST_CLASS_REG);
854 set_jump_tgt_here(desc, key_jump_cmd);
855
856 /* class 1 operation */
857 append_operation(desc, ctx->class1_alg_type |
858 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
859
860 /* if assoclen is ZERO, skip reading the assoc data */
861 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
862 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
863 JUMP_COND_MATH_Z);
864
865 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
866
867 /* skip assoc data */
868 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
869
870 /* read assoc data */
871 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
872 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
873
874 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
875
876 /* cryptlen = seqoutlen - assoclen */
877 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
878
879 /* jump to zero-payload command if cryptlen is zero */
880 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
881 JUMP_COND_MATH_Z);
882
883 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
884
885 /* store encrypted data */
886 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
887
888 /* read payload data */
889 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
890 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
891
892 /* zero-payload command */
893 set_jump_tgt_here(desc, zero_payload_jump_cmd);
894
895 /* read ICV */
896 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
897 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
898
899 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
900 desc_bytes(desc),
901 DMA_TO_DEVICE);
902 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
903 dev_err(jrdev, "unable to map shared descriptor\n");
904 return -ENOMEM;
905 }
906 #ifdef DEBUG
907 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
908 DUMP_PREFIX_ADDRESS, 16, 4, desc,
909 desc_bytes(desc), 1);
910 #endif
911
912 return 0;
913 }
914
915 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
916 {
917 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
918
919 ctx->authsize = authsize;
920 gcm_set_sh_desc(authenc);
921
922 return 0;
923 }
924
925 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
926 {
927 struct caam_ctx *ctx = crypto_aead_ctx(aead);
928 struct device *jrdev = ctx->jrdev;
929 bool keys_fit_inline = false;
930 u32 *key_jump_cmd;
931 u32 *desc;
932
933 if (!ctx->enckeylen || !ctx->authsize)
934 return 0;
935
936 /*
937 * RFC4106 encrypt shared descriptor
938 * Job Descriptor and Shared Descriptor
939 * must fit into the 64-word Descriptor h/w Buffer
940 */
941 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN +
942 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
943 keys_fit_inline = true;
944
945 desc = ctx->sh_desc_enc;
946
947 init_sh_desc(desc, HDR_SHARE_SERIAL);
948
949 /* Skip key loading if it is loaded due to sharing */
950 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
951 JUMP_COND_SHRD);
952 if (keys_fit_inline)
953 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
954 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
955 else
956 append_key(desc, ctx->key_dma, ctx->enckeylen,
957 CLASS_1 | KEY_DEST_CLASS_REG);
958 set_jump_tgt_here(desc, key_jump_cmd);
959
960 /* Class 1 operation */
961 append_operation(desc, ctx->class1_alg_type |
962 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
963
964 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
965 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
966
967 /* Read assoc data */
968 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
969 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
970
971 /* Skip IV */
972 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
973
974 /* Will read cryptlen bytes */
975 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
976
977 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
978 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
979
980 /* Skip assoc data */
981 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
982
983 /* cryptlen = seqoutlen - assoclen */
984 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
985
986 /* Write encrypted data */
987 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
988
989 /* Read payload data */
990 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
991 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
992
993 /* Write ICV */
994 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
995 LDST_SRCDST_BYTE_CONTEXT);
996
997 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
998 desc_bytes(desc),
999 DMA_TO_DEVICE);
1000 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1001 dev_err(jrdev, "unable to map shared descriptor\n");
1002 return -ENOMEM;
1003 }
1004 #ifdef DEBUG
1005 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1006 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1007 desc_bytes(desc), 1);
1008 #endif
1009
1010 /*
1011 * Job Descriptor and Shared Descriptors
1012 * must all fit into the 64-word Descriptor h/w Buffer
1013 */
1014 keys_fit_inline = false;
1015 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1016 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1017 keys_fit_inline = true;
1018
1019 desc = ctx->sh_desc_dec;
1020
1021 init_sh_desc(desc, HDR_SHARE_SERIAL);
1022
1023 /* Skip key loading if it is loaded due to sharing */
1024 key_jump_cmd = append_jump(desc, JUMP_JSL |
1025 JUMP_TEST_ALL | JUMP_COND_SHRD);
1026 if (keys_fit_inline)
1027 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1028 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1029 else
1030 append_key(desc, ctx->key_dma, ctx->enckeylen,
1031 CLASS_1 | KEY_DEST_CLASS_REG);
1032 set_jump_tgt_here(desc, key_jump_cmd);
1033
1034 /* Class 1 operation */
1035 append_operation(desc, ctx->class1_alg_type |
1036 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1037
1038 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
1039 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1040
1041 /* Read assoc data */
1042 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1043 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1044
1045 /* Skip IV */
1046 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1047
1048 /* Will read cryptlen bytes */
1049 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1050
1051 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1052 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1053
1054 /* Skip assoc data */
1055 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1056
1057 /* Will write cryptlen bytes */
1058 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1059
1060 /* Store payload data */
1061 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1062
1063 /* Read encrypted data */
1064 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1065 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1066
1067 /* Read ICV */
1068 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1069 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1070
1071 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1072 desc_bytes(desc),
1073 DMA_TO_DEVICE);
1074 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1075 dev_err(jrdev, "unable to map shared descriptor\n");
1076 return -ENOMEM;
1077 }
1078 #ifdef DEBUG
1079 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1080 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1081 desc_bytes(desc), 1);
1082 #endif
1083
1084 return 0;
1085 }
1086
1087 static int rfc4106_setauthsize(struct crypto_aead *authenc,
1088 unsigned int authsize)
1089 {
1090 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1091
1092 ctx->authsize = authsize;
1093 rfc4106_set_sh_desc(authenc);
1094
1095 return 0;
1096 }
1097
1098 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1099 {
1100 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1101 struct device *jrdev = ctx->jrdev;
1102 bool keys_fit_inline = false;
1103 u32 *key_jump_cmd;
1104 u32 *read_move_cmd, *write_move_cmd;
1105 u32 *desc;
1106
1107 if (!ctx->enckeylen || !ctx->authsize)
1108 return 0;
1109
1110 /*
1111 * RFC4543 encrypt shared descriptor
1112 * Job Descriptor and Shared Descriptor
1113 * must fit into the 64-word Descriptor h/w Buffer
1114 */
1115 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN +
1116 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1117 keys_fit_inline = true;
1118
1119 desc = ctx->sh_desc_enc;
1120
1121 init_sh_desc(desc, HDR_SHARE_SERIAL);
1122
1123 /* Skip key loading if it is loaded due to sharing */
1124 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1125 JUMP_COND_SHRD);
1126 if (keys_fit_inline)
1127 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1128 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1129 else
1130 append_key(desc, ctx->key_dma, ctx->enckeylen,
1131 CLASS_1 | KEY_DEST_CLASS_REG);
1132 set_jump_tgt_here(desc, key_jump_cmd);
1133
1134 /* Class 1 operation */
1135 append_operation(desc, ctx->class1_alg_type |
1136 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1137
1138 /* assoclen + cryptlen = seqinlen */
1139 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1140
1141 /*
1142 * MOVE_LEN opcode is not available in all SEC HW revisions,
1143 * thus need to do some magic, i.e. self-patch the descriptor
1144 * buffer.
1145 */
1146 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1147 (0x6 << MOVE_LEN_SHIFT));
1148 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1149 (0x8 << MOVE_LEN_SHIFT));
1150
1151 /* Will read assoclen + cryptlen bytes */
1152 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1153
1154 /* Will write assoclen + cryptlen bytes */
1155 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1156
1157 /* Read and write assoclen + cryptlen bytes */
1158 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1159
1160 set_move_tgt_here(desc, read_move_cmd);
1161 set_move_tgt_here(desc, write_move_cmd);
1162 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1163 /* Move payload data to OFIFO */
1164 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1165
1166 /* Write ICV */
1167 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1168 LDST_SRCDST_BYTE_CONTEXT);
1169
1170 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1171 desc_bytes(desc),
1172 DMA_TO_DEVICE);
1173 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1174 dev_err(jrdev, "unable to map shared descriptor\n");
1175 return -ENOMEM;
1176 }
1177 #ifdef DEBUG
1178 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1179 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1180 desc_bytes(desc), 1);
1181 #endif
1182
1183 /*
1184 * Job Descriptor and Shared Descriptors
1185 * must all fit into the 64-word Descriptor h/w Buffer
1186 */
1187 keys_fit_inline = false;
1188 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN +
1189 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1190 keys_fit_inline = true;
1191
1192 desc = ctx->sh_desc_dec;
1193
1194 init_sh_desc(desc, HDR_SHARE_SERIAL);
1195
1196 /* Skip key loading if it is loaded due to sharing */
1197 key_jump_cmd = append_jump(desc, JUMP_JSL |
1198 JUMP_TEST_ALL | JUMP_COND_SHRD);
1199 if (keys_fit_inline)
1200 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1201 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1202 else
1203 append_key(desc, ctx->key_dma, ctx->enckeylen,
1204 CLASS_1 | KEY_DEST_CLASS_REG);
1205 set_jump_tgt_here(desc, key_jump_cmd);
1206
1207 /* Class 1 operation */
1208 append_operation(desc, ctx->class1_alg_type |
1209 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1210
1211 /* assoclen + cryptlen = seqoutlen */
1212 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1213
1214 /*
1215 * MOVE_LEN opcode is not available in all SEC HW revisions,
1216 * thus need to do some magic, i.e. self-patch the descriptor
1217 * buffer.
1218 */
1219 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1220 (0x6 << MOVE_LEN_SHIFT));
1221 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1222 (0x8 << MOVE_LEN_SHIFT));
1223
1224 /* Will read assoclen + cryptlen bytes */
1225 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1226
1227 /* Will write assoclen + cryptlen bytes */
1228 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1229
1230 /* Store payload data */
1231 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1232
1233 /* In-snoop assoclen + cryptlen data */
1234 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1235 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1236
1237 set_move_tgt_here(desc, read_move_cmd);
1238 set_move_tgt_here(desc, write_move_cmd);
1239 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1240 /* Move payload data to OFIFO */
1241 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1242 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1243
1244 /* Read ICV */
1245 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1246 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1247
1248 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1249 desc_bytes(desc),
1250 DMA_TO_DEVICE);
1251 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1252 dev_err(jrdev, "unable to map shared descriptor\n");
1253 return -ENOMEM;
1254 }
1255 #ifdef DEBUG
1256 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1257 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1258 desc_bytes(desc), 1);
1259 #endif
1260
1261 return 0;
1262 }
1263
1264 static int rfc4543_setauthsize(struct crypto_aead *authenc,
1265 unsigned int authsize)
1266 {
1267 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1268
1269 ctx->authsize = authsize;
1270 rfc4543_set_sh_desc(authenc);
1271
1272 return 0;
1273 }
1274
1275 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1276 u32 authkeylen)
1277 {
1278 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1279 ctx->split_key_pad_len, key_in, authkeylen,
1280 ctx->alg_op);
1281 }
1282
1283 static int aead_setkey(struct crypto_aead *aead,
1284 const u8 *key, unsigned int keylen)
1285 {
1286 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1287 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1288 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1289 struct device *jrdev = ctx->jrdev;
1290 struct crypto_authenc_keys keys;
1291 int ret = 0;
1292
1293 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1294 goto badkey;
1295
1296 /* Pick class 2 key length from algorithm submask */
1297 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1298 OP_ALG_ALGSEL_SHIFT] * 2;
1299 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1300
1301 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1302 goto badkey;
1303
1304 #ifdef DEBUG
1305 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
1306 keys.authkeylen + keys.enckeylen, keys.enckeylen,
1307 keys.authkeylen);
1308 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1309 ctx->split_key_len, ctx->split_key_pad_len);
1310 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1311 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1312 #endif
1313
1314 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
1315 if (ret) {
1316 goto badkey;
1317 }
1318
1319 /* postpend encryption key to auth split key */
1320 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
1321
1322 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
1323 keys.enckeylen, DMA_TO_DEVICE);
1324 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1325 dev_err(jrdev, "unable to map key i/o memory\n");
1326 return -ENOMEM;
1327 }
1328 #ifdef DEBUG
1329 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
1330 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
1331 ctx->split_key_pad_len + keys.enckeylen, 1);
1332 #endif
1333
1334 ctx->enckeylen = keys.enckeylen;
1335
1336 ret = aead_set_sh_desc(aead);
1337 if (ret) {
1338 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
1339 keys.enckeylen, DMA_TO_DEVICE);
1340 }
1341
1342 return ret;
1343 badkey:
1344 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1345 return -EINVAL;
1346 }
1347
1348 static int gcm_setkey(struct crypto_aead *aead,
1349 const u8 *key, unsigned int keylen)
1350 {
1351 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1352 struct device *jrdev = ctx->jrdev;
1353 int ret = 0;
1354
1355 #ifdef DEBUG
1356 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1357 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1358 #endif
1359
1360 memcpy(ctx->key, key, keylen);
1361 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1362 DMA_TO_DEVICE);
1363 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1364 dev_err(jrdev, "unable to map key i/o memory\n");
1365 return -ENOMEM;
1366 }
1367 ctx->enckeylen = keylen;
1368
1369 ret = gcm_set_sh_desc(aead);
1370 if (ret) {
1371 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1372 DMA_TO_DEVICE);
1373 }
1374
1375 return ret;
1376 }
1377
1378 static int rfc4106_setkey(struct crypto_aead *aead,
1379 const u8 *key, unsigned int keylen)
1380 {
1381 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1382 struct device *jrdev = ctx->jrdev;
1383 int ret = 0;
1384
1385 if (keylen < 4)
1386 return -EINVAL;
1387
1388 #ifdef DEBUG
1389 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1390 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1391 #endif
1392
1393 memcpy(ctx->key, key, keylen);
1394
1395 /*
1396 * The last four bytes of the key material are used as the salt value
1397 * in the nonce. Update the AES key length.
1398 */
1399 ctx->enckeylen = keylen - 4;
1400
1401 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1402 DMA_TO_DEVICE);
1403 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1404 dev_err(jrdev, "unable to map key i/o memory\n");
1405 return -ENOMEM;
1406 }
1407
1408 ret = rfc4106_set_sh_desc(aead);
1409 if (ret) {
1410 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1411 DMA_TO_DEVICE);
1412 }
1413
1414 return ret;
1415 }
1416
1417 static int rfc4543_setkey(struct crypto_aead *aead,
1418 const u8 *key, unsigned int keylen)
1419 {
1420 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1421 struct device *jrdev = ctx->jrdev;
1422 int ret = 0;
1423
1424 if (keylen < 4)
1425 return -EINVAL;
1426
1427 #ifdef DEBUG
1428 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1429 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1430 #endif
1431
1432 memcpy(ctx->key, key, keylen);
1433
1434 /*
1435 * The last four bytes of the key material are used as the salt value
1436 * in the nonce. Update the AES key length.
1437 */
1438 ctx->enckeylen = keylen - 4;
1439
1440 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1441 DMA_TO_DEVICE);
1442 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1443 dev_err(jrdev, "unable to map key i/o memory\n");
1444 return -ENOMEM;
1445 }
1446
1447 ret = rfc4543_set_sh_desc(aead);
1448 if (ret) {
1449 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1450 DMA_TO_DEVICE);
1451 }
1452
1453 return ret;
1454 }
1455
1456 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1457 const u8 *key, unsigned int keylen)
1458 {
1459 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1460 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1461 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1462 const char *alg_name = crypto_tfm_alg_name(tfm);
1463 struct device *jrdev = ctx->jrdev;
1464 int ret = 0;
1465 u32 *key_jump_cmd;
1466 u32 *desc;
1467 u32 *nonce;
1468 u32 geniv;
1469 u32 ctx1_iv_off = 0;
1470 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1471 OP_ALG_AAI_CTR_MOD128);
1472 const bool is_rfc3686 = (ctr_mode &&
1473 (strstr(alg_name, "rfc3686") != NULL));
1474
1475 #ifdef DEBUG
1476 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1477 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1478 #endif
1479 /*
1480 * AES-CTR needs to load IV in CONTEXT1 reg
1481 * at an offset of 128bits (16bytes)
1482 * CONTEXT1[255:128] = IV
1483 */
1484 if (ctr_mode)
1485 ctx1_iv_off = 16;
1486
1487 /*
1488 * RFC3686 specific:
1489 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1490 * | *key = {KEY, NONCE}
1491 */
1492 if (is_rfc3686) {
1493 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1494 keylen -= CTR_RFC3686_NONCE_SIZE;
1495 }
1496
1497 memcpy(ctx->key, key, keylen);
1498 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1499 DMA_TO_DEVICE);
1500 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1501 dev_err(jrdev, "unable to map key i/o memory\n");
1502 return -ENOMEM;
1503 }
1504 ctx->enckeylen = keylen;
1505
1506 /* ablkcipher_encrypt shared descriptor */
1507 desc = ctx->sh_desc_enc;
1508 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1509 /* Skip if already shared */
1510 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1511 JUMP_COND_SHRD);
1512
1513 /* Load class1 key only */
1514 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1515 ctx->enckeylen, CLASS_1 |
1516 KEY_DEST_CLASS_REG);
1517
1518 /* Load nonce into CONTEXT1 reg */
1519 if (is_rfc3686) {
1520 nonce = (u32 *)(key + keylen);
1521 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1522 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1523 append_move(desc, MOVE_WAITCOMP |
1524 MOVE_SRC_OUTFIFO |
1525 MOVE_DEST_CLASS1CTX |
1526 (16 << MOVE_OFFSET_SHIFT) |
1527 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1528 }
1529
1530 set_jump_tgt_here(desc, key_jump_cmd);
1531
1532 /* Load iv */
1533 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1534 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1535
1536 /* Load counter into CONTEXT1 reg */
1537 if (is_rfc3686)
1538 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1539 LDST_CLASS_1_CCB |
1540 LDST_SRCDST_BYTE_CONTEXT |
1541 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1542 LDST_OFFSET_SHIFT));
1543
1544 /* Load operation */
1545 append_operation(desc, ctx->class1_alg_type |
1546 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1547
1548 /* Perform operation */
1549 ablkcipher_append_src_dst(desc);
1550
1551 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1552 desc_bytes(desc),
1553 DMA_TO_DEVICE);
1554 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1555 dev_err(jrdev, "unable to map shared descriptor\n");
1556 return -ENOMEM;
1557 }
1558 #ifdef DEBUG
1559 print_hex_dump(KERN_ERR,
1560 "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
1561 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1562 desc_bytes(desc), 1);
1563 #endif
1564 /* ablkcipher_decrypt shared descriptor */
1565 desc = ctx->sh_desc_dec;
1566
1567 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1568 /* Skip if already shared */
1569 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1570 JUMP_COND_SHRD);
1571
1572 /* Load class1 key only */
1573 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1574 ctx->enckeylen, CLASS_1 |
1575 KEY_DEST_CLASS_REG);
1576
1577 /* Load nonce into CONTEXT1 reg */
1578 if (is_rfc3686) {
1579 nonce = (u32 *)(key + keylen);
1580 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1581 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1582 append_move(desc, MOVE_WAITCOMP |
1583 MOVE_SRC_OUTFIFO |
1584 MOVE_DEST_CLASS1CTX |
1585 (16 << MOVE_OFFSET_SHIFT) |
1586 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1587 }
1588
1589 set_jump_tgt_here(desc, key_jump_cmd);
1590
1591 /* load IV */
1592 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1593 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1594
1595 /* Load counter into CONTEXT1 reg */
1596 if (is_rfc3686)
1597 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1598 LDST_CLASS_1_CCB |
1599 LDST_SRCDST_BYTE_CONTEXT |
1600 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1601 LDST_OFFSET_SHIFT));
1602
1603 /* Choose operation */
1604 if (ctr_mode)
1605 append_operation(desc, ctx->class1_alg_type |
1606 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1607 else
1608 append_dec_op1(desc, ctx->class1_alg_type);
1609
1610 /* Perform operation */
1611 ablkcipher_append_src_dst(desc);
1612
1613 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1614 desc_bytes(desc),
1615 DMA_TO_DEVICE);
1616 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1617 dev_err(jrdev, "unable to map shared descriptor\n");
1618 return -ENOMEM;
1619 }
1620
1621 #ifdef DEBUG
1622 print_hex_dump(KERN_ERR,
1623 "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
1624 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1625 desc_bytes(desc), 1);
1626 #endif
1627 /* ablkcipher_givencrypt shared descriptor */
1628 desc = ctx->sh_desc_givenc;
1629
1630 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1631 /* Skip if already shared */
1632 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1633 JUMP_COND_SHRD);
1634
1635 /* Load class1 key only */
1636 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1637 ctx->enckeylen, CLASS_1 |
1638 KEY_DEST_CLASS_REG);
1639
1640 /* Load Nonce into CONTEXT1 reg */
1641 if (is_rfc3686) {
1642 nonce = (u32 *)(key + keylen);
1643 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1644 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1645 append_move(desc, MOVE_WAITCOMP |
1646 MOVE_SRC_OUTFIFO |
1647 MOVE_DEST_CLASS1CTX |
1648 (16 << MOVE_OFFSET_SHIFT) |
1649 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1650 }
1651 set_jump_tgt_here(desc, key_jump_cmd);
1652
1653 /* Generate IV */
1654 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1655 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1656 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
1657 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1658 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1659 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1660 append_move(desc, MOVE_WAITCOMP |
1661 MOVE_SRC_INFIFO |
1662 MOVE_DEST_CLASS1CTX |
1663 (crt->ivsize << MOVE_LEN_SHIFT) |
1664 (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1665 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1666
1667 /* Copy generated IV to memory */
1668 append_seq_store(desc, crt->ivsize,
1669 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1670 (ctx1_iv_off << LDST_OFFSET_SHIFT));
1671
1672 /* Load Counter into CONTEXT1 reg */
1673 if (is_rfc3686)
1674 append_load_imm_u32(desc, (u32)1, LDST_IMM |
1675 LDST_CLASS_1_CCB |
1676 LDST_SRCDST_BYTE_CONTEXT |
1677 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1678 LDST_OFFSET_SHIFT));
1679
1680 if (ctx1_iv_off)
1681 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1682 (1 << JUMP_OFFSET_SHIFT));
1683
1684 /* Load operation */
1685 append_operation(desc, ctx->class1_alg_type |
1686 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1687
1688 /* Perform operation */
1689 ablkcipher_append_src_dst(desc);
1690
1691 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1692 desc_bytes(desc),
1693 DMA_TO_DEVICE);
1694 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1695 dev_err(jrdev, "unable to map shared descriptor\n");
1696 return -ENOMEM;
1697 }
1698 #ifdef DEBUG
1699 print_hex_dump(KERN_ERR,
1700 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1701 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1702 desc_bytes(desc), 1);
1703 #endif
1704
1705 return ret;
1706 }
1707
1708 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1709 const u8 *key, unsigned int keylen)
1710 {
1711 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1712 struct device *jrdev = ctx->jrdev;
1713 u32 *key_jump_cmd, *desc;
1714 __be64 sector_size = cpu_to_be64(512);
1715
1716 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
1717 crypto_ablkcipher_set_flags(ablkcipher,
1718 CRYPTO_TFM_RES_BAD_KEY_LEN);
1719 dev_err(jrdev, "key size mismatch\n");
1720 return -EINVAL;
1721 }
1722
1723 memcpy(ctx->key, key, keylen);
1724 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE);
1725 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1726 dev_err(jrdev, "unable to map key i/o memory\n");
1727 return -ENOMEM;
1728 }
1729 ctx->enckeylen = keylen;
1730
1731 /* xts_ablkcipher_encrypt shared descriptor */
1732 desc = ctx->sh_desc_enc;
1733 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1734 /* Skip if already shared */
1735 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1736 JUMP_COND_SHRD);
1737
1738 /* Load class1 keys only */
1739 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1740 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1741
1742 /* Load sector size with index 40 bytes (0x28) */
1743 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
1744 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
1745 append_data(desc, (void *)&sector_size, 8);
1746
1747 set_jump_tgt_here(desc, key_jump_cmd);
1748
1749 /*
1750 * create sequence for loading the sector index
1751 * Upper 8B of IV - will be used as sector index
1752 * Lower 8B of IV - will be discarded
1753 */
1754 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
1755 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
1756 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1757
1758 /* Load operation */
1759 append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL |
1760 OP_ALG_ENCRYPT);
1761
1762 /* Perform operation */
1763 ablkcipher_append_src_dst(desc);
1764
1765 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
1766 DMA_TO_DEVICE);
1767 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1768 dev_err(jrdev, "unable to map shared descriptor\n");
1769 return -ENOMEM;
1770 }
1771 #ifdef DEBUG
1772 print_hex_dump(KERN_ERR,
1773 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1774 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1775 #endif
1776
1777 /* xts_ablkcipher_decrypt shared descriptor */
1778 desc = ctx->sh_desc_dec;
1779
1780 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1781 /* Skip if already shared */
1782 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1783 JUMP_COND_SHRD);
1784
1785 /* Load class1 key only */
1786 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1787 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1788
1789 /* Load sector size with index 40 bytes (0x28) */
1790 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
1791 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
1792 append_data(desc, (void *)&sector_size, 8);
1793
1794 set_jump_tgt_here(desc, key_jump_cmd);
1795
1796 /*
1797 * create sequence for loading the sector index
1798 * Upper 8B of IV - will be used as sector index
1799 * Lower 8B of IV - will be discarded
1800 */
1801 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
1802 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
1803 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1804
1805 /* Load operation */
1806 append_dec_op1(desc, ctx->class1_alg_type);
1807
1808 /* Perform operation */
1809 ablkcipher_append_src_dst(desc);
1810
1811 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
1812 DMA_TO_DEVICE);
1813 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1814 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma,
1815 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
1816 dev_err(jrdev, "unable to map shared descriptor\n");
1817 return -ENOMEM;
1818 }
1819 #ifdef DEBUG
1820 print_hex_dump(KERN_ERR,
1821 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1822 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1823 #endif
1824
1825 return 0;
1826 }
1827
1828 /*
1829 * aead_edesc - s/w-extended aead descriptor
1830 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
1831 * @src_nents: number of segments in input scatterlist
1832 * @dst_nents: number of segments in output scatterlist
1833 * @iv_dma: dma address of iv for checking continuity and link table
1834 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1835 * @sec4_sg_bytes: length of dma mapped sec4_sg space
1836 * @sec4_sg_dma: bus physical mapped address of h/w link table
1837 * @hw_desc: the h/w job descriptor followed by any referenced link tables
1838 */
1839 struct aead_edesc {
1840 int assoc_nents;
1841 int src_nents;
1842 int dst_nents;
1843 dma_addr_t iv_dma;
1844 int sec4_sg_bytes;
1845 dma_addr_t sec4_sg_dma;
1846 struct sec4_sg_entry *sec4_sg;
1847 u32 hw_desc[];
1848 };
1849
1850 /*
1851 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
1852 * @src_nents: number of segments in input scatterlist
1853 * @dst_nents: number of segments in output scatterlist
1854 * @iv_dma: dma address of iv for checking continuity and link table
1855 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1856 * @sec4_sg_bytes: length of dma mapped sec4_sg space
1857 * @sec4_sg_dma: bus physical mapped address of h/w link table
1858 * @hw_desc: the h/w job descriptor followed by any referenced link tables
1859 */
1860 struct ablkcipher_edesc {
1861 int src_nents;
1862 int dst_nents;
1863 dma_addr_t iv_dma;
1864 int sec4_sg_bytes;
1865 dma_addr_t sec4_sg_dma;
1866 struct sec4_sg_entry *sec4_sg;
1867 u32 hw_desc[0];
1868 };
1869
1870 static void caam_unmap(struct device *dev, struct scatterlist *src,
1871 struct scatterlist *dst, int src_nents,
1872 int dst_nents,
1873 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
1874 int sec4_sg_bytes)
1875 {
1876 if (dst != src) {
1877 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE);
1878 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE);
1879 } else {
1880 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL);
1881 }
1882
1883 if (iv_dma)
1884 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1885 if (sec4_sg_bytes)
1886 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
1887 DMA_TO_DEVICE);
1888 }
1889
1890 static void aead_unmap(struct device *dev,
1891 struct aead_edesc *edesc,
1892 struct aead_request *req)
1893 {
1894 caam_unmap(dev, req->src, req->dst,
1895 edesc->src_nents, edesc->dst_nents, 0, 0,
1896 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1897 }
1898
1899 static void ablkcipher_unmap(struct device *dev,
1900 struct ablkcipher_edesc *edesc,
1901 struct ablkcipher_request *req)
1902 {
1903 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1904 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1905
1906 caam_unmap(dev, req->src, req->dst,
1907 edesc->src_nents, edesc->dst_nents,
1908 edesc->iv_dma, ivsize,
1909 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1910 }
1911
1912 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1913 void *context)
1914 {
1915 struct aead_request *req = context;
1916 struct aead_edesc *edesc;
1917
1918 #ifdef DEBUG
1919 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1920 #endif
1921
1922 edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1923
1924 if (err)
1925 caam_jr_strstatus(jrdev, err);
1926
1927 aead_unmap(jrdev, edesc, req);
1928
1929 kfree(edesc);
1930
1931 aead_request_complete(req, err);
1932 }
1933
1934 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1935 void *context)
1936 {
1937 struct aead_request *req = context;
1938 struct aead_edesc *edesc;
1939
1940 #ifdef DEBUG
1941 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1942 #endif
1943
1944 edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1945
1946 if (err)
1947 caam_jr_strstatus(jrdev, err);
1948
1949 aead_unmap(jrdev, edesc, req);
1950
1951 /*
1952 * verify hw auth check passed else return -EBADMSG
1953 */
1954 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
1955 err = -EBADMSG;
1956
1957 kfree(edesc);
1958
1959 aead_request_complete(req, err);
1960 }
1961
1962 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1963 void *context)
1964 {
1965 struct ablkcipher_request *req = context;
1966 struct ablkcipher_edesc *edesc;
1967 #ifdef DEBUG
1968 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1969 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1970
1971 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1972 #endif
1973
1974 edesc = (struct ablkcipher_edesc *)((char *)desc -
1975 offsetof(struct ablkcipher_edesc, hw_desc));
1976
1977 if (err)
1978 caam_jr_strstatus(jrdev, err);
1979
1980 #ifdef DEBUG
1981 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
1982 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1983 edesc->src_nents > 1 ? 100 : ivsize, 1);
1984 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
1985 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1986 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
1987 #endif
1988
1989 ablkcipher_unmap(jrdev, edesc, req);
1990 kfree(edesc);
1991
1992 ablkcipher_request_complete(req, err);
1993 }
1994
1995 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1996 void *context)
1997 {
1998 struct ablkcipher_request *req = context;
1999 struct ablkcipher_edesc *edesc;
2000 #ifdef DEBUG
2001 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2002 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2003
2004 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2005 #endif
2006
2007 edesc = (struct ablkcipher_edesc *)((char *)desc -
2008 offsetof(struct ablkcipher_edesc, hw_desc));
2009 if (err)
2010 caam_jr_strstatus(jrdev, err);
2011
2012 #ifdef DEBUG
2013 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
2014 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2015 ivsize, 1);
2016 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
2017 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2018 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2019 #endif
2020
2021 ablkcipher_unmap(jrdev, edesc, req);
2022 kfree(edesc);
2023
2024 ablkcipher_request_complete(req, err);
2025 }
2026
2027 /*
2028 * Fill in aead job descriptor
2029 */
2030 static void init_aead_job(struct aead_request *req,
2031 struct aead_edesc *edesc,
2032 bool all_contig, bool encrypt)
2033 {
2034 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2035 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2036 int authsize = ctx->authsize;
2037 u32 *desc = edesc->hw_desc;
2038 u32 out_options, in_options;
2039 dma_addr_t dst_dma, src_dma;
2040 int len, sec4_sg_index = 0;
2041 dma_addr_t ptr;
2042 u32 *sh_desc;
2043
2044 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
2045 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
2046
2047 len = desc_len(sh_desc);
2048 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2049
2050 if (all_contig) {
2051 src_dma = sg_dma_address(req->src);
2052 in_options = 0;
2053 } else {
2054 src_dma = edesc->sec4_sg_dma;
2055 sec4_sg_index += edesc->src_nents;
2056 in_options = LDST_SGF;
2057 }
2058
2059 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
2060 in_options);
2061
2062 dst_dma = src_dma;
2063 out_options = in_options;
2064
2065 if (unlikely(req->src != req->dst)) {
2066 if (!edesc->dst_nents) {
2067 dst_dma = sg_dma_address(req->dst);
2068 } else {
2069 dst_dma = edesc->sec4_sg_dma +
2070 sec4_sg_index *
2071 sizeof(struct sec4_sg_entry);
2072 out_options = LDST_SGF;
2073 }
2074 }
2075
2076 if (encrypt)
2077 append_seq_out_ptr(desc, dst_dma,
2078 req->assoclen + req->cryptlen + authsize,
2079 out_options);
2080 else
2081 append_seq_out_ptr(desc, dst_dma,
2082 req->assoclen + req->cryptlen - authsize,
2083 out_options);
2084
2085 /* REG3 = assoclen */
2086 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
2087 }
2088
2089 static void init_gcm_job(struct aead_request *req,
2090 struct aead_edesc *edesc,
2091 bool all_contig, bool encrypt)
2092 {
2093 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2094 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2095 unsigned int ivsize = crypto_aead_ivsize(aead);
2096 u32 *desc = edesc->hw_desc;
2097 bool generic_gcm = (ivsize == 12);
2098 unsigned int last;
2099
2100 init_aead_job(req, edesc, all_contig, encrypt);
2101
2102 /* BUG This should not be specific to generic GCM. */
2103 last = 0;
2104 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
2105 last = FIFOLD_TYPE_LAST1;
2106
2107 /* Read GCM IV */
2108 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
2109 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
2110 /* Append Salt */
2111 if (!generic_gcm)
2112 append_data(desc, ctx->key + ctx->enckeylen, 4);
2113 /* Append IV */
2114 append_data(desc, req->iv, ivsize);
2115 /* End of blank commands */
2116 }
2117
2118 static void init_authenc_job(struct aead_request *req,
2119 struct aead_edesc *edesc,
2120 bool all_contig, bool encrypt)
2121 {
2122 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2123 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
2124 struct caam_aead_alg, aead);
2125 unsigned int ivsize = crypto_aead_ivsize(aead);
2126 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2127 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
2128 OP_ALG_AAI_CTR_MOD128);
2129 const bool is_rfc3686 = alg->caam.rfc3686;
2130 u32 *desc = edesc->hw_desc;
2131 u32 ivoffset = 0;
2132
2133 /*
2134 * AES-CTR needs to load IV in CONTEXT1 reg
2135 * at an offset of 128bits (16bytes)
2136 * CONTEXT1[255:128] = IV
2137 */
2138 if (ctr_mode)
2139 ivoffset = 16;
2140
2141 /*
2142 * RFC3686 specific:
2143 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
2144 */
2145 if (is_rfc3686)
2146 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
2147
2148 init_aead_job(req, edesc, all_contig, encrypt);
2149
2150 if (ivsize && (is_rfc3686 || !(alg->caam.geniv && encrypt)))
2151 append_load_as_imm(desc, req->iv, ivsize,
2152 LDST_CLASS_1_CCB |
2153 LDST_SRCDST_BYTE_CONTEXT |
2154 (ivoffset << LDST_OFFSET_SHIFT));
2155 }
2156
2157 /*
2158 * Fill in ablkcipher job descriptor
2159 */
2160 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2161 struct ablkcipher_edesc *edesc,
2162 struct ablkcipher_request *req,
2163 bool iv_contig)
2164 {
2165 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2166 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2167 u32 *desc = edesc->hw_desc;
2168 u32 out_options = 0, in_options;
2169 dma_addr_t dst_dma, src_dma;
2170 int len, sec4_sg_index = 0;
2171
2172 #ifdef DEBUG
2173 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2174 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2175 ivsize, 1);
2176 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
2177 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2178 edesc->src_nents ? 100 : req->nbytes, 1);
2179 #endif
2180
2181 len = desc_len(sh_desc);
2182 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2183
2184 if (iv_contig) {
2185 src_dma = edesc->iv_dma;
2186 in_options = 0;
2187 } else {
2188 src_dma = edesc->sec4_sg_dma;
2189 sec4_sg_index += edesc->src_nents + 1;
2190 in_options = LDST_SGF;
2191 }
2192 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2193
2194 if (likely(req->src == req->dst)) {
2195 if (!edesc->src_nents && iv_contig) {
2196 dst_dma = sg_dma_address(req->src);
2197 } else {
2198 dst_dma = edesc->sec4_sg_dma +
2199 sizeof(struct sec4_sg_entry);
2200 out_options = LDST_SGF;
2201 }
2202 } else {
2203 if (!edesc->dst_nents) {
2204 dst_dma = sg_dma_address(req->dst);
2205 } else {
2206 dst_dma = edesc->sec4_sg_dma +
2207 sec4_sg_index * sizeof(struct sec4_sg_entry);
2208 out_options = LDST_SGF;
2209 }
2210 }
2211 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2212 }
2213
2214 /*
2215 * Fill in ablkcipher givencrypt job descriptor
2216 */
2217 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2218 struct ablkcipher_edesc *edesc,
2219 struct ablkcipher_request *req,
2220 bool iv_contig)
2221 {
2222 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2223 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2224 u32 *desc = edesc->hw_desc;
2225 u32 out_options, in_options;
2226 dma_addr_t dst_dma, src_dma;
2227 int len, sec4_sg_index = 0;
2228
2229 #ifdef DEBUG
2230 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2231 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2232 ivsize, 1);
2233 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ",
2234 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2235 edesc->src_nents ? 100 : req->nbytes, 1);
2236 #endif
2237
2238 len = desc_len(sh_desc);
2239 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2240
2241 if (!edesc->src_nents) {
2242 src_dma = sg_dma_address(req->src);
2243 in_options = 0;
2244 } else {
2245 src_dma = edesc->sec4_sg_dma;
2246 sec4_sg_index += edesc->src_nents;
2247 in_options = LDST_SGF;
2248 }
2249 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2250
2251 if (iv_contig) {
2252 dst_dma = edesc->iv_dma;
2253 out_options = 0;
2254 } else {
2255 dst_dma = edesc->sec4_sg_dma +
2256 sec4_sg_index * sizeof(struct sec4_sg_entry);
2257 out_options = LDST_SGF;
2258 }
2259 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2260 }
2261
2262 /*
2263 * allocate and map the aead extended descriptor
2264 */
2265 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
2266 int desc_bytes, bool *all_contig_ptr,
2267 bool encrypt)
2268 {
2269 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2270 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2271 struct device *jrdev = ctx->jrdev;
2272 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2273 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2274 int src_nents, dst_nents = 0;
2275 struct aead_edesc *edesc;
2276 int sgc;
2277 bool all_contig = true;
2278 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
2279 unsigned int authsize = ctx->authsize;
2280
2281 if (unlikely(req->dst != req->src)) {
2282 src_nents = sg_count(req->src, req->assoclen + req->cryptlen);
2283 dst_nents = sg_count(req->dst,
2284 req->assoclen + req->cryptlen +
2285 (encrypt ? authsize : (-authsize)));
2286 } else {
2287 src_nents = sg_count(req->src,
2288 req->assoclen + req->cryptlen +
2289 (encrypt ? authsize : 0));
2290 }
2291
2292 /* Check if data are contiguous. */
2293 all_contig = !src_nents;
2294 if (!all_contig) {
2295 src_nents = src_nents ? : 1;
2296 sec4_sg_len = src_nents;
2297 }
2298
2299 sec4_sg_len += dst_nents;
2300
2301 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2302
2303 /* allocate space for base edesc and hw desc commands, link tables */
2304 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2305 GFP_DMA | flags);
2306 if (!edesc) {
2307 dev_err(jrdev, "could not allocate extended descriptor\n");
2308 return ERR_PTR(-ENOMEM);
2309 }
2310
2311 if (likely(req->src == req->dst)) {
2312 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2313 DMA_BIDIRECTIONAL);
2314 if (unlikely(!sgc)) {
2315 dev_err(jrdev, "unable to map source\n");
2316 kfree(edesc);
2317 return ERR_PTR(-ENOMEM);
2318 }
2319 } else {
2320 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2321 DMA_TO_DEVICE);
2322 if (unlikely(!sgc)) {
2323 dev_err(jrdev, "unable to map source\n");
2324 kfree(edesc);
2325 return ERR_PTR(-ENOMEM);
2326 }
2327
2328 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2329 DMA_FROM_DEVICE);
2330 if (unlikely(!sgc)) {
2331 dev_err(jrdev, "unable to map destination\n");
2332 dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
2333 DMA_TO_DEVICE);
2334 kfree(edesc);
2335 return ERR_PTR(-ENOMEM);
2336 }
2337 }
2338
2339 edesc->src_nents = src_nents;
2340 edesc->dst_nents = dst_nents;
2341 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2342 desc_bytes;
2343 *all_contig_ptr = all_contig;
2344
2345 sec4_sg_index = 0;
2346 if (!all_contig) {
2347 sg_to_sec4_sg_last(req->src, src_nents,
2348 edesc->sec4_sg + sec4_sg_index, 0);
2349 sec4_sg_index += src_nents;
2350 }
2351 if (dst_nents) {
2352 sg_to_sec4_sg_last(req->dst, dst_nents,
2353 edesc->sec4_sg + sec4_sg_index, 0);
2354 }
2355
2356 if (!sec4_sg_bytes)
2357 return edesc;
2358
2359 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2360 sec4_sg_bytes, DMA_TO_DEVICE);
2361 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2362 dev_err(jrdev, "unable to map S/G table\n");
2363 aead_unmap(jrdev, edesc, req);
2364 kfree(edesc);
2365 return ERR_PTR(-ENOMEM);
2366 }
2367
2368 edesc->sec4_sg_bytes = sec4_sg_bytes;
2369
2370 return edesc;
2371 }
2372
2373 static int gcm_encrypt(struct aead_request *req)
2374 {
2375 struct aead_edesc *edesc;
2376 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2377 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2378 struct device *jrdev = ctx->jrdev;
2379 bool all_contig;
2380 u32 *desc;
2381 int ret = 0;
2382
2383 /* allocate extended descriptor */
2384 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
2385 if (IS_ERR(edesc))
2386 return PTR_ERR(edesc);
2387
2388 /* Create and submit job descriptor */
2389 init_gcm_job(req, edesc, all_contig, true);
2390 #ifdef DEBUG
2391 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2392 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2393 desc_bytes(edesc->hw_desc), 1);
2394 #endif
2395
2396 desc = edesc->hw_desc;
2397 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2398 if (!ret) {
2399 ret = -EINPROGRESS;
2400 } else {
2401 aead_unmap(jrdev, edesc, req);
2402 kfree(edesc);
2403 }
2404
2405 return ret;
2406 }
2407
2408 static int ipsec_gcm_encrypt(struct aead_request *req)
2409 {
2410 if (req->assoclen < 8)
2411 return -EINVAL;
2412
2413 return gcm_encrypt(req);
2414 }
2415
2416 static int aead_encrypt(struct aead_request *req)
2417 {
2418 struct aead_edesc *edesc;
2419 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2420 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2421 struct device *jrdev = ctx->jrdev;
2422 bool all_contig;
2423 u32 *desc;
2424 int ret = 0;
2425
2426 /* allocate extended descriptor */
2427 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2428 &all_contig, true);
2429 if (IS_ERR(edesc))
2430 return PTR_ERR(edesc);
2431
2432 /* Create and submit job descriptor */
2433 init_authenc_job(req, edesc, all_contig, true);
2434 #ifdef DEBUG
2435 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2436 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2437 desc_bytes(edesc->hw_desc), 1);
2438 #endif
2439
2440 desc = edesc->hw_desc;
2441 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2442 if (!ret) {
2443 ret = -EINPROGRESS;
2444 } else {
2445 aead_unmap(jrdev, edesc, req);
2446 kfree(edesc);
2447 }
2448
2449 return ret;
2450 }
2451
2452 static int gcm_decrypt(struct aead_request *req)
2453 {
2454 struct aead_edesc *edesc;
2455 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2456 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2457 struct device *jrdev = ctx->jrdev;
2458 bool all_contig;
2459 u32 *desc;
2460 int ret = 0;
2461
2462 /* allocate extended descriptor */
2463 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
2464 if (IS_ERR(edesc))
2465 return PTR_ERR(edesc);
2466
2467 /* Create and submit job descriptor*/
2468 init_gcm_job(req, edesc, all_contig, false);
2469 #ifdef DEBUG
2470 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2471 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2472 desc_bytes(edesc->hw_desc), 1);
2473 #endif
2474
2475 desc = edesc->hw_desc;
2476 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2477 if (!ret) {
2478 ret = -EINPROGRESS;
2479 } else {
2480 aead_unmap(jrdev, edesc, req);
2481 kfree(edesc);
2482 }
2483
2484 return ret;
2485 }
2486
2487 static int ipsec_gcm_decrypt(struct aead_request *req)
2488 {
2489 if (req->assoclen < 8)
2490 return -EINVAL;
2491
2492 return gcm_decrypt(req);
2493 }
2494
2495 static int aead_decrypt(struct aead_request *req)
2496 {
2497 struct aead_edesc *edesc;
2498 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2499 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2500 struct device *jrdev = ctx->jrdev;
2501 bool all_contig;
2502 u32 *desc;
2503 int ret = 0;
2504
2505 /* allocate extended descriptor */
2506 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2507 &all_contig, false);
2508 if (IS_ERR(edesc))
2509 return PTR_ERR(edesc);
2510
2511 #ifdef DEBUG
2512 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
2513 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2514 req->assoclen + req->cryptlen, 1);
2515 #endif
2516
2517 /* Create and submit job descriptor*/
2518 init_authenc_job(req, edesc, all_contig, false);
2519 #ifdef DEBUG
2520 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2521 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2522 desc_bytes(edesc->hw_desc), 1);
2523 #endif
2524
2525 desc = edesc->hw_desc;
2526 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2527 if (!ret) {
2528 ret = -EINPROGRESS;
2529 } else {
2530 aead_unmap(jrdev, edesc, req);
2531 kfree(edesc);
2532 }
2533
2534 return ret;
2535 }
2536
2537 static int aead_givdecrypt(struct aead_request *req)
2538 {
2539 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2540 unsigned int ivsize = crypto_aead_ivsize(aead);
2541
2542 if (req->cryptlen < ivsize)
2543 return -EINVAL;
2544
2545 req->cryptlen -= ivsize;
2546 req->assoclen += ivsize;
2547
2548 return aead_decrypt(req);
2549 }
2550
2551 /*
2552 * allocate and map the ablkcipher extended descriptor for ablkcipher
2553 */
2554 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
2555 *req, int desc_bytes,
2556 bool *iv_contig_out)
2557 {
2558 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2559 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2560 struct device *jrdev = ctx->jrdev;
2561 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2562 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2563 GFP_KERNEL : GFP_ATOMIC;
2564 int src_nents, dst_nents = 0, sec4_sg_bytes;
2565 struct ablkcipher_edesc *edesc;
2566 dma_addr_t iv_dma = 0;
2567 bool iv_contig = false;
2568 int sgc;
2569 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2570 int sec4_sg_index;
2571
2572 src_nents = sg_count(req->src, req->nbytes);
2573
2574 if (req->dst != req->src)
2575 dst_nents = sg_count(req->dst, req->nbytes);
2576
2577 if (likely(req->src == req->dst)) {
2578 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2579 DMA_BIDIRECTIONAL);
2580 } else {
2581 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2582 DMA_TO_DEVICE);
2583 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2584 DMA_FROM_DEVICE);
2585 }
2586
2587 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
2588 if (dma_mapping_error(jrdev, iv_dma)) {
2589 dev_err(jrdev, "unable to map IV\n");
2590 return ERR_PTR(-ENOMEM);
2591 }
2592
2593 /*
2594 * Check if iv can be contiguous with source and destination.
2595 * If so, include it. If not, create scatterlist.
2596 */
2597 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
2598 iv_contig = true;
2599 else
2600 src_nents = src_nents ? : 1;
2601 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2602 sizeof(struct sec4_sg_entry);
2603
2604 /* allocate space for base edesc and hw desc commands, link tables */
2605 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2606 GFP_DMA | flags);
2607 if (!edesc) {
2608 dev_err(jrdev, "could not allocate extended descriptor\n");
2609 return ERR_PTR(-ENOMEM);
2610 }
2611
2612 edesc->src_nents = src_nents;
2613 edesc->dst_nents = dst_nents;
2614 edesc->sec4_sg_bytes = sec4_sg_bytes;
2615 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2616 desc_bytes;
2617
2618 sec4_sg_index = 0;
2619 if (!iv_contig) {
2620 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
2621 sg_to_sec4_sg_last(req->src, src_nents,
2622 edesc->sec4_sg + 1, 0);
2623 sec4_sg_index += 1 + src_nents;
2624 }
2625
2626 if (dst_nents) {
2627 sg_to_sec4_sg_last(req->dst, dst_nents,
2628 edesc->sec4_sg + sec4_sg_index, 0);
2629 }
2630
2631 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2632 sec4_sg_bytes, DMA_TO_DEVICE);
2633 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2634 dev_err(jrdev, "unable to map S/G table\n");
2635 return ERR_PTR(-ENOMEM);
2636 }
2637
2638 edesc->iv_dma = iv_dma;
2639
2640 #ifdef DEBUG
2641 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
2642 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2643 sec4_sg_bytes, 1);
2644 #endif
2645
2646 *iv_contig_out = iv_contig;
2647 return edesc;
2648 }
2649
2650 static int ablkcipher_encrypt(struct ablkcipher_request *req)
2651 {
2652 struct ablkcipher_edesc *edesc;
2653 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2654 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2655 struct device *jrdev = ctx->jrdev;
2656 bool iv_contig;
2657 u32 *desc;
2658 int ret = 0;
2659
2660 /* allocate extended descriptor */
2661 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2662 CAAM_CMD_SZ, &iv_contig);
2663 if (IS_ERR(edesc))
2664 return PTR_ERR(edesc);
2665
2666 /* Create and submit job descriptor*/
2667 init_ablkcipher_job(ctx->sh_desc_enc,
2668 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
2669 #ifdef DEBUG
2670 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2671 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2672 desc_bytes(edesc->hw_desc), 1);
2673 #endif
2674 desc = edesc->hw_desc;
2675 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2676
2677 if (!ret) {
2678 ret = -EINPROGRESS;
2679 } else {
2680 ablkcipher_unmap(jrdev, edesc, req);
2681 kfree(edesc);
2682 }
2683
2684 return ret;
2685 }
2686
2687 static int ablkcipher_decrypt(struct ablkcipher_request *req)
2688 {
2689 struct ablkcipher_edesc *edesc;
2690 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2691 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2692 struct device *jrdev = ctx->jrdev;
2693 bool iv_contig;
2694 u32 *desc;
2695 int ret = 0;
2696
2697 /* allocate extended descriptor */
2698 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2699 CAAM_CMD_SZ, &iv_contig);
2700 if (IS_ERR(edesc))
2701 return PTR_ERR(edesc);
2702
2703 /* Create and submit job descriptor*/
2704 init_ablkcipher_job(ctx->sh_desc_dec,
2705 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
2706 desc = edesc->hw_desc;
2707 #ifdef DEBUG
2708 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2709 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2710 desc_bytes(edesc->hw_desc), 1);
2711 #endif
2712
2713 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
2714 if (!ret) {
2715 ret = -EINPROGRESS;
2716 } else {
2717 ablkcipher_unmap(jrdev, edesc, req);
2718 kfree(edesc);
2719 }
2720
2721 return ret;
2722 }
2723
2724 /*
2725 * allocate and map the ablkcipher extended descriptor
2726 * for ablkcipher givencrypt
2727 */
2728 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
2729 struct skcipher_givcrypt_request *greq,
2730 int desc_bytes,
2731 bool *iv_contig_out)
2732 {
2733 struct ablkcipher_request *req = &greq->creq;
2734 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2735 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2736 struct device *jrdev = ctx->jrdev;
2737 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2738 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2739 GFP_KERNEL : GFP_ATOMIC;
2740 int src_nents, dst_nents = 0, sec4_sg_bytes;
2741 struct ablkcipher_edesc *edesc;
2742 dma_addr_t iv_dma = 0;
2743 bool iv_contig = false;
2744 int sgc;
2745 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2746 int sec4_sg_index;
2747
2748 src_nents = sg_count(req->src, req->nbytes);
2749
2750 if (unlikely(req->dst != req->src))
2751 dst_nents = sg_count(req->dst, req->nbytes);
2752
2753 if (likely(req->src == req->dst)) {
2754 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2755 DMA_BIDIRECTIONAL);
2756 } else {
2757 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2758 DMA_TO_DEVICE);
2759 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2760 DMA_FROM_DEVICE);
2761 }
2762
2763 /*
2764 * Check if iv can be contiguous with source and destination.
2765 * If so, include it. If not, create scatterlist.
2766 */
2767 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
2768 if (dma_mapping_error(jrdev, iv_dma)) {
2769 dev_err(jrdev, "unable to map IV\n");
2770 return ERR_PTR(-ENOMEM);
2771 }
2772
2773 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
2774 iv_contig = true;
2775 else
2776 dst_nents = dst_nents ? : 1;
2777 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2778 sizeof(struct sec4_sg_entry);
2779
2780 /* allocate space for base edesc and hw desc commands, link tables */
2781 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2782 GFP_DMA | flags);
2783 if (!edesc) {
2784 dev_err(jrdev, "could not allocate extended descriptor\n");
2785 return ERR_PTR(-ENOMEM);
2786 }
2787
2788 edesc->src_nents = src_nents;
2789 edesc->dst_nents = dst_nents;
2790 edesc->sec4_sg_bytes = sec4_sg_bytes;
2791 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2792 desc_bytes;
2793
2794 sec4_sg_index = 0;
2795 if (src_nents) {
2796 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
2797 sec4_sg_index += src_nents;
2798 }
2799
2800 if (!iv_contig) {
2801 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2802 iv_dma, ivsize, 0);
2803 sec4_sg_index += 1;
2804 sg_to_sec4_sg_last(req->dst, dst_nents,
2805 edesc->sec4_sg + sec4_sg_index, 0);
2806 }
2807
2808 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2809 sec4_sg_bytes, DMA_TO_DEVICE);
2810 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2811 dev_err(jrdev, "unable to map S/G table\n");
2812 return ERR_PTR(-ENOMEM);
2813 }
2814 edesc->iv_dma = iv_dma;
2815
2816 #ifdef DEBUG
2817 print_hex_dump(KERN_ERR,
2818 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
2819 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2820 sec4_sg_bytes, 1);
2821 #endif
2822
2823 *iv_contig_out = iv_contig;
2824 return edesc;
2825 }
2826
2827 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
2828 {
2829 struct ablkcipher_request *req = &creq->creq;
2830 struct ablkcipher_edesc *edesc;
2831 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2832 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2833 struct device *jrdev = ctx->jrdev;
2834 bool iv_contig;
2835 u32 *desc;
2836 int ret = 0;
2837
2838 /* allocate extended descriptor */
2839 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
2840 CAAM_CMD_SZ, &iv_contig);
2841 if (IS_ERR(edesc))
2842 return PTR_ERR(edesc);
2843
2844 /* Create and submit job descriptor*/
2845 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
2846 edesc, req, iv_contig);
2847 #ifdef DEBUG
2848 print_hex_dump(KERN_ERR,
2849 "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
2850 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2851 desc_bytes(edesc->hw_desc), 1);
2852 #endif
2853 desc = edesc->hw_desc;
2854 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2855
2856 if (!ret) {
2857 ret = -EINPROGRESS;
2858 } else {
2859 ablkcipher_unmap(jrdev, edesc, req);
2860 kfree(edesc);
2861 }
2862
2863 return ret;
2864 }
2865
2866 #define template_aead template_u.aead
2867 #define template_ablkcipher template_u.ablkcipher
2868 struct caam_alg_template {
2869 char name[CRYPTO_MAX_ALG_NAME];
2870 char driver_name[CRYPTO_MAX_ALG_NAME];
2871 unsigned int blocksize;
2872 u32 type;
2873 union {
2874 struct ablkcipher_alg ablkcipher;
2875 } template_u;
2876 u32 class1_alg_type;
2877 u32 class2_alg_type;
2878 u32 alg_op;
2879 };
2880
2881 static struct caam_alg_template driver_algs[] = {
2882 /* ablkcipher descriptor */
2883 {
2884 .name = "cbc(aes)",
2885 .driver_name = "cbc-aes-caam",
2886 .blocksize = AES_BLOCK_SIZE,
2887 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2888 .template_ablkcipher = {
2889 .setkey = ablkcipher_setkey,
2890 .encrypt = ablkcipher_encrypt,
2891 .decrypt = ablkcipher_decrypt,
2892 .givencrypt = ablkcipher_givencrypt,
2893 .geniv = "<built-in>",
2894 .min_keysize = AES_MIN_KEY_SIZE,
2895 .max_keysize = AES_MAX_KEY_SIZE,
2896 .ivsize = AES_BLOCK_SIZE,
2897 },
2898 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2899 },
2900 {
2901 .name = "cbc(des3_ede)",
2902 .driver_name = "cbc-3des-caam",
2903 .blocksize = DES3_EDE_BLOCK_SIZE,
2904 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2905 .template_ablkcipher = {
2906 .setkey = ablkcipher_setkey,
2907 .encrypt = ablkcipher_encrypt,
2908 .decrypt = ablkcipher_decrypt,
2909 .givencrypt = ablkcipher_givencrypt,
2910 .geniv = "<built-in>",
2911 .min_keysize = DES3_EDE_KEY_SIZE,
2912 .max_keysize = DES3_EDE_KEY_SIZE,
2913 .ivsize = DES3_EDE_BLOCK_SIZE,
2914 },
2915 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2916 },
2917 {
2918 .name = "cbc(des)",
2919 .driver_name = "cbc-des-caam",
2920 .blocksize = DES_BLOCK_SIZE,
2921 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2922 .template_ablkcipher = {
2923 .setkey = ablkcipher_setkey,
2924 .encrypt = ablkcipher_encrypt,
2925 .decrypt = ablkcipher_decrypt,
2926 .givencrypt = ablkcipher_givencrypt,
2927 .geniv = "<built-in>",
2928 .min_keysize = DES_KEY_SIZE,
2929 .max_keysize = DES_KEY_SIZE,
2930 .ivsize = DES_BLOCK_SIZE,
2931 },
2932 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2933 },
2934 {
2935 .name = "ctr(aes)",
2936 .driver_name = "ctr-aes-caam",
2937 .blocksize = 1,
2938 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2939 .template_ablkcipher = {
2940 .setkey = ablkcipher_setkey,
2941 .encrypt = ablkcipher_encrypt,
2942 .decrypt = ablkcipher_decrypt,
2943 .geniv = "chainiv",
2944 .min_keysize = AES_MIN_KEY_SIZE,
2945 .max_keysize = AES_MAX_KEY_SIZE,
2946 .ivsize = AES_BLOCK_SIZE,
2947 },
2948 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2949 },
2950 {
2951 .name = "rfc3686(ctr(aes))",
2952 .driver_name = "rfc3686-ctr-aes-caam",
2953 .blocksize = 1,
2954 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2955 .template_ablkcipher = {
2956 .setkey = ablkcipher_setkey,
2957 .encrypt = ablkcipher_encrypt,
2958 .decrypt = ablkcipher_decrypt,
2959 .givencrypt = ablkcipher_givencrypt,
2960 .geniv = "<built-in>",
2961 .min_keysize = AES_MIN_KEY_SIZE +
2962 CTR_RFC3686_NONCE_SIZE,
2963 .max_keysize = AES_MAX_KEY_SIZE +
2964 CTR_RFC3686_NONCE_SIZE,
2965 .ivsize = CTR_RFC3686_IV_SIZE,
2966 },
2967 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2968 },
2969 {
2970 .name = "xts(aes)",
2971 .driver_name = "xts-aes-caam",
2972 .blocksize = AES_BLOCK_SIZE,
2973 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2974 .template_ablkcipher = {
2975 .setkey = xts_ablkcipher_setkey,
2976 .encrypt = ablkcipher_encrypt,
2977 .decrypt = ablkcipher_decrypt,
2978 .geniv = "eseqiv",
2979 .min_keysize = 2 * AES_MIN_KEY_SIZE,
2980 .max_keysize = 2 * AES_MAX_KEY_SIZE,
2981 .ivsize = AES_BLOCK_SIZE,
2982 },
2983 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
2984 },
2985 };
2986
2987 static struct caam_aead_alg driver_aeads[] = {
2988 {
2989 .aead = {
2990 .base = {
2991 .cra_name = "rfc4106(gcm(aes))",
2992 .cra_driver_name = "rfc4106-gcm-aes-caam",
2993 .cra_blocksize = 1,
2994 },
2995 .setkey = rfc4106_setkey,
2996 .setauthsize = rfc4106_setauthsize,
2997 .encrypt = ipsec_gcm_encrypt,
2998 .decrypt = ipsec_gcm_decrypt,
2999 .ivsize = 8,
3000 .maxauthsize = AES_BLOCK_SIZE,
3001 },
3002 .caam = {
3003 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3004 },
3005 },
3006 {
3007 .aead = {
3008 .base = {
3009 .cra_name = "rfc4543(gcm(aes))",
3010 .cra_driver_name = "rfc4543-gcm-aes-caam",
3011 .cra_blocksize = 1,
3012 },
3013 .setkey = rfc4543_setkey,
3014 .setauthsize = rfc4543_setauthsize,
3015 .encrypt = ipsec_gcm_encrypt,
3016 .decrypt = ipsec_gcm_decrypt,
3017 .ivsize = 8,
3018 .maxauthsize = AES_BLOCK_SIZE,
3019 },
3020 .caam = {
3021 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3022 },
3023 },
3024 /* Galois Counter Mode */
3025 {
3026 .aead = {
3027 .base = {
3028 .cra_name = "gcm(aes)",
3029 .cra_driver_name = "gcm-aes-caam",
3030 .cra_blocksize = 1,
3031 },
3032 .setkey = gcm_setkey,
3033 .setauthsize = gcm_setauthsize,
3034 .encrypt = gcm_encrypt,
3035 .decrypt = gcm_decrypt,
3036 .ivsize = 12,
3037 .maxauthsize = AES_BLOCK_SIZE,
3038 },
3039 .caam = {
3040 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3041 },
3042 },
3043 /* single-pass ipsec_esp descriptor */
3044 {
3045 .aead = {
3046 .base = {
3047 .cra_name = "authenc(hmac(md5),"
3048 "ecb(cipher_null))",
3049 .cra_driver_name = "authenc-hmac-md5-"
3050 "ecb-cipher_null-caam",
3051 .cra_blocksize = NULL_BLOCK_SIZE,
3052 },
3053 .setkey = aead_setkey,
3054 .setauthsize = aead_setauthsize,
3055 .encrypt = aead_encrypt,
3056 .decrypt = aead_decrypt,
3057 .ivsize = NULL_IV_SIZE,
3058 .maxauthsize = MD5_DIGEST_SIZE,
3059 },
3060 .caam = {
3061 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3062 OP_ALG_AAI_HMAC_PRECOMP,
3063 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3064 },
3065 },
3066 {
3067 .aead = {
3068 .base = {
3069 .cra_name = "authenc(hmac(sha1),"
3070 "ecb(cipher_null))",
3071 .cra_driver_name = "authenc-hmac-sha1-"
3072 "ecb-cipher_null-caam",
3073 .cra_blocksize = NULL_BLOCK_SIZE,
3074 },
3075 .setkey = aead_setkey,
3076 .setauthsize = aead_setauthsize,
3077 .encrypt = aead_encrypt,
3078 .decrypt = aead_decrypt,
3079 .ivsize = NULL_IV_SIZE,
3080 .maxauthsize = SHA1_DIGEST_SIZE,
3081 },
3082 .caam = {
3083 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3084 OP_ALG_AAI_HMAC_PRECOMP,
3085 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3086 },
3087 },
3088 {
3089 .aead = {
3090 .base = {
3091 .cra_name = "authenc(hmac(sha224),"
3092 "ecb(cipher_null))",
3093 .cra_driver_name = "authenc-hmac-sha224-"
3094 "ecb-cipher_null-caam",
3095 .cra_blocksize = NULL_BLOCK_SIZE,
3096 },
3097 .setkey = aead_setkey,
3098 .setauthsize = aead_setauthsize,
3099 .encrypt = aead_encrypt,
3100 .decrypt = aead_decrypt,
3101 .ivsize = NULL_IV_SIZE,
3102 .maxauthsize = SHA224_DIGEST_SIZE,
3103 },
3104 .caam = {
3105 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3106 OP_ALG_AAI_HMAC_PRECOMP,
3107 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3108 },
3109 },
3110 {
3111 .aead = {
3112 .base = {
3113 .cra_name = "authenc(hmac(sha256),"
3114 "ecb(cipher_null))",
3115 .cra_driver_name = "authenc-hmac-sha256-"
3116 "ecb-cipher_null-caam",
3117 .cra_blocksize = NULL_BLOCK_SIZE,
3118 },
3119 .setkey = aead_setkey,
3120 .setauthsize = aead_setauthsize,
3121 .encrypt = aead_encrypt,
3122 .decrypt = aead_decrypt,
3123 .ivsize = NULL_IV_SIZE,
3124 .maxauthsize = SHA256_DIGEST_SIZE,
3125 },
3126 .caam = {
3127 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3128 OP_ALG_AAI_HMAC_PRECOMP,
3129 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3130 },
3131 },
3132 {
3133 .aead = {
3134 .base = {
3135 .cra_name = "authenc(hmac(sha384),"
3136 "ecb(cipher_null))",
3137 .cra_driver_name = "authenc-hmac-sha384-"
3138 "ecb-cipher_null-caam",
3139 .cra_blocksize = NULL_BLOCK_SIZE,
3140 },
3141 .setkey = aead_setkey,
3142 .setauthsize = aead_setauthsize,
3143 .encrypt = aead_encrypt,
3144 .decrypt = aead_decrypt,
3145 .ivsize = NULL_IV_SIZE,
3146 .maxauthsize = SHA384_DIGEST_SIZE,
3147 },
3148 .caam = {
3149 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3150 OP_ALG_AAI_HMAC_PRECOMP,
3151 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3152 },
3153 },
3154 {
3155 .aead = {
3156 .base = {
3157 .cra_name = "authenc(hmac(sha512),"
3158 "ecb(cipher_null))",
3159 .cra_driver_name = "authenc-hmac-sha512-"
3160 "ecb-cipher_null-caam",
3161 .cra_blocksize = NULL_BLOCK_SIZE,
3162 },
3163 .setkey = aead_setkey,
3164 .setauthsize = aead_setauthsize,
3165 .encrypt = aead_encrypt,
3166 .decrypt = aead_decrypt,
3167 .ivsize = NULL_IV_SIZE,
3168 .maxauthsize = SHA512_DIGEST_SIZE,
3169 },
3170 .caam = {
3171 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3172 OP_ALG_AAI_HMAC_PRECOMP,
3173 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3174 },
3175 },
3176 {
3177 .aead = {
3178 .base = {
3179 .cra_name = "authenc(hmac(md5),cbc(aes))",
3180 .cra_driver_name = "authenc-hmac-md5-"
3181 "cbc-aes-caam",
3182 .cra_blocksize = AES_BLOCK_SIZE,
3183 },
3184 .setkey = aead_setkey,
3185 .setauthsize = aead_setauthsize,
3186 .encrypt = aead_encrypt,
3187 .decrypt = aead_decrypt,
3188 .ivsize = AES_BLOCK_SIZE,
3189 .maxauthsize = MD5_DIGEST_SIZE,
3190 },
3191 .caam = {
3192 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3193 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3194 OP_ALG_AAI_HMAC_PRECOMP,
3195 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3196 },
3197 },
3198 {
3199 .aead = {
3200 .base = {
3201 .cra_name = "echainiv(authenc(hmac(md5),"
3202 "cbc(aes)))",
3203 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3204 "cbc-aes-caam",
3205 .cra_blocksize = AES_BLOCK_SIZE,
3206 },
3207 .setkey = aead_setkey,
3208 .setauthsize = aead_setauthsize,
3209 .encrypt = aead_encrypt,
3210 .decrypt = aead_givdecrypt,
3211 .ivsize = AES_BLOCK_SIZE,
3212 .maxauthsize = MD5_DIGEST_SIZE,
3213 },
3214 .caam = {
3215 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3216 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3217 OP_ALG_AAI_HMAC_PRECOMP,
3218 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3219 .geniv = true,
3220 },
3221 },
3222 {
3223 .aead = {
3224 .base = {
3225 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3226 .cra_driver_name = "authenc-hmac-sha1-"
3227 "cbc-aes-caam",
3228 .cra_blocksize = AES_BLOCK_SIZE,
3229 },
3230 .setkey = aead_setkey,
3231 .setauthsize = aead_setauthsize,
3232 .encrypt = aead_encrypt,
3233 .decrypt = aead_decrypt,
3234 .ivsize = AES_BLOCK_SIZE,
3235 .maxauthsize = SHA1_DIGEST_SIZE,
3236 },
3237 .caam = {
3238 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3239 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3240 OP_ALG_AAI_HMAC_PRECOMP,
3241 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3242 },
3243 },
3244 {
3245 .aead = {
3246 .base = {
3247 .cra_name = "echainiv(authenc(hmac(sha1),"
3248 "cbc(aes)))",
3249 .cra_driver_name = "echainiv-authenc-"
3250 "hmac-sha1-cbc-aes-caam",
3251 .cra_blocksize = AES_BLOCK_SIZE,
3252 },
3253 .setkey = aead_setkey,
3254 .setauthsize = aead_setauthsize,
3255 .encrypt = aead_encrypt,
3256 .decrypt = aead_givdecrypt,
3257 .ivsize = AES_BLOCK_SIZE,
3258 .maxauthsize = SHA1_DIGEST_SIZE,
3259 },
3260 .caam = {
3261 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3262 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3263 OP_ALG_AAI_HMAC_PRECOMP,
3264 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3265 .geniv = true,
3266 },
3267 },
3268 {
3269 .aead = {
3270 .base = {
3271 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3272 .cra_driver_name = "authenc-hmac-sha224-"
3273 "cbc-aes-caam",
3274 .cra_blocksize = AES_BLOCK_SIZE,
3275 },
3276 .setkey = aead_setkey,
3277 .setauthsize = aead_setauthsize,
3278 .encrypt = aead_encrypt,
3279 .decrypt = aead_decrypt,
3280 .ivsize = AES_BLOCK_SIZE,
3281 .maxauthsize = SHA224_DIGEST_SIZE,
3282 },
3283 .caam = {
3284 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3285 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3286 OP_ALG_AAI_HMAC_PRECOMP,
3287 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3288 },
3289 },
3290 {
3291 .aead = {
3292 .base = {
3293 .cra_name = "echainiv(authenc(hmac(sha224),"
3294 "cbc(aes)))",
3295 .cra_driver_name = "echainiv-authenc-"
3296 "hmac-sha224-cbc-aes-caam",
3297 .cra_blocksize = AES_BLOCK_SIZE,
3298 },
3299 .setkey = aead_setkey,
3300 .setauthsize = aead_setauthsize,
3301 .encrypt = aead_encrypt,
3302 .decrypt = aead_givdecrypt,
3303 .ivsize = AES_BLOCK_SIZE,
3304 .maxauthsize = SHA224_DIGEST_SIZE,
3305 },
3306 .caam = {
3307 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3308 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3309 OP_ALG_AAI_HMAC_PRECOMP,
3310 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3311 .geniv = true,
3312 },
3313 },
3314 {
3315 .aead = {
3316 .base = {
3317 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3318 .cra_driver_name = "authenc-hmac-sha256-"
3319 "cbc-aes-caam",
3320 .cra_blocksize = AES_BLOCK_SIZE,
3321 },
3322 .setkey = aead_setkey,
3323 .setauthsize = aead_setauthsize,
3324 .encrypt = aead_encrypt,
3325 .decrypt = aead_decrypt,
3326 .ivsize = AES_BLOCK_SIZE,
3327 .maxauthsize = SHA256_DIGEST_SIZE,
3328 },
3329 .caam = {
3330 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3331 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3332 OP_ALG_AAI_HMAC_PRECOMP,
3333 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3334 },
3335 },
3336 {
3337 .aead = {
3338 .base = {
3339 .cra_name = "echainiv(authenc(hmac(sha256),"
3340 "cbc(aes)))",
3341 .cra_driver_name = "echainiv-authenc-"
3342 "hmac-sha256-cbc-aes-caam",
3343 .cra_blocksize = AES_BLOCK_SIZE,
3344 },
3345 .setkey = aead_setkey,
3346 .setauthsize = aead_setauthsize,
3347 .encrypt = aead_encrypt,
3348 .decrypt = aead_givdecrypt,
3349 .ivsize = AES_BLOCK_SIZE,
3350 .maxauthsize = SHA256_DIGEST_SIZE,
3351 },
3352 .caam = {
3353 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3354 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3355 OP_ALG_AAI_HMAC_PRECOMP,
3356 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3357 .geniv = true,
3358 },
3359 },
3360 {
3361 .aead = {
3362 .base = {
3363 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3364 .cra_driver_name = "authenc-hmac-sha384-"
3365 "cbc-aes-caam",
3366 .cra_blocksize = AES_BLOCK_SIZE,
3367 },
3368 .setkey = aead_setkey,
3369 .setauthsize = aead_setauthsize,
3370 .encrypt = aead_encrypt,
3371 .decrypt = aead_decrypt,
3372 .ivsize = AES_BLOCK_SIZE,
3373 .maxauthsize = SHA384_DIGEST_SIZE,
3374 },
3375 .caam = {
3376 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3377 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3378 OP_ALG_AAI_HMAC_PRECOMP,
3379 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3380 },
3381 },
3382 {
3383 .aead = {
3384 .base = {
3385 .cra_name = "echainiv(authenc(hmac(sha384),"
3386 "cbc(aes)))",
3387 .cra_driver_name = "echainiv-authenc-"
3388 "hmac-sha384-cbc-aes-caam",
3389 .cra_blocksize = AES_BLOCK_SIZE,
3390 },
3391 .setkey = aead_setkey,
3392 .setauthsize = aead_setauthsize,
3393 .encrypt = aead_encrypt,
3394 .decrypt = aead_givdecrypt,
3395 .ivsize = AES_BLOCK_SIZE,
3396 .maxauthsize = SHA384_DIGEST_SIZE,
3397 },
3398 .caam = {
3399 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3400 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3401 OP_ALG_AAI_HMAC_PRECOMP,
3402 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3403 .geniv = true,
3404 },
3405 },
3406 {
3407 .aead = {
3408 .base = {
3409 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3410 .cra_driver_name = "authenc-hmac-sha512-"
3411 "cbc-aes-caam",
3412 .cra_blocksize = AES_BLOCK_SIZE,
3413 },
3414 .setkey = aead_setkey,
3415 .setauthsize = aead_setauthsize,
3416 .encrypt = aead_encrypt,
3417 .decrypt = aead_decrypt,
3418 .ivsize = AES_BLOCK_SIZE,
3419 .maxauthsize = SHA512_DIGEST_SIZE,
3420 },
3421 .caam = {
3422 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3423 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3424 OP_ALG_AAI_HMAC_PRECOMP,
3425 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3426 },
3427 },
3428 {
3429 .aead = {
3430 .base = {
3431 .cra_name = "echainiv(authenc(hmac(sha512),"
3432 "cbc(aes)))",
3433 .cra_driver_name = "echainiv-authenc-"
3434 "hmac-sha512-cbc-aes-caam",
3435 .cra_blocksize = AES_BLOCK_SIZE,
3436 },
3437 .setkey = aead_setkey,
3438 .setauthsize = aead_setauthsize,
3439 .encrypt = aead_encrypt,
3440 .decrypt = aead_givdecrypt,
3441 .ivsize = AES_BLOCK_SIZE,
3442 .maxauthsize = SHA512_DIGEST_SIZE,
3443 },
3444 .caam = {
3445 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3446 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3447 OP_ALG_AAI_HMAC_PRECOMP,
3448 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3449 .geniv = true,
3450 },
3451 },
3452 {
3453 .aead = {
3454 .base = {
3455 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3456 .cra_driver_name = "authenc-hmac-md5-"
3457 "cbc-des3_ede-caam",
3458 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3459 },
3460 .setkey = aead_setkey,
3461 .setauthsize = aead_setauthsize,
3462 .encrypt = aead_encrypt,
3463 .decrypt = aead_decrypt,
3464 .ivsize = DES3_EDE_BLOCK_SIZE,
3465 .maxauthsize = MD5_DIGEST_SIZE,
3466 },
3467 .caam = {
3468 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3469 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3470 OP_ALG_AAI_HMAC_PRECOMP,
3471 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3472 }
3473 },
3474 {
3475 .aead = {
3476 .base = {
3477 .cra_name = "echainiv(authenc(hmac(md5),"
3478 "cbc(des3_ede)))",
3479 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3480 "cbc-des3_ede-caam",
3481 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3482 },
3483 .setkey = aead_setkey,
3484 .setauthsize = aead_setauthsize,
3485 .encrypt = aead_encrypt,
3486 .decrypt = aead_givdecrypt,
3487 .ivsize = DES3_EDE_BLOCK_SIZE,
3488 .maxauthsize = MD5_DIGEST_SIZE,
3489 },
3490 .caam = {
3491 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3492 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3493 OP_ALG_AAI_HMAC_PRECOMP,
3494 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3495 .geniv = true,
3496 }
3497 },
3498 {
3499 .aead = {
3500 .base = {
3501 .cra_name = "authenc(hmac(sha1),"
3502 "cbc(des3_ede))",
3503 .cra_driver_name = "authenc-hmac-sha1-"
3504 "cbc-des3_ede-caam",
3505 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3506 },
3507 .setkey = aead_setkey,
3508 .setauthsize = aead_setauthsize,
3509 .encrypt = aead_encrypt,
3510 .decrypt = aead_decrypt,
3511 .ivsize = DES3_EDE_BLOCK_SIZE,
3512 .maxauthsize = SHA1_DIGEST_SIZE,
3513 },
3514 .caam = {
3515 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3516 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3517 OP_ALG_AAI_HMAC_PRECOMP,
3518 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3519 },
3520 },
3521 {
3522 .aead = {
3523 .base = {
3524 .cra_name = "echainiv(authenc(hmac(sha1),"
3525 "cbc(des3_ede)))",
3526 .cra_driver_name = "echainiv-authenc-"
3527 "hmac-sha1-"
3528 "cbc-des3_ede-caam",
3529 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3530 },
3531 .setkey = aead_setkey,
3532 .setauthsize = aead_setauthsize,
3533 .encrypt = aead_encrypt,
3534 .decrypt = aead_givdecrypt,
3535 .ivsize = DES3_EDE_BLOCK_SIZE,
3536 .maxauthsize = SHA1_DIGEST_SIZE,
3537 },
3538 .caam = {
3539 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3540 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3541 OP_ALG_AAI_HMAC_PRECOMP,
3542 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3543 .geniv = true,
3544 },
3545 },
3546 {
3547 .aead = {
3548 .base = {
3549 .cra_name = "authenc(hmac(sha224),"
3550 "cbc(des3_ede))",
3551 .cra_driver_name = "authenc-hmac-sha224-"
3552 "cbc-des3_ede-caam",
3553 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3554 },
3555 .setkey = aead_setkey,
3556 .setauthsize = aead_setauthsize,
3557 .encrypt = aead_encrypt,
3558 .decrypt = aead_decrypt,
3559 .ivsize = DES3_EDE_BLOCK_SIZE,
3560 .maxauthsize = SHA224_DIGEST_SIZE,
3561 },
3562 .caam = {
3563 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3564 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3565 OP_ALG_AAI_HMAC_PRECOMP,
3566 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3567 },
3568 },
3569 {
3570 .aead = {
3571 .base = {
3572 .cra_name = "echainiv(authenc(hmac(sha224),"
3573 "cbc(des3_ede)))",
3574 .cra_driver_name = "echainiv-authenc-"
3575 "hmac-sha224-"
3576 "cbc-des3_ede-caam",
3577 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3578 },
3579 .setkey = aead_setkey,
3580 .setauthsize = aead_setauthsize,
3581 .encrypt = aead_encrypt,
3582 .decrypt = aead_givdecrypt,
3583 .ivsize = DES3_EDE_BLOCK_SIZE,
3584 .maxauthsize = SHA224_DIGEST_SIZE,
3585 },
3586 .caam = {
3587 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3588 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3589 OP_ALG_AAI_HMAC_PRECOMP,
3590 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3591 .geniv = true,
3592 },
3593 },
3594 {
3595 .aead = {
3596 .base = {
3597 .cra_name = "authenc(hmac(sha256),"
3598 "cbc(des3_ede))",
3599 .cra_driver_name = "authenc-hmac-sha256-"
3600 "cbc-des3_ede-caam",
3601 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3602 },
3603 .setkey = aead_setkey,
3604 .setauthsize = aead_setauthsize,
3605 .encrypt = aead_encrypt,
3606 .decrypt = aead_decrypt,
3607 .ivsize = DES3_EDE_BLOCK_SIZE,
3608 .maxauthsize = SHA256_DIGEST_SIZE,
3609 },
3610 .caam = {
3611 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3612 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3613 OP_ALG_AAI_HMAC_PRECOMP,
3614 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3615 },
3616 },
3617 {
3618 .aead = {
3619 .base = {
3620 .cra_name = "echainiv(authenc(hmac(sha256),"
3621 "cbc(des3_ede)))",
3622 .cra_driver_name = "echainiv-authenc-"
3623 "hmac-sha256-"
3624 "cbc-des3_ede-caam",
3625 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3626 },
3627 .setkey = aead_setkey,
3628 .setauthsize = aead_setauthsize,
3629 .encrypt = aead_encrypt,
3630 .decrypt = aead_givdecrypt,
3631 .ivsize = DES3_EDE_BLOCK_SIZE,
3632 .maxauthsize = SHA256_DIGEST_SIZE,
3633 },
3634 .caam = {
3635 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3636 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3637 OP_ALG_AAI_HMAC_PRECOMP,
3638 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3639 .geniv = true,
3640 },
3641 },
3642 {
3643 .aead = {
3644 .base = {
3645 .cra_name = "authenc(hmac(sha384),"
3646 "cbc(des3_ede))",
3647 .cra_driver_name = "authenc-hmac-sha384-"
3648 "cbc-des3_ede-caam",
3649 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3650 },
3651 .setkey = aead_setkey,
3652 .setauthsize = aead_setauthsize,
3653 .encrypt = aead_encrypt,
3654 .decrypt = aead_decrypt,
3655 .ivsize = DES3_EDE_BLOCK_SIZE,
3656 .maxauthsize = SHA384_DIGEST_SIZE,
3657 },
3658 .caam = {
3659 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3660 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3661 OP_ALG_AAI_HMAC_PRECOMP,
3662 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3663 },
3664 },
3665 {
3666 .aead = {
3667 .base = {
3668 .cra_name = "echainiv(authenc(hmac(sha384),"
3669 "cbc(des3_ede)))",
3670 .cra_driver_name = "echainiv-authenc-"
3671 "hmac-sha384-"
3672 "cbc-des3_ede-caam",
3673 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3674 },
3675 .setkey = aead_setkey,
3676 .setauthsize = aead_setauthsize,
3677 .encrypt = aead_encrypt,
3678 .decrypt = aead_givdecrypt,
3679 .ivsize = DES3_EDE_BLOCK_SIZE,
3680 .maxauthsize = SHA384_DIGEST_SIZE,
3681 },
3682 .caam = {
3683 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3684 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3685 OP_ALG_AAI_HMAC_PRECOMP,
3686 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3687 .geniv = true,
3688 },
3689 },
3690 {
3691 .aead = {
3692 .base = {
3693 .cra_name = "authenc(hmac(sha512),"
3694 "cbc(des3_ede))",
3695 .cra_driver_name = "authenc-hmac-sha512-"
3696 "cbc-des3_ede-caam",
3697 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3698 },
3699 .setkey = aead_setkey,
3700 .setauthsize = aead_setauthsize,
3701 .encrypt = aead_encrypt,
3702 .decrypt = aead_decrypt,
3703 .ivsize = DES3_EDE_BLOCK_SIZE,
3704 .maxauthsize = SHA512_DIGEST_SIZE,
3705 },
3706 .caam = {
3707 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3708 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3709 OP_ALG_AAI_HMAC_PRECOMP,
3710 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3711 },
3712 },
3713 {
3714 .aead = {
3715 .base = {
3716 .cra_name = "echainiv(authenc(hmac(sha512),"
3717 "cbc(des3_ede)))",
3718 .cra_driver_name = "echainiv-authenc-"
3719 "hmac-sha512-"
3720 "cbc-des3_ede-caam",
3721 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3722 },
3723 .setkey = aead_setkey,
3724 .setauthsize = aead_setauthsize,
3725 .encrypt = aead_encrypt,
3726 .decrypt = aead_givdecrypt,
3727 .ivsize = DES3_EDE_BLOCK_SIZE,
3728 .maxauthsize = SHA512_DIGEST_SIZE,
3729 },
3730 .caam = {
3731 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3732 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3733 OP_ALG_AAI_HMAC_PRECOMP,
3734 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3735 .geniv = true,
3736 },
3737 },
3738 {
3739 .aead = {
3740 .base = {
3741 .cra_name = "authenc(hmac(md5),cbc(des))",
3742 .cra_driver_name = "authenc-hmac-md5-"
3743 "cbc-des-caam",
3744 .cra_blocksize = DES_BLOCK_SIZE,
3745 },
3746 .setkey = aead_setkey,
3747 .setauthsize = aead_setauthsize,
3748 .encrypt = aead_encrypt,
3749 .decrypt = aead_decrypt,
3750 .ivsize = DES_BLOCK_SIZE,
3751 .maxauthsize = MD5_DIGEST_SIZE,
3752 },
3753 .caam = {
3754 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3755 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3756 OP_ALG_AAI_HMAC_PRECOMP,
3757 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3758 },
3759 },
3760 {
3761 .aead = {
3762 .base = {
3763 .cra_name = "echainiv(authenc(hmac(md5),"
3764 "cbc(des)))",
3765 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3766 "cbc-des-caam",
3767 .cra_blocksize = DES_BLOCK_SIZE,
3768 },
3769 .setkey = aead_setkey,
3770 .setauthsize = aead_setauthsize,
3771 .encrypt = aead_encrypt,
3772 .decrypt = aead_givdecrypt,
3773 .ivsize = DES_BLOCK_SIZE,
3774 .maxauthsize = MD5_DIGEST_SIZE,
3775 },
3776 .caam = {
3777 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3778 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3779 OP_ALG_AAI_HMAC_PRECOMP,
3780 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3781 .geniv = true,
3782 },
3783 },
3784 {
3785 .aead = {
3786 .base = {
3787 .cra_name = "authenc(hmac(sha1),cbc(des))",
3788 .cra_driver_name = "authenc-hmac-sha1-"
3789 "cbc-des-caam",
3790 .cra_blocksize = DES_BLOCK_SIZE,
3791 },
3792 .setkey = aead_setkey,
3793 .setauthsize = aead_setauthsize,
3794 .encrypt = aead_encrypt,
3795 .decrypt = aead_decrypt,
3796 .ivsize = DES_BLOCK_SIZE,
3797 .maxauthsize = SHA1_DIGEST_SIZE,
3798 },
3799 .caam = {
3800 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3801 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3802 OP_ALG_AAI_HMAC_PRECOMP,
3803 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3804 },
3805 },
3806 {
3807 .aead = {
3808 .base = {
3809 .cra_name = "echainiv(authenc(hmac(sha1),"
3810 "cbc(des)))",
3811 .cra_driver_name = "echainiv-authenc-"
3812 "hmac-sha1-cbc-des-caam",
3813 .cra_blocksize = DES_BLOCK_SIZE,
3814 },
3815 .setkey = aead_setkey,
3816 .setauthsize = aead_setauthsize,
3817 .encrypt = aead_encrypt,
3818 .decrypt = aead_givdecrypt,
3819 .ivsize = DES_BLOCK_SIZE,
3820 .maxauthsize = SHA1_DIGEST_SIZE,
3821 },
3822 .caam = {
3823 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3824 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3825 OP_ALG_AAI_HMAC_PRECOMP,
3826 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3827 .geniv = true,
3828 },
3829 },
3830 {
3831 .aead = {
3832 .base = {
3833 .cra_name = "authenc(hmac(sha224),cbc(des))",
3834 .cra_driver_name = "authenc-hmac-sha224-"
3835 "cbc-des-caam",
3836 .cra_blocksize = DES_BLOCK_SIZE,
3837 },
3838 .setkey = aead_setkey,
3839 .setauthsize = aead_setauthsize,
3840 .encrypt = aead_encrypt,
3841 .decrypt = aead_decrypt,
3842 .ivsize = DES_BLOCK_SIZE,
3843 .maxauthsize = SHA224_DIGEST_SIZE,
3844 },
3845 .caam = {
3846 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3847 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3848 OP_ALG_AAI_HMAC_PRECOMP,
3849 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3850 },
3851 },
3852 {
3853 .aead = {
3854 .base = {
3855 .cra_name = "echainiv(authenc(hmac(sha224),"
3856 "cbc(des)))",
3857 .cra_driver_name = "echainiv-authenc-"
3858 "hmac-sha224-cbc-des-caam",
3859 .cra_blocksize = DES_BLOCK_SIZE,
3860 },
3861 .setkey = aead_setkey,
3862 .setauthsize = aead_setauthsize,
3863 .encrypt = aead_encrypt,
3864 .decrypt = aead_givdecrypt,
3865 .ivsize = DES_BLOCK_SIZE,
3866 .maxauthsize = SHA224_DIGEST_SIZE,
3867 },
3868 .caam = {
3869 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3870 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3871 OP_ALG_AAI_HMAC_PRECOMP,
3872 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3873 .geniv = true,
3874 },
3875 },
3876 {
3877 .aead = {
3878 .base = {
3879 .cra_name = "authenc(hmac(sha256),cbc(des))",
3880 .cra_driver_name = "authenc-hmac-sha256-"
3881 "cbc-des-caam",
3882 .cra_blocksize = DES_BLOCK_SIZE,
3883 },
3884 .setkey = aead_setkey,
3885 .setauthsize = aead_setauthsize,
3886 .encrypt = aead_encrypt,
3887 .decrypt = aead_decrypt,
3888 .ivsize = DES_BLOCK_SIZE,
3889 .maxauthsize = SHA256_DIGEST_SIZE,
3890 },
3891 .caam = {
3892 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3893 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3894 OP_ALG_AAI_HMAC_PRECOMP,
3895 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3896 },
3897 },
3898 {
3899 .aead = {
3900 .base = {
3901 .cra_name = "echainiv(authenc(hmac(sha256),"
3902 "cbc(des)))",
3903 .cra_driver_name = "echainiv-authenc-"
3904 "hmac-sha256-cbc-des-caam",
3905 .cra_blocksize = DES_BLOCK_SIZE,
3906 },
3907 .setkey = aead_setkey,
3908 .setauthsize = aead_setauthsize,
3909 .encrypt = aead_encrypt,
3910 .decrypt = aead_givdecrypt,
3911 .ivsize = DES_BLOCK_SIZE,
3912 .maxauthsize = SHA256_DIGEST_SIZE,
3913 },
3914 .caam = {
3915 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3916 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3917 OP_ALG_AAI_HMAC_PRECOMP,
3918 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3919 .geniv = true,
3920 },
3921 },
3922 {
3923 .aead = {
3924 .base = {
3925 .cra_name = "authenc(hmac(sha384),cbc(des))",
3926 .cra_driver_name = "authenc-hmac-sha384-"
3927 "cbc-des-caam",
3928 .cra_blocksize = DES_BLOCK_SIZE,
3929 },
3930 .setkey = aead_setkey,
3931 .setauthsize = aead_setauthsize,
3932 .encrypt = aead_encrypt,
3933 .decrypt = aead_decrypt,
3934 .ivsize = DES_BLOCK_SIZE,
3935 .maxauthsize = SHA384_DIGEST_SIZE,
3936 },
3937 .caam = {
3938 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3939 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3940 OP_ALG_AAI_HMAC_PRECOMP,
3941 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3942 },
3943 },
3944 {
3945 .aead = {
3946 .base = {
3947 .cra_name = "echainiv(authenc(hmac(sha384),"
3948 "cbc(des)))",
3949 .cra_driver_name = "echainiv-authenc-"
3950 "hmac-sha384-cbc-des-caam",
3951 .cra_blocksize = DES_BLOCK_SIZE,
3952 },
3953 .setkey = aead_setkey,
3954 .setauthsize = aead_setauthsize,
3955 .encrypt = aead_encrypt,
3956 .decrypt = aead_givdecrypt,
3957 .ivsize = DES_BLOCK_SIZE,
3958 .maxauthsize = SHA384_DIGEST_SIZE,
3959 },
3960 .caam = {
3961 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3962 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3963 OP_ALG_AAI_HMAC_PRECOMP,
3964 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3965 .geniv = true,
3966 },
3967 },
3968 {
3969 .aead = {
3970 .base = {
3971 .cra_name = "authenc(hmac(sha512),cbc(des))",
3972 .cra_driver_name = "authenc-hmac-sha512-"
3973 "cbc-des-caam",
3974 .cra_blocksize = DES_BLOCK_SIZE,
3975 },
3976 .setkey = aead_setkey,
3977 .setauthsize = aead_setauthsize,
3978 .encrypt = aead_encrypt,
3979 .decrypt = aead_decrypt,
3980 .ivsize = DES_BLOCK_SIZE,
3981 .maxauthsize = SHA512_DIGEST_SIZE,
3982 },
3983 .caam = {
3984 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3985 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3986 OP_ALG_AAI_HMAC_PRECOMP,
3987 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3988 },
3989 },
3990 {
3991 .aead = {
3992 .base = {
3993 .cra_name = "echainiv(authenc(hmac(sha512),"
3994 "cbc(des)))",
3995 .cra_driver_name = "echainiv-authenc-"
3996 "hmac-sha512-cbc-des-caam",
3997 .cra_blocksize = DES_BLOCK_SIZE,
3998 },
3999 .setkey = aead_setkey,
4000 .setauthsize = aead_setauthsize,
4001 .encrypt = aead_encrypt,
4002 .decrypt = aead_givdecrypt,
4003 .ivsize = DES_BLOCK_SIZE,
4004 .maxauthsize = SHA512_DIGEST_SIZE,
4005 },
4006 .caam = {
4007 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
4008 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4009 OP_ALG_AAI_HMAC_PRECOMP,
4010 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4011 .geniv = true,
4012 },
4013 },
4014 {
4015 .aead = {
4016 .base = {
4017 .cra_name = "authenc(hmac(md5),"
4018 "rfc3686(ctr(aes)))",
4019 .cra_driver_name = "authenc-hmac-md5-"
4020 "rfc3686-ctr-aes-caam",
4021 .cra_blocksize = 1,
4022 },
4023 .setkey = aead_setkey,
4024 .setauthsize = aead_setauthsize,
4025 .encrypt = aead_encrypt,
4026 .decrypt = aead_decrypt,
4027 .ivsize = CTR_RFC3686_IV_SIZE,
4028 .maxauthsize = MD5_DIGEST_SIZE,
4029 },
4030 .caam = {
4031 .class1_alg_type = OP_ALG_ALGSEL_AES |
4032 OP_ALG_AAI_CTR_MOD128,
4033 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
4034 OP_ALG_AAI_HMAC_PRECOMP,
4035 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
4036 .rfc3686 = true,
4037 },
4038 },
4039 {
4040 .aead = {
4041 .base = {
4042 .cra_name = "seqiv(authenc("
4043 "hmac(md5),rfc3686(ctr(aes))))",
4044 .cra_driver_name = "seqiv-authenc-hmac-md5-"
4045 "rfc3686-ctr-aes-caam",
4046 .cra_blocksize = 1,
4047 },
4048 .setkey = aead_setkey,
4049 .setauthsize = aead_setauthsize,
4050 .encrypt = aead_encrypt,
4051 .decrypt = aead_givdecrypt,
4052 .ivsize = CTR_RFC3686_IV_SIZE,
4053 .maxauthsize = MD5_DIGEST_SIZE,
4054 },
4055 .caam = {
4056 .class1_alg_type = OP_ALG_ALGSEL_AES |
4057 OP_ALG_AAI_CTR_MOD128,
4058 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
4059 OP_ALG_AAI_HMAC_PRECOMP,
4060 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
4061 .rfc3686 = true,
4062 .geniv = true,
4063 },
4064 },
4065 {
4066 .aead = {
4067 .base = {
4068 .cra_name = "authenc(hmac(sha1),"
4069 "rfc3686(ctr(aes)))",
4070 .cra_driver_name = "authenc-hmac-sha1-"
4071 "rfc3686-ctr-aes-caam",
4072 .cra_blocksize = 1,
4073 },
4074 .setkey = aead_setkey,
4075 .setauthsize = aead_setauthsize,
4076 .encrypt = aead_encrypt,
4077 .decrypt = aead_decrypt,
4078 .ivsize = CTR_RFC3686_IV_SIZE,
4079 .maxauthsize = SHA1_DIGEST_SIZE,
4080 },
4081 .caam = {
4082 .class1_alg_type = OP_ALG_ALGSEL_AES |
4083 OP_ALG_AAI_CTR_MOD128,
4084 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4085 OP_ALG_AAI_HMAC_PRECOMP,
4086 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
4087 .rfc3686 = true,
4088 },
4089 },
4090 {
4091 .aead = {
4092 .base = {
4093 .cra_name = "seqiv(authenc("
4094 "hmac(sha1),rfc3686(ctr(aes))))",
4095 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
4096 "rfc3686-ctr-aes-caam",
4097 .cra_blocksize = 1,
4098 },
4099 .setkey = aead_setkey,
4100 .setauthsize = aead_setauthsize,
4101 .encrypt = aead_encrypt,
4102 .decrypt = aead_givdecrypt,
4103 .ivsize = CTR_RFC3686_IV_SIZE,
4104 .maxauthsize = SHA1_DIGEST_SIZE,
4105 },
4106 .caam = {
4107 .class1_alg_type = OP_ALG_ALGSEL_AES |
4108 OP_ALG_AAI_CTR_MOD128,
4109 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4110 OP_ALG_AAI_HMAC_PRECOMP,
4111 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
4112 .rfc3686 = true,
4113 .geniv = true,
4114 },
4115 },
4116 {
4117 .aead = {
4118 .base = {
4119 .cra_name = "authenc(hmac(sha224),"
4120 "rfc3686(ctr(aes)))",
4121 .cra_driver_name = "authenc-hmac-sha224-"
4122 "rfc3686-ctr-aes-caam",
4123 .cra_blocksize = 1,
4124 },
4125 .setkey = aead_setkey,
4126 .setauthsize = aead_setauthsize,
4127 .encrypt = aead_encrypt,
4128 .decrypt = aead_decrypt,
4129 .ivsize = CTR_RFC3686_IV_SIZE,
4130 .maxauthsize = SHA224_DIGEST_SIZE,
4131 },
4132 .caam = {
4133 .class1_alg_type = OP_ALG_ALGSEL_AES |
4134 OP_ALG_AAI_CTR_MOD128,
4135 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4136 OP_ALG_AAI_HMAC_PRECOMP,
4137 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4138 .rfc3686 = true,
4139 },
4140 },
4141 {
4142 .aead = {
4143 .base = {
4144 .cra_name = "seqiv(authenc("
4145 "hmac(sha224),rfc3686(ctr(aes))))",
4146 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
4147 "rfc3686-ctr-aes-caam",
4148 .cra_blocksize = 1,
4149 },
4150 .setkey = aead_setkey,
4151 .setauthsize = aead_setauthsize,
4152 .encrypt = aead_encrypt,
4153 .decrypt = aead_givdecrypt,
4154 .ivsize = CTR_RFC3686_IV_SIZE,
4155 .maxauthsize = SHA224_DIGEST_SIZE,
4156 },
4157 .caam = {
4158 .class1_alg_type = OP_ALG_ALGSEL_AES |
4159 OP_ALG_AAI_CTR_MOD128,
4160 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4161 OP_ALG_AAI_HMAC_PRECOMP,
4162 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4163 .rfc3686 = true,
4164 .geniv = true,
4165 },
4166 },
4167 {
4168 .aead = {
4169 .base = {
4170 .cra_name = "authenc(hmac(sha256),"
4171 "rfc3686(ctr(aes)))",
4172 .cra_driver_name = "authenc-hmac-sha256-"
4173 "rfc3686-ctr-aes-caam",
4174 .cra_blocksize = 1,
4175 },
4176 .setkey = aead_setkey,
4177 .setauthsize = aead_setauthsize,
4178 .encrypt = aead_encrypt,
4179 .decrypt = aead_decrypt,
4180 .ivsize = CTR_RFC3686_IV_SIZE,
4181 .maxauthsize = SHA256_DIGEST_SIZE,
4182 },
4183 .caam = {
4184 .class1_alg_type = OP_ALG_ALGSEL_AES |
4185 OP_ALG_AAI_CTR_MOD128,
4186 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4187 OP_ALG_AAI_HMAC_PRECOMP,
4188 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4189 .rfc3686 = true,
4190 },
4191 },
4192 {
4193 .aead = {
4194 .base = {
4195 .cra_name = "seqiv(authenc(hmac(sha256),"
4196 "rfc3686(ctr(aes))))",
4197 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
4198 "rfc3686-ctr-aes-caam",
4199 .cra_blocksize = 1,
4200 },
4201 .setkey = aead_setkey,
4202 .setauthsize = aead_setauthsize,
4203 .encrypt = aead_encrypt,
4204 .decrypt = aead_givdecrypt,
4205 .ivsize = CTR_RFC3686_IV_SIZE,
4206 .maxauthsize = SHA256_DIGEST_SIZE,
4207 },
4208 .caam = {
4209 .class1_alg_type = OP_ALG_ALGSEL_AES |
4210 OP_ALG_AAI_CTR_MOD128,
4211 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4212 OP_ALG_AAI_HMAC_PRECOMP,
4213 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4214 .rfc3686 = true,
4215 .geniv = true,
4216 },
4217 },
4218 {
4219 .aead = {
4220 .base = {
4221 .cra_name = "authenc(hmac(sha384),"
4222 "rfc3686(ctr(aes)))",
4223 .cra_driver_name = "authenc-hmac-sha384-"
4224 "rfc3686-ctr-aes-caam",
4225 .cra_blocksize = 1,
4226 },
4227 .setkey = aead_setkey,
4228 .setauthsize = aead_setauthsize,
4229 .encrypt = aead_encrypt,
4230 .decrypt = aead_decrypt,
4231 .ivsize = CTR_RFC3686_IV_SIZE,
4232 .maxauthsize = SHA384_DIGEST_SIZE,
4233 },
4234 .caam = {
4235 .class1_alg_type = OP_ALG_ALGSEL_AES |
4236 OP_ALG_AAI_CTR_MOD128,
4237 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4238 OP_ALG_AAI_HMAC_PRECOMP,
4239 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4240 .rfc3686 = true,
4241 },
4242 },
4243 {
4244 .aead = {
4245 .base = {
4246 .cra_name = "seqiv(authenc(hmac(sha384),"
4247 "rfc3686(ctr(aes))))",
4248 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
4249 "rfc3686-ctr-aes-caam",
4250 .cra_blocksize = 1,
4251 },
4252 .setkey = aead_setkey,
4253 .setauthsize = aead_setauthsize,
4254 .encrypt = aead_encrypt,
4255 .decrypt = aead_givdecrypt,
4256 .ivsize = CTR_RFC3686_IV_SIZE,
4257 .maxauthsize = SHA384_DIGEST_SIZE,
4258 },
4259 .caam = {
4260 .class1_alg_type = OP_ALG_ALGSEL_AES |
4261 OP_ALG_AAI_CTR_MOD128,
4262 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4263 OP_ALG_AAI_HMAC_PRECOMP,
4264 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4265 .rfc3686 = true,
4266 .geniv = true,
4267 },
4268 },
4269 {
4270 .aead = {
4271 .base = {
4272 .cra_name = "authenc(hmac(sha512),"
4273 "rfc3686(ctr(aes)))",
4274 .cra_driver_name = "authenc-hmac-sha512-"
4275 "rfc3686-ctr-aes-caam",
4276 .cra_blocksize = 1,
4277 },
4278 .setkey = aead_setkey,
4279 .setauthsize = aead_setauthsize,
4280 .encrypt = aead_encrypt,
4281 .decrypt = aead_decrypt,
4282 .ivsize = CTR_RFC3686_IV_SIZE,
4283 .maxauthsize = SHA512_DIGEST_SIZE,
4284 },
4285 .caam = {
4286 .class1_alg_type = OP_ALG_ALGSEL_AES |
4287 OP_ALG_AAI_CTR_MOD128,
4288 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4289 OP_ALG_AAI_HMAC_PRECOMP,
4290 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4291 .rfc3686 = true,
4292 },
4293 },
4294 {
4295 .aead = {
4296 .base = {
4297 .cra_name = "seqiv(authenc(hmac(sha512),"
4298 "rfc3686(ctr(aes))))",
4299 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
4300 "rfc3686-ctr-aes-caam",
4301 .cra_blocksize = 1,
4302 },
4303 .setkey = aead_setkey,
4304 .setauthsize = aead_setauthsize,
4305 .encrypt = aead_encrypt,
4306 .decrypt = aead_givdecrypt,
4307 .ivsize = CTR_RFC3686_IV_SIZE,
4308 .maxauthsize = SHA512_DIGEST_SIZE,
4309 },
4310 .caam = {
4311 .class1_alg_type = OP_ALG_ALGSEL_AES |
4312 OP_ALG_AAI_CTR_MOD128,
4313 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4314 OP_ALG_AAI_HMAC_PRECOMP,
4315 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4316 .rfc3686 = true,
4317 .geniv = true,
4318 },
4319 },
4320 };
4321
4322 struct caam_crypto_alg {
4323 struct crypto_alg crypto_alg;
4324 struct list_head entry;
4325 struct caam_alg_entry caam;
4326 };
4327
4328 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
4329 {
4330 ctx->jrdev = caam_jr_alloc();
4331 if (IS_ERR(ctx->jrdev)) {
4332 pr_err("Job Ring Device allocation for transform failed\n");
4333 return PTR_ERR(ctx->jrdev);
4334 }
4335
4336 /* copy descriptor header template value */
4337 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
4338 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
4339 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op;
4340
4341 return 0;
4342 }
4343
4344 static int caam_cra_init(struct crypto_tfm *tfm)
4345 {
4346 struct crypto_alg *alg = tfm->__crt_alg;
4347 struct caam_crypto_alg *caam_alg =
4348 container_of(alg, struct caam_crypto_alg, crypto_alg);
4349 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4350
4351 return caam_init_common(ctx, &caam_alg->caam);
4352 }
4353
4354 static int caam_aead_init(struct crypto_aead *tfm)
4355 {
4356 struct aead_alg *alg = crypto_aead_alg(tfm);
4357 struct caam_aead_alg *caam_alg =
4358 container_of(alg, struct caam_aead_alg, aead);
4359 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
4360
4361 return caam_init_common(ctx, &caam_alg->caam);
4362 }
4363
4364 static void caam_exit_common(struct caam_ctx *ctx)
4365 {
4366 if (ctx->sh_desc_enc_dma &&
4367 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4368 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4369 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4370 if (ctx->sh_desc_dec_dma &&
4371 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4372 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4373 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4374 if (ctx->sh_desc_givenc_dma &&
4375 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4376 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4377 desc_bytes(ctx->sh_desc_givenc),
4378 DMA_TO_DEVICE);
4379 if (ctx->key_dma &&
4380 !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4381 dma_unmap_single(ctx->jrdev, ctx->key_dma,
4382 ctx->enckeylen + ctx->split_key_pad_len,
4383 DMA_TO_DEVICE);
4384
4385 caam_jr_free(ctx->jrdev);
4386 }
4387
4388 static void caam_cra_exit(struct crypto_tfm *tfm)
4389 {
4390 caam_exit_common(crypto_tfm_ctx(tfm));
4391 }
4392
4393 static void caam_aead_exit(struct crypto_aead *tfm)
4394 {
4395 caam_exit_common(crypto_aead_ctx(tfm));
4396 }
4397
4398 static void __exit caam_algapi_exit(void)
4399 {
4400
4401 struct caam_crypto_alg *t_alg, *n;
4402 int i;
4403
4404 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4405 struct caam_aead_alg *t_alg = driver_aeads + i;
4406
4407 if (t_alg->registered)
4408 crypto_unregister_aead(&t_alg->aead);
4409 }
4410
4411 if (!alg_list.next)
4412 return;
4413
4414 list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
4415 crypto_unregister_alg(&t_alg->crypto_alg);
4416 list_del(&t_alg->entry);
4417 kfree(t_alg);
4418 }
4419 }
4420
4421 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
4422 *template)
4423 {
4424 struct caam_crypto_alg *t_alg;
4425 struct crypto_alg *alg;
4426
4427 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4428 if (!t_alg) {
4429 pr_err("failed to allocate t_alg\n");
4430 return ERR_PTR(-ENOMEM);
4431 }
4432
4433 alg = &t_alg->crypto_alg;
4434
4435 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4436 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4437 template->driver_name);
4438 alg->cra_module = THIS_MODULE;
4439 alg->cra_init = caam_cra_init;
4440 alg->cra_exit = caam_cra_exit;
4441 alg->cra_priority = CAAM_CRA_PRIORITY;
4442 alg->cra_blocksize = template->blocksize;
4443 alg->cra_alignmask = 0;
4444 alg->cra_ctxsize = sizeof(struct caam_ctx);
4445 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4446 template->type;
4447 switch (template->type) {
4448 case CRYPTO_ALG_TYPE_GIVCIPHER:
4449 alg->cra_type = &crypto_givcipher_type;
4450 alg->cra_ablkcipher = template->template_ablkcipher;
4451 break;
4452 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4453 alg->cra_type = &crypto_ablkcipher_type;
4454 alg->cra_ablkcipher = template->template_ablkcipher;
4455 break;
4456 }
4457
4458 t_alg->caam.class1_alg_type = template->class1_alg_type;
4459 t_alg->caam.class2_alg_type = template->class2_alg_type;
4460 t_alg->caam.alg_op = template->alg_op;
4461
4462 return t_alg;
4463 }
4464
4465 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
4466 {
4467 struct aead_alg *alg = &t_alg->aead;
4468
4469 alg->base.cra_module = THIS_MODULE;
4470 alg->base.cra_priority = CAAM_CRA_PRIORITY;
4471 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
4472 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
4473
4474 alg->init = caam_aead_init;
4475 alg->exit = caam_aead_exit;
4476 }
4477
4478 static int __init caam_algapi_init(void)
4479 {
4480 struct device_node *dev_node;
4481 struct platform_device *pdev;
4482 struct device *ctrldev;
4483 struct caam_drv_private *priv;
4484 int i = 0, err = 0;
4485 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
4486 unsigned int md_limit = SHA512_DIGEST_SIZE;
4487 bool registered = false;
4488
4489 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4490 if (!dev_node) {
4491 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4492 if (!dev_node)
4493 return -ENODEV;
4494 }
4495
4496 pdev = of_find_device_by_node(dev_node);
4497 if (!pdev) {
4498 of_node_put(dev_node);
4499 return -ENODEV;
4500 }
4501
4502 ctrldev = &pdev->dev;
4503 priv = dev_get_drvdata(ctrldev);
4504 of_node_put(dev_node);
4505
4506 /*
4507 * If priv is NULL, it's probably because the caam driver wasn't
4508 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4509 */
4510 if (!priv)
4511 return -ENODEV;
4512
4513
4514 INIT_LIST_HEAD(&alg_list);
4515
4516 /*
4517 * Register crypto algorithms the device supports.
4518 * First, detect presence and attributes of DES, AES, and MD blocks.
4519 */
4520 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
4521 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
4522 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
4523 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
4524 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4525
4526 /* If MD is present, limit digest size based on LP256 */
4527 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
4528 md_limit = SHA256_DIGEST_SIZE;
4529
4530 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4531 struct caam_crypto_alg *t_alg;
4532 struct caam_alg_template *alg = driver_algs + i;
4533 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
4534
4535 /* Skip DES algorithms if not supported by device */
4536 if (!des_inst &&
4537 ((alg_sel == OP_ALG_ALGSEL_3DES) ||
4538 (alg_sel == OP_ALG_ALGSEL_DES)))
4539 continue;
4540
4541 /* Skip AES algorithms if not supported by device */
4542 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
4543 continue;
4544
4545 t_alg = caam_alg_alloc(alg);
4546 if (IS_ERR(t_alg)) {
4547 err = PTR_ERR(t_alg);
4548 pr_warn("%s alg allocation failed\n", alg->driver_name);
4549 continue;
4550 }
4551
4552 err = crypto_register_alg(&t_alg->crypto_alg);
4553 if (err) {
4554 pr_warn("%s alg registration failed\n",
4555 t_alg->crypto_alg.cra_driver_name);
4556 kfree(t_alg);
4557 continue;
4558 }
4559
4560 list_add_tail(&t_alg->entry, &alg_list);
4561 registered = true;
4562 }
4563
4564 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4565 struct caam_aead_alg *t_alg = driver_aeads + i;
4566 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
4567 OP_ALG_ALGSEL_MASK;
4568 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
4569 OP_ALG_ALGSEL_MASK;
4570 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
4571
4572 /* Skip DES algorithms if not supported by device */
4573 if (!des_inst &&
4574 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
4575 (c1_alg_sel == OP_ALG_ALGSEL_DES)))
4576 continue;
4577
4578 /* Skip AES algorithms if not supported by device */
4579 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
4580 continue;
4581
4582 /*
4583 * Check support for AES algorithms not available
4584 * on LP devices.
4585 */
4586 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
4587 if (alg_aai == OP_ALG_AAI_GCM)
4588 continue;
4589
4590 /*
4591 * Skip algorithms requiring message digests
4592 * if MD or MD size is not supported by device.
4593 */
4594 if (c2_alg_sel &&
4595 (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
4596 continue;
4597
4598 caam_aead_alg_init(t_alg);
4599
4600 err = crypto_register_aead(&t_alg->aead);
4601 if (err) {
4602 pr_warn("%s alg registration failed\n",
4603 t_alg->aead.base.cra_driver_name);
4604 continue;
4605 }
4606
4607 t_alg->registered = true;
4608 registered = true;
4609 }
4610
4611 if (registered)
4612 pr_info("caam algorithms registered in /proc/crypto\n");
4613
4614 return err;
4615 }
4616
4617 module_init(caam_algapi_init);
4618 module_exit(caam_algapi_exit);
4619
4620 MODULE_LICENSE("GPL");
4621 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4622 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
Linux with added FPC-III support