?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / crypto / caam / caamalg.c
blob2188235be02dfc2487125e54bfb04e8dc4416845
1 /*
2 * caam - Freescale FSL CAAM support for crypto API
3 *
4 * Copyright 2008-2011 Freescale Semiconductor, Inc.
5 * Copyright 2016 NXP
6 *
7 * Based on talitos crypto API driver.
8 *
9 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
10 *
11 * --------------- ---------------
12 * | JobDesc #1 |-------------------->| ShareDesc |
13 * | *(packet 1) | | (PDB) |
14 * --------------- |------------->| (hashKey) |
15 * . | | (cipherKey) |
16 * . | |-------->| (operation) |
17 * --------------- | | ---------------
18 * | JobDesc #2 |------| |
19 * | *(packet 2) | |
20 * --------------- |
21 * . |
22 * . |
23 * --------------- |
24 * | JobDesc #3 |------------
25 * | *(packet 3) |
26 * ---------------
27 *
28 * The SharedDesc never changes for a connection unless rekeyed, but
29 * each packet will likely be in a different place. So all we need
30 * to know to process the packet is where the input is, where the
31 * output goes, and what context we want to process with. Context is
32 * in the SharedDesc, packet references in the JobDesc.
33 *
34 * So, a job desc looks like:
35 *
36 * ---------------------
37 * | Header |
38 * | ShareDesc Pointer |
39 * | SEQ_OUT_PTR |
40 * | (output buffer) |
41 * | (output length) |
42 * | SEQ_IN_PTR |
43 * | (input buffer) |
44 * | (input length) |
45 * ---------------------
46 */
47
48 #include "compat.h"
49
50 #include "regs.h"
51 #include "intern.h"
52 #include "desc_constr.h"
53 #include "jr.h"
54 #include "error.h"
55 #include "sg_sw_sec4.h"
56 #include "key_gen.h"
57 #include "caamalg_desc.h"
58
59 /*
60 * crypto alg
61 */
62 #define CAAM_CRA_PRIORITY 3000
63 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
64 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
65 CTR_RFC3686_NONCE_SIZE + \
66 SHA512_DIGEST_SIZE * 2)
67
68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
70 CAAM_CMD_SZ * 4)
71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
72 CAAM_CMD_SZ * 5)
73
74 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
75 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
76
77 #ifdef DEBUG
78 /* for print_hex_dumps with line references */
79 #define debug(format, arg...) printk(format, arg)
80 #else
81 #define debug(format, arg...)
82 #endif
83
84 static struct list_head alg_list;
85
86 struct caam_alg_entry {
87 int class1_alg_type;
88 int class2_alg_type;
89 bool rfc3686;
90 bool geniv;
91 };
92
93 struct caam_aead_alg {
94 struct aead_alg aead;
95 struct caam_alg_entry caam;
96 bool registered;
97 };
98
99 /*
100 * per-session context
101 */
102 struct caam_ctx {
103 u32 sh_desc_enc[DESC_MAX_USED_LEN];
104 u32 sh_desc_dec[DESC_MAX_USED_LEN];
105 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
106 u8 key[CAAM_MAX_KEY_SIZE];
107 dma_addr_t sh_desc_enc_dma;
108 dma_addr_t sh_desc_dec_dma;
109 dma_addr_t sh_desc_givenc_dma;
110 dma_addr_t key_dma;
111 enum dma_data_direction dir;
112 struct device *jrdev;
113 struct alginfo adata;
114 struct alginfo cdata;
115 unsigned int authsize;
116 };
117
118 static int aead_null_set_sh_desc(struct crypto_aead *aead)
119 {
120 struct caam_ctx *ctx = crypto_aead_ctx(aead);
121 struct device *jrdev = ctx->jrdev;
122 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
123 u32 *desc;
124 int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN -
125 ctx->adata.keylen_pad;
126
127 /*
128 * Job Descriptor and Shared Descriptors
129 * must all fit into the 64-word Descriptor h/w Buffer
130 */
131 if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) {
132 ctx->adata.key_inline = true;
133 ctx->adata.key_virt = ctx->key;
134 } else {
135 ctx->adata.key_inline = false;
136 ctx->adata.key_dma = ctx->key_dma;
137 }
138
139 /* aead_encrypt shared descriptor */
140 desc = ctx->sh_desc_enc;
141 cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize,
142 ctrlpriv->era);
143 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
144 desc_bytes(desc), ctx->dir);
145
146 /*
147 * Job Descriptor and Shared Descriptors
148 * must all fit into the 64-word Descriptor h/w Buffer
149 */
150 if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) {
151 ctx->adata.key_inline = true;
152 ctx->adata.key_virt = ctx->key;
153 } else {
154 ctx->adata.key_inline = false;
155 ctx->adata.key_dma = ctx->key_dma;
156 }
157
158 /* aead_decrypt shared descriptor */
159 desc = ctx->sh_desc_dec;
160 cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize,
161 ctrlpriv->era);
162 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
163 desc_bytes(desc), ctx->dir);
164
165 return 0;
166 }
167
168 static int aead_set_sh_desc(struct crypto_aead *aead)
169 {
170 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
171 struct caam_aead_alg, aead);
172 unsigned int ivsize = crypto_aead_ivsize(aead);
173 struct caam_ctx *ctx = crypto_aead_ctx(aead);
174 struct device *jrdev = ctx->jrdev;
175 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
176 u32 ctx1_iv_off = 0;
177 u32 *desc, *nonce = NULL;
178 u32 inl_mask;
179 unsigned int data_len[2];
180 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
181 OP_ALG_AAI_CTR_MOD128);
182 const bool is_rfc3686 = alg->caam.rfc3686;
183
184 if (!ctx->authsize)
185 return 0;
186
187 /* NULL encryption / decryption */
188 if (!ctx->cdata.keylen)
189 return aead_null_set_sh_desc(aead);
190
191 /*
192 * AES-CTR needs to load IV in CONTEXT1 reg
193 * at an offset of 128bits (16bytes)
194 * CONTEXT1[255:128] = IV
195 */
196 if (ctr_mode)
197 ctx1_iv_off = 16;
198
199 /*
200 * RFC3686 specific:
201 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
202 */
203 if (is_rfc3686) {
204 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
205 nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad +
206 ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE);
207 }
208
209 data_len[0] = ctx->adata.keylen_pad;
210 data_len[1] = ctx->cdata.keylen;
211
212 if (alg->caam.geniv)
213 goto skip_enc;
214
215 /*
216 * Job Descriptor and Shared Descriptors
217 * must all fit into the 64-word Descriptor h/w Buffer
218 */
219 if (desc_inline_query(DESC_AEAD_ENC_LEN +
220 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
221 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
222 ARRAY_SIZE(data_len)) < 0)
223 return -EINVAL;
224
225 if (inl_mask & 1)
226 ctx->adata.key_virt = ctx->key;
227 else
228 ctx->adata.key_dma = ctx->key_dma;
229
230 if (inl_mask & 2)
231 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
232 else
233 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
234
235 ctx->adata.key_inline = !!(inl_mask & 1);
236 ctx->cdata.key_inline = !!(inl_mask & 2);
237
238 /* aead_encrypt shared descriptor */
239 desc = ctx->sh_desc_enc;
240 cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize,
241 ctx->authsize, is_rfc3686, nonce, ctx1_iv_off,
242 false, ctrlpriv->era);
243 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
244 desc_bytes(desc), ctx->dir);
245
246 skip_enc:
247 /*
248 * Job Descriptor and Shared Descriptors
249 * must all fit into the 64-word Descriptor h/w Buffer
250 */
251 if (desc_inline_query(DESC_AEAD_DEC_LEN +
252 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
253 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
254 ARRAY_SIZE(data_len)) < 0)
255 return -EINVAL;
256
257 if (inl_mask & 1)
258 ctx->adata.key_virt = ctx->key;
259 else
260 ctx->adata.key_dma = ctx->key_dma;
261
262 if (inl_mask & 2)
263 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
264 else
265 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
266
267 ctx->adata.key_inline = !!(inl_mask & 1);
268 ctx->cdata.key_inline = !!(inl_mask & 2);
269
270 /* aead_decrypt shared descriptor */
271 desc = ctx->sh_desc_dec;
272 cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize,
273 ctx->authsize, alg->caam.geniv, is_rfc3686,
274 nonce, ctx1_iv_off, false, ctrlpriv->era);
275 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
276 desc_bytes(desc), ctx->dir);
277
278 if (!alg->caam.geniv)
279 goto skip_givenc;
280
281 /*
282 * Job Descriptor and Shared Descriptors
283 * must all fit into the 64-word Descriptor h/w Buffer
284 */
285 if (desc_inline_query(DESC_AEAD_GIVENC_LEN +
286 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0),
287 AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask,
288 ARRAY_SIZE(data_len)) < 0)
289 return -EINVAL;
290
291 if (inl_mask & 1)
292 ctx->adata.key_virt = ctx->key;
293 else
294 ctx->adata.key_dma = ctx->key_dma;
295
296 if (inl_mask & 2)
297 ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad;
298 else
299 ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad;
300
301 ctx->adata.key_inline = !!(inl_mask & 1);
302 ctx->cdata.key_inline = !!(inl_mask & 2);
303
304 /* aead_givencrypt shared descriptor */
305 desc = ctx->sh_desc_enc;
306 cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize,
307 ctx->authsize, is_rfc3686, nonce,
308 ctx1_iv_off, false, ctrlpriv->era);
309 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
310 desc_bytes(desc), ctx->dir);
311
312 skip_givenc:
313 return 0;
314 }
315
316 static int aead_setauthsize(struct crypto_aead *authenc,
317 unsigned int authsize)
318 {
319 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
320
321 ctx->authsize = authsize;
322 aead_set_sh_desc(authenc);
323
324 return 0;
325 }
326
327 static int gcm_set_sh_desc(struct crypto_aead *aead)
328 {
329 struct caam_ctx *ctx = crypto_aead_ctx(aead);
330 struct device *jrdev = ctx->jrdev;
331 u32 *desc;
332 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
333 ctx->cdata.keylen;
334
335 if (!ctx->cdata.keylen || !ctx->authsize)
336 return 0;
337
338 /*
339 * AES GCM encrypt shared descriptor
340 * Job Descriptor and Shared Descriptor
341 * must fit into the 64-word Descriptor h/w Buffer
342 */
343 if (rem_bytes >= DESC_GCM_ENC_LEN) {
344 ctx->cdata.key_inline = true;
345 ctx->cdata.key_virt = ctx->key;
346 } else {
347 ctx->cdata.key_inline = false;
348 ctx->cdata.key_dma = ctx->key_dma;
349 }
350
351 desc = ctx->sh_desc_enc;
352 cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ctx->authsize);
353 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
354 desc_bytes(desc), ctx->dir);
355
356 /*
357 * Job Descriptor and Shared Descriptors
358 * must all fit into the 64-word Descriptor h/w Buffer
359 */
360 if (rem_bytes >= DESC_GCM_DEC_LEN) {
361 ctx->cdata.key_inline = true;
362 ctx->cdata.key_virt = ctx->key;
363 } else {
364 ctx->cdata.key_inline = false;
365 ctx->cdata.key_dma = ctx->key_dma;
366 }
367
368 desc = ctx->sh_desc_dec;
369 cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ctx->authsize);
370 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
371 desc_bytes(desc), ctx->dir);
372
373 return 0;
374 }
375
376 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
377 {
378 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
379
380 ctx->authsize = authsize;
381 gcm_set_sh_desc(authenc);
382
383 return 0;
384 }
385
386 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
387 {
388 struct caam_ctx *ctx = crypto_aead_ctx(aead);
389 struct device *jrdev = ctx->jrdev;
390 u32 *desc;
391 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
392 ctx->cdata.keylen;
393
394 if (!ctx->cdata.keylen || !ctx->authsize)
395 return 0;
396
397 /*
398 * RFC4106 encrypt shared descriptor
399 * Job Descriptor and Shared Descriptor
400 * must fit into the 64-word Descriptor h/w Buffer
401 */
402 if (rem_bytes >= DESC_RFC4106_ENC_LEN) {
403 ctx->cdata.key_inline = true;
404 ctx->cdata.key_virt = ctx->key;
405 } else {
406 ctx->cdata.key_inline = false;
407 ctx->cdata.key_dma = ctx->key_dma;
408 }
409
410 desc = ctx->sh_desc_enc;
411 cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ctx->authsize);
412 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
413 desc_bytes(desc), ctx->dir);
414
415 /*
416 * Job Descriptor and Shared Descriptors
417 * must all fit into the 64-word Descriptor h/w Buffer
418 */
419 if (rem_bytes >= DESC_RFC4106_DEC_LEN) {
420 ctx->cdata.key_inline = true;
421 ctx->cdata.key_virt = ctx->key;
422 } else {
423 ctx->cdata.key_inline = false;
424 ctx->cdata.key_dma = ctx->key_dma;
425 }
426
427 desc = ctx->sh_desc_dec;
428 cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ctx->authsize);
429 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
430 desc_bytes(desc), ctx->dir);
431
432 return 0;
433 }
434
435 static int rfc4106_setauthsize(struct crypto_aead *authenc,
436 unsigned int authsize)
437 {
438 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
439
440 ctx->authsize = authsize;
441 rfc4106_set_sh_desc(authenc);
442
443 return 0;
444 }
445
446 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
447 {
448 struct caam_ctx *ctx = crypto_aead_ctx(aead);
449 struct device *jrdev = ctx->jrdev;
450 u32 *desc;
451 int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN -
452 ctx->cdata.keylen;
453
454 if (!ctx->cdata.keylen || !ctx->authsize)
455 return 0;
456
457 /*
458 * RFC4543 encrypt shared descriptor
459 * Job Descriptor and Shared Descriptor
460 * must fit into the 64-word Descriptor h/w Buffer
461 */
462 if (rem_bytes >= DESC_RFC4543_ENC_LEN) {
463 ctx->cdata.key_inline = true;
464 ctx->cdata.key_virt = ctx->key;
465 } else {
466 ctx->cdata.key_inline = false;
467 ctx->cdata.key_dma = ctx->key_dma;
468 }
469
470 desc = ctx->sh_desc_enc;
471 cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ctx->authsize);
472 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
473 desc_bytes(desc), ctx->dir);
474
475 /*
476 * Job Descriptor and Shared Descriptors
477 * must all fit into the 64-word Descriptor h/w Buffer
478 */
479 if (rem_bytes >= DESC_RFC4543_DEC_LEN) {
480 ctx->cdata.key_inline = true;
481 ctx->cdata.key_virt = ctx->key;
482 } else {
483 ctx->cdata.key_inline = false;
484 ctx->cdata.key_dma = ctx->key_dma;
485 }
486
487 desc = ctx->sh_desc_dec;
488 cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ctx->authsize);
489 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
490 desc_bytes(desc), ctx->dir);
491
492 return 0;
493 }
494
495 static int rfc4543_setauthsize(struct crypto_aead *authenc,
496 unsigned int authsize)
497 {
498 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
499
500 ctx->authsize = authsize;
501 rfc4543_set_sh_desc(authenc);
502
503 return 0;
504 }
505
506 static int aead_setkey(struct crypto_aead *aead,
507 const u8 *key, unsigned int keylen)
508 {
509 struct caam_ctx *ctx = crypto_aead_ctx(aead);
510 struct device *jrdev = ctx->jrdev;
511 struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent);
512 struct crypto_authenc_keys keys;
513 int ret = 0;
514
515 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
516 goto badkey;
517
518 #ifdef DEBUG
519 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
520 keys.authkeylen + keys.enckeylen, keys.enckeylen,
521 keys.authkeylen);
522 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
523 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
524 #endif
525
526 /*
527 * If DKP is supported, use it in the shared descriptor to generate
528 * the split key.
529 */
530 if (ctrlpriv->era >= 6) {
531 ctx->adata.keylen = keys.authkeylen;
532 ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype &
533 OP_ALG_ALGSEL_MASK);
534
535 if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE)
536 goto badkey;
537
538 memcpy(ctx->key, keys.authkey, keys.authkeylen);
539 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey,
540 keys.enckeylen);
541 dma_sync_single_for_device(jrdev, ctx->key_dma,
542 ctx->adata.keylen_pad +
543 keys.enckeylen, ctx->dir);
544 goto skip_split_key;
545 }
546
547 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey,
548 keys.authkeylen, CAAM_MAX_KEY_SIZE -
549 keys.enckeylen);
550 if (ret) {
551 goto badkey;
552 }
553
554 /* postpend encryption key to auth split key */
555 memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen);
556 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad +
557 keys.enckeylen, ctx->dir);
558 #ifdef DEBUG
559 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
560 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
561 ctx->adata.keylen_pad + keys.enckeylen, 1);
562 #endif
563
564 skip_split_key:
565 ctx->cdata.keylen = keys.enckeylen;
566 return aead_set_sh_desc(aead);
567 badkey:
568 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
569 return -EINVAL;
570 }
571
572 static int gcm_setkey(struct crypto_aead *aead,
573 const u8 *key, unsigned int keylen)
574 {
575 struct caam_ctx *ctx = crypto_aead_ctx(aead);
576 struct device *jrdev = ctx->jrdev;
577
578 #ifdef DEBUG
579 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
580 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
581 #endif
582
583 memcpy(ctx->key, key, keylen);
584 dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir);
585 ctx->cdata.keylen = keylen;
586
587 return gcm_set_sh_desc(aead);
588 }
589
590 static int rfc4106_setkey(struct crypto_aead *aead,
591 const u8 *key, unsigned int keylen)
592 {
593 struct caam_ctx *ctx = crypto_aead_ctx(aead);
594 struct device *jrdev = ctx->jrdev;
595
596 if (keylen < 4)
597 return -EINVAL;
598
599 #ifdef DEBUG
600 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
601 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
602 #endif
603
604 memcpy(ctx->key, key, keylen);
605
606 /*
607 * The last four bytes of the key material are used as the salt value
608 * in the nonce. Update the AES key length.
609 */
610 ctx->cdata.keylen = keylen - 4;
611 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
612 ctx->dir);
613 return rfc4106_set_sh_desc(aead);
614 }
615
616 static int rfc4543_setkey(struct crypto_aead *aead,
617 const u8 *key, unsigned int keylen)
618 {
619 struct caam_ctx *ctx = crypto_aead_ctx(aead);
620 struct device *jrdev = ctx->jrdev;
621
622 if (keylen < 4)
623 return -EINVAL;
624
625 #ifdef DEBUG
626 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
627 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
628 #endif
629
630 memcpy(ctx->key, key, keylen);
631
632 /*
633 * The last four bytes of the key material are used as the salt value
634 * in the nonce. Update the AES key length.
635 */
636 ctx->cdata.keylen = keylen - 4;
637 dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen,
638 ctx->dir);
639 return rfc4543_set_sh_desc(aead);
640 }
641
642 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
643 const u8 *key, unsigned int keylen)
644 {
645 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
646 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
647 const char *alg_name = crypto_tfm_alg_name(tfm);
648 struct device *jrdev = ctx->jrdev;
649 unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
650 u32 *desc;
651 u32 ctx1_iv_off = 0;
652 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
653 OP_ALG_AAI_CTR_MOD128);
654 const bool is_rfc3686 = (ctr_mode &&
655 (strstr(alg_name, "rfc3686") != NULL));
656
657 #ifdef DEBUG
658 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
659 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
660 #endif
661 /*
662 * AES-CTR needs to load IV in CONTEXT1 reg
663 * at an offset of 128bits (16bytes)
664 * CONTEXT1[255:128] = IV
665 */
666 if (ctr_mode)
667 ctx1_iv_off = 16;
668
669 /*
670 * RFC3686 specific:
671 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
672 * | *key = {KEY, NONCE}
673 */
674 if (is_rfc3686) {
675 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
676 keylen -= CTR_RFC3686_NONCE_SIZE;
677 }
678
679 ctx->cdata.keylen = keylen;
680 ctx->cdata.key_virt = key;
681 ctx->cdata.key_inline = true;
682
683 /* ablkcipher_encrypt shared descriptor */
684 desc = ctx->sh_desc_enc;
685 cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686,
686 ctx1_iv_off);
687 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
688 desc_bytes(desc), ctx->dir);
689
690 /* ablkcipher_decrypt shared descriptor */
691 desc = ctx->sh_desc_dec;
692 cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686,
693 ctx1_iv_off);
694 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
695 desc_bytes(desc), ctx->dir);
696
697 /* ablkcipher_givencrypt shared descriptor */
698 desc = ctx->sh_desc_givenc;
699 cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686,
700 ctx1_iv_off);
701 dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma,
702 desc_bytes(desc), ctx->dir);
703
704 return 0;
705 }
706
707 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
708 const u8 *key, unsigned int keylen)
709 {
710 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
711 struct device *jrdev = ctx->jrdev;
712 u32 *desc;
713
714 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
715 crypto_ablkcipher_set_flags(ablkcipher,
716 CRYPTO_TFM_RES_BAD_KEY_LEN);
717 dev_err(jrdev, "key size mismatch\n");
718 return -EINVAL;
719 }
720
721 ctx->cdata.keylen = keylen;
722 ctx->cdata.key_virt = key;
723 ctx->cdata.key_inline = true;
724
725 /* xts_ablkcipher_encrypt shared descriptor */
726 desc = ctx->sh_desc_enc;
727 cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata);
728 dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma,
729 desc_bytes(desc), ctx->dir);
730
731 /* xts_ablkcipher_decrypt shared descriptor */
732 desc = ctx->sh_desc_dec;
733 cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata);
734 dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma,
735 desc_bytes(desc), ctx->dir);
736
737 return 0;
738 }
739
740 /*
741 * aead_edesc - s/w-extended aead descriptor
742 * @src_nents: number of segments in input s/w scatterlist
743 * @dst_nents: number of segments in output s/w scatterlist
744 * @sec4_sg_bytes: length of dma mapped sec4_sg space
745 * @sec4_sg_dma: bus physical mapped address of h/w link table
746 * @sec4_sg: pointer to h/w link table
747 * @hw_desc: the h/w job descriptor followed by any referenced link tables
748 */
749 struct aead_edesc {
750 int src_nents;
751 int dst_nents;
752 int sec4_sg_bytes;
753 dma_addr_t sec4_sg_dma;
754 struct sec4_sg_entry *sec4_sg;
755 u32 hw_desc[];
756 };
757
758 /*
759 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
760 * @src_nents: number of segments in input s/w scatterlist
761 * @dst_nents: number of segments in output s/w scatterlist
762 * @iv_dma: dma address of iv for checking continuity and link table
763 * @sec4_sg_bytes: length of dma mapped sec4_sg space
764 * @sec4_sg_dma: bus physical mapped address of h/w link table
765 * @sec4_sg: pointer to h/w link table
766 * @hw_desc: the h/w job descriptor followed by any referenced link tables
767 */
768 struct ablkcipher_edesc {
769 int src_nents;
770 int dst_nents;
771 dma_addr_t iv_dma;
772 int sec4_sg_bytes;
773 dma_addr_t sec4_sg_dma;
774 struct sec4_sg_entry *sec4_sg;
775 u32 hw_desc[0];
776 };
777
778 static void caam_unmap(struct device *dev, struct scatterlist *src,
779 struct scatterlist *dst, int src_nents,
780 int dst_nents,
781 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
782 int sec4_sg_bytes)
783 {
784 if (dst != src) {
785 if (src_nents)
786 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
787 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
788 } else {
789 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
790 }
791
792 if (iv_dma)
793 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
794 if (sec4_sg_bytes)
795 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
796 DMA_TO_DEVICE);
797 }
798
799 static void aead_unmap(struct device *dev,
800 struct aead_edesc *edesc,
801 struct aead_request *req)
802 {
803 caam_unmap(dev, req->src, req->dst,
804 edesc->src_nents, edesc->dst_nents, 0, 0,
805 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
806 }
807
808 static void ablkcipher_unmap(struct device *dev,
809 struct ablkcipher_edesc *edesc,
810 struct ablkcipher_request *req)
811 {
812 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
813 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
814
815 caam_unmap(dev, req->src, req->dst,
816 edesc->src_nents, edesc->dst_nents,
817 edesc->iv_dma, ivsize,
818 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
819 }
820
821 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
822 void *context)
823 {
824 struct aead_request *req = context;
825 struct aead_edesc *edesc;
826
827 #ifdef DEBUG
828 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
829 #endif
830
831 edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
832
833 if (err)
834 caam_jr_strstatus(jrdev, err);
835
836 aead_unmap(jrdev, edesc, req);
837
838 kfree(edesc);
839
840 aead_request_complete(req, err);
841 }
842
843 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
844 void *context)
845 {
846 struct aead_request *req = context;
847 struct aead_edesc *edesc;
848
849 #ifdef DEBUG
850 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
851 #endif
852
853 edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
854
855 if (err)
856 caam_jr_strstatus(jrdev, err);
857
858 aead_unmap(jrdev, edesc, req);
859
860 /*
861 * verify hw auth check passed else return -EBADMSG
862 */
863 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
864 err = -EBADMSG;
865
866 kfree(edesc);
867
868 aead_request_complete(req, err);
869 }
870
871 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
872 void *context)
873 {
874 struct ablkcipher_request *req = context;
875 struct ablkcipher_edesc *edesc;
876 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
877 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
878
879 #ifdef DEBUG
880 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
881 #endif
882
883 edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
884
885 if (err)
886 caam_jr_strstatus(jrdev, err);
887
888 #ifdef DEBUG
889 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
890 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
891 edesc->src_nents > 1 ? 100 : ivsize, 1);
892 #endif
893 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
894 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
895 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
896
897 ablkcipher_unmap(jrdev, edesc, req);
898
899 /*
900 * The crypto API expects us to set the IV (req->info) to the last
901 * ciphertext block. This is used e.g. by the CTS mode.
902 */
903 scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize,
904 ivsize, 0);
905
906 kfree(edesc);
907
908 ablkcipher_request_complete(req, err);
909 }
910
911 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
912 void *context)
913 {
914 struct ablkcipher_request *req = context;
915 struct ablkcipher_edesc *edesc;
916 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
917 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
918
919 #ifdef DEBUG
920 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
921 #endif
922
923 edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]);
924 if (err)
925 caam_jr_strstatus(jrdev, err);
926
927 #ifdef DEBUG
928 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
929 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
930 ivsize, 1);
931 #endif
932 caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ",
933 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
934 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
935
936 ablkcipher_unmap(jrdev, edesc, req);
937
938 /*
939 * The crypto API expects us to set the IV (req->info) to the last
940 * ciphertext block.
941 */
942 scatterwalk_map_and_copy(req->info, req->src, req->nbytes - ivsize,
943 ivsize, 0);
944
945 kfree(edesc);
946
947 ablkcipher_request_complete(req, err);
948 }
949
950 /*
951 * Fill in aead job descriptor
952 */
953 static void init_aead_job(struct aead_request *req,
954 struct aead_edesc *edesc,
955 bool all_contig, bool encrypt)
956 {
957 struct crypto_aead *aead = crypto_aead_reqtfm(req);
958 struct caam_ctx *ctx = crypto_aead_ctx(aead);
959 int authsize = ctx->authsize;
960 u32 *desc = edesc->hw_desc;
961 u32 out_options, in_options;
962 dma_addr_t dst_dma, src_dma;
963 int len, sec4_sg_index = 0;
964 dma_addr_t ptr;
965 u32 *sh_desc;
966
967 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
968 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
969
970 len = desc_len(sh_desc);
971 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
972
973 if (all_contig) {
974 src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0;
975 in_options = 0;
976 } else {
977 src_dma = edesc->sec4_sg_dma;
978 sec4_sg_index += edesc->src_nents;
979 in_options = LDST_SGF;
980 }
981
982 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
983 in_options);
984
985 dst_dma = src_dma;
986 out_options = in_options;
987
988 if (unlikely(req->src != req->dst)) {
989 if (edesc->dst_nents == 1) {
990 dst_dma = sg_dma_address(req->dst);
991 } else {
992 dst_dma = edesc->sec4_sg_dma +
993 sec4_sg_index *
994 sizeof(struct sec4_sg_entry);
995 out_options = LDST_SGF;
996 }
997 }
998
999 if (encrypt)
1000 append_seq_out_ptr(desc, dst_dma,
1001 req->assoclen + req->cryptlen + authsize,
1002 out_options);
1003 else
1004 append_seq_out_ptr(desc, dst_dma,
1005 req->assoclen + req->cryptlen - authsize,
1006 out_options);
1007 }
1008
1009 static void init_gcm_job(struct aead_request *req,
1010 struct aead_edesc *edesc,
1011 bool all_contig, bool encrypt)
1012 {
1013 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1014 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1015 unsigned int ivsize = crypto_aead_ivsize(aead);
1016 u32 *desc = edesc->hw_desc;
1017 bool generic_gcm = (ivsize == GCM_AES_IV_SIZE);
1018 unsigned int last;
1019
1020 init_aead_job(req, edesc, all_contig, encrypt);
1021 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1022
1023 /* BUG This should not be specific to generic GCM. */
1024 last = 0;
1025 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
1026 last = FIFOLD_TYPE_LAST1;
1027
1028 /* Read GCM IV */
1029 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1030 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | GCM_AES_IV_SIZE | last);
1031 /* Append Salt */
1032 if (!generic_gcm)
1033 append_data(desc, ctx->key + ctx->cdata.keylen, 4);
1034 /* Append IV */
1035 append_data(desc, req->iv, ivsize);
1036 /* End of blank commands */
1037 }
1038
1039 static void init_authenc_job(struct aead_request *req,
1040 struct aead_edesc *edesc,
1041 bool all_contig, bool encrypt)
1042 {
1043 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1044 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
1045 struct caam_aead_alg, aead);
1046 unsigned int ivsize = crypto_aead_ivsize(aead);
1047 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1048 struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent);
1049 const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) ==
1050 OP_ALG_AAI_CTR_MOD128);
1051 const bool is_rfc3686 = alg->caam.rfc3686;
1052 u32 *desc = edesc->hw_desc;
1053 u32 ivoffset = 0;
1054
1055 /*
1056 * AES-CTR needs to load IV in CONTEXT1 reg
1057 * at an offset of 128bits (16bytes)
1058 * CONTEXT1[255:128] = IV
1059 */
1060 if (ctr_mode)
1061 ivoffset = 16;
1062
1063 /*
1064 * RFC3686 specific:
1065 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1066 */
1067 if (is_rfc3686)
1068 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
1069
1070 init_aead_job(req, edesc, all_contig, encrypt);
1071
1072 /*
1073 * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports
1074 * having DPOVRD as destination.
1075 */
1076 if (ctrlpriv->era < 3)
1077 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
1078 else
1079 append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen);
1080
1081 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
1082 append_load_as_imm(desc, req->iv, ivsize,
1083 LDST_CLASS_1_CCB |
1084 LDST_SRCDST_BYTE_CONTEXT |
1085 (ivoffset << LDST_OFFSET_SHIFT));
1086 }
1087
1088 /*
1089 * Fill in ablkcipher job descriptor
1090 */
1091 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
1092 struct ablkcipher_edesc *edesc,
1093 struct ablkcipher_request *req,
1094 bool iv_contig)
1095 {
1096 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1097 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1098 u32 *desc = edesc->hw_desc;
1099 u32 out_options = 0, in_options;
1100 dma_addr_t dst_dma, src_dma;
1101 int len, sec4_sg_index = 0;
1102
1103 #ifdef DEBUG
1104 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1105 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1106 ivsize, 1);
1107 pr_err("asked=%d, nbytes%d\n",
1108 (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes);
1109 #endif
1110 caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ",
1111 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1112 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1113
1114 len = desc_len(sh_desc);
1115 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1116
1117 if (iv_contig) {
1118 src_dma = edesc->iv_dma;
1119 in_options = 0;
1120 } else {
1121 src_dma = edesc->sec4_sg_dma;
1122 sec4_sg_index += edesc->src_nents + 1;
1123 in_options = LDST_SGF;
1124 }
1125 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
1126
1127 if (likely(req->src == req->dst)) {
1128 if (edesc->src_nents == 1 && iv_contig) {
1129 dst_dma = sg_dma_address(req->src);
1130 } else {
1131 dst_dma = edesc->sec4_sg_dma +
1132 sizeof(struct sec4_sg_entry);
1133 out_options = LDST_SGF;
1134 }
1135 } else {
1136 if (edesc->dst_nents == 1) {
1137 dst_dma = sg_dma_address(req->dst);
1138 } else {
1139 dst_dma = edesc->sec4_sg_dma +
1140 sec4_sg_index * sizeof(struct sec4_sg_entry);
1141 out_options = LDST_SGF;
1142 }
1143 }
1144 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
1145 }
1146
1147 /*
1148 * Fill in ablkcipher givencrypt job descriptor
1149 */
1150 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
1151 struct ablkcipher_edesc *edesc,
1152 struct ablkcipher_request *req,
1153 bool iv_contig)
1154 {
1155 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1156 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1157 u32 *desc = edesc->hw_desc;
1158 u32 out_options, in_options;
1159 dma_addr_t dst_dma, src_dma;
1160 int len, sec4_sg_index = 0;
1161
1162 #ifdef DEBUG
1163 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
1164 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
1165 ivsize, 1);
1166 #endif
1167 caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ",
1168 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1169 edesc->src_nents > 1 ? 100 : req->nbytes, 1);
1170
1171 len = desc_len(sh_desc);
1172 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
1173
1174 if (edesc->src_nents == 1) {
1175 src_dma = sg_dma_address(req->src);
1176 in_options = 0;
1177 } else {
1178 src_dma = edesc->sec4_sg_dma;
1179 sec4_sg_index += edesc->src_nents;
1180 in_options = LDST_SGF;
1181 }
1182 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
1183
1184 if (iv_contig) {
1185 dst_dma = edesc->iv_dma;
1186 out_options = 0;
1187 } else {
1188 dst_dma = edesc->sec4_sg_dma +
1189 sec4_sg_index * sizeof(struct sec4_sg_entry);
1190 out_options = LDST_SGF;
1191 }
1192 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
1193 }
1194
1195 /*
1196 * allocate and map the aead extended descriptor
1197 */
1198 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
1199 int desc_bytes, bool *all_contig_ptr,
1200 bool encrypt)
1201 {
1202 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1203 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1204 struct device *jrdev = ctx->jrdev;
1205 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1206 GFP_KERNEL : GFP_ATOMIC;
1207 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1208 struct aead_edesc *edesc;
1209 int sec4_sg_index, sec4_sg_len, sec4_sg_bytes;
1210 unsigned int authsize = ctx->authsize;
1211
1212 if (unlikely(req->dst != req->src)) {
1213 src_nents = sg_nents_for_len(req->src, req->assoclen +
1214 req->cryptlen);
1215 if (unlikely(src_nents < 0)) {
1216 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1217 req->assoclen + req->cryptlen);
1218 return ERR_PTR(src_nents);
1219 }
1220
1221 dst_nents = sg_nents_for_len(req->dst, req->assoclen +
1222 req->cryptlen +
1223 (encrypt ? authsize :
1224 (-authsize)));
1225 if (unlikely(dst_nents < 0)) {
1226 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1227 req->assoclen + req->cryptlen +
1228 (encrypt ? authsize : (-authsize)));
1229 return ERR_PTR(dst_nents);
1230 }
1231 } else {
1232 src_nents = sg_nents_for_len(req->src, req->assoclen +
1233 req->cryptlen +
1234 (encrypt ? authsize : 0));
1235 if (unlikely(src_nents < 0)) {
1236 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1237 req->assoclen + req->cryptlen +
1238 (encrypt ? authsize : 0));
1239 return ERR_PTR(src_nents);
1240 }
1241 }
1242
1243 if (likely(req->src == req->dst)) {
1244 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1245 DMA_BIDIRECTIONAL);
1246 if (unlikely(!mapped_src_nents)) {
1247 dev_err(jrdev, "unable to map source\n");
1248 return ERR_PTR(-ENOMEM);
1249 }
1250 } else {
1251 /* Cover also the case of null (zero length) input data */
1252 if (src_nents) {
1253 mapped_src_nents = dma_map_sg(jrdev, req->src,
1254 src_nents, DMA_TO_DEVICE);
1255 if (unlikely(!mapped_src_nents)) {
1256 dev_err(jrdev, "unable to map source\n");
1257 return ERR_PTR(-ENOMEM);
1258 }
1259 } else {
1260 mapped_src_nents = 0;
1261 }
1262
1263 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1264 DMA_FROM_DEVICE);
1265 if (unlikely(!mapped_dst_nents)) {
1266 dev_err(jrdev, "unable to map destination\n");
1267 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1268 return ERR_PTR(-ENOMEM);
1269 }
1270 }
1271
1272 sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0;
1273 sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1274 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1275
1276 /* allocate space for base edesc and hw desc commands, link tables */
1277 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1278 GFP_DMA | flags);
1279 if (!edesc) {
1280 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1281 0, 0, 0);
1282 return ERR_PTR(-ENOMEM);
1283 }
1284
1285 edesc->src_nents = src_nents;
1286 edesc->dst_nents = dst_nents;
1287 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
1288 desc_bytes;
1289 *all_contig_ptr = !(mapped_src_nents > 1);
1290
1291 sec4_sg_index = 0;
1292 if (mapped_src_nents > 1) {
1293 sg_to_sec4_sg_last(req->src, mapped_src_nents,
1294 edesc->sec4_sg + sec4_sg_index, 0);
1295 sec4_sg_index += mapped_src_nents;
1296 }
1297 if (mapped_dst_nents > 1) {
1298 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1299 edesc->sec4_sg + sec4_sg_index, 0);
1300 }
1301
1302 if (!sec4_sg_bytes)
1303 return edesc;
1304
1305 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1306 sec4_sg_bytes, DMA_TO_DEVICE);
1307 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1308 dev_err(jrdev, "unable to map S/G table\n");
1309 aead_unmap(jrdev, edesc, req);
1310 kfree(edesc);
1311 return ERR_PTR(-ENOMEM);
1312 }
1313
1314 edesc->sec4_sg_bytes = sec4_sg_bytes;
1315
1316 return edesc;
1317 }
1318
1319 static int gcm_encrypt(struct aead_request *req)
1320 {
1321 struct aead_edesc *edesc;
1322 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1323 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1324 struct device *jrdev = ctx->jrdev;
1325 bool all_contig;
1326 u32 *desc;
1327 int ret = 0;
1328
1329 /* allocate extended descriptor */
1330 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
1331 if (IS_ERR(edesc))
1332 return PTR_ERR(edesc);
1333
1334 /* Create and submit job descriptor */
1335 init_gcm_job(req, edesc, all_contig, true);
1336 #ifdef DEBUG
1337 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1338 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1339 desc_bytes(edesc->hw_desc), 1);
1340 #endif
1341
1342 desc = edesc->hw_desc;
1343 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1344 if (!ret) {
1345 ret = -EINPROGRESS;
1346 } else {
1347 aead_unmap(jrdev, edesc, req);
1348 kfree(edesc);
1349 }
1350
1351 return ret;
1352 }
1353
1354 static int ipsec_gcm_encrypt(struct aead_request *req)
1355 {
1356 if (req->assoclen < 8)
1357 return -EINVAL;
1358
1359 return gcm_encrypt(req);
1360 }
1361
1362 static int aead_encrypt(struct aead_request *req)
1363 {
1364 struct aead_edesc *edesc;
1365 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1366 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1367 struct device *jrdev = ctx->jrdev;
1368 bool all_contig;
1369 u32 *desc;
1370 int ret = 0;
1371
1372 /* allocate extended descriptor */
1373 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1374 &all_contig, true);
1375 if (IS_ERR(edesc))
1376 return PTR_ERR(edesc);
1377
1378 /* Create and submit job descriptor */
1379 init_authenc_job(req, edesc, all_contig, true);
1380 #ifdef DEBUG
1381 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1382 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1383 desc_bytes(edesc->hw_desc), 1);
1384 #endif
1385
1386 desc = edesc->hw_desc;
1387 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
1388 if (!ret) {
1389 ret = -EINPROGRESS;
1390 } else {
1391 aead_unmap(jrdev, edesc, req);
1392 kfree(edesc);
1393 }
1394
1395 return ret;
1396 }
1397
1398 static int gcm_decrypt(struct aead_request *req)
1399 {
1400 struct aead_edesc *edesc;
1401 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1402 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1403 struct device *jrdev = ctx->jrdev;
1404 bool all_contig;
1405 u32 *desc;
1406 int ret = 0;
1407
1408 /* allocate extended descriptor */
1409 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
1410 if (IS_ERR(edesc))
1411 return PTR_ERR(edesc);
1412
1413 /* Create and submit job descriptor*/
1414 init_gcm_job(req, edesc, all_contig, false);
1415 #ifdef DEBUG
1416 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1417 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1418 desc_bytes(edesc->hw_desc), 1);
1419 #endif
1420
1421 desc = edesc->hw_desc;
1422 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1423 if (!ret) {
1424 ret = -EINPROGRESS;
1425 } else {
1426 aead_unmap(jrdev, edesc, req);
1427 kfree(edesc);
1428 }
1429
1430 return ret;
1431 }
1432
1433 static int ipsec_gcm_decrypt(struct aead_request *req)
1434 {
1435 if (req->assoclen < 8)
1436 return -EINVAL;
1437
1438 return gcm_decrypt(req);
1439 }
1440
1441 static int aead_decrypt(struct aead_request *req)
1442 {
1443 struct aead_edesc *edesc;
1444 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1445 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1446 struct device *jrdev = ctx->jrdev;
1447 bool all_contig;
1448 u32 *desc;
1449 int ret = 0;
1450
1451 caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ",
1452 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
1453 req->assoclen + req->cryptlen, 1);
1454
1455 /* allocate extended descriptor */
1456 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
1457 &all_contig, false);
1458 if (IS_ERR(edesc))
1459 return PTR_ERR(edesc);
1460
1461 /* Create and submit job descriptor*/
1462 init_authenc_job(req, edesc, all_contig, false);
1463 #ifdef DEBUG
1464 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1465 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1466 desc_bytes(edesc->hw_desc), 1);
1467 #endif
1468
1469 desc = edesc->hw_desc;
1470 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
1471 if (!ret) {
1472 ret = -EINPROGRESS;
1473 } else {
1474 aead_unmap(jrdev, edesc, req);
1475 kfree(edesc);
1476 }
1477
1478 return ret;
1479 }
1480
1481 /*
1482 * allocate and map the ablkcipher extended descriptor for ablkcipher
1483 */
1484 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
1485 *req, int desc_bytes,
1486 bool *iv_contig_out)
1487 {
1488 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1489 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1490 struct device *jrdev = ctx->jrdev;
1491 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1492 GFP_KERNEL : GFP_ATOMIC;
1493 int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0;
1494 struct ablkcipher_edesc *edesc;
1495 dma_addr_t iv_dma = 0;
1496 bool in_contig;
1497 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1498 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1499
1500 src_nents = sg_nents_for_len(req->src, req->nbytes);
1501 if (unlikely(src_nents < 0)) {
1502 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1503 req->nbytes);
1504 return ERR_PTR(src_nents);
1505 }
1506
1507 if (req->dst != req->src) {
1508 dst_nents = sg_nents_for_len(req->dst, req->nbytes);
1509 if (unlikely(dst_nents < 0)) {
1510 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1511 req->nbytes);
1512 return ERR_PTR(dst_nents);
1513 }
1514 }
1515
1516 if (likely(req->src == req->dst)) {
1517 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1518 DMA_BIDIRECTIONAL);
1519 if (unlikely(!mapped_src_nents)) {
1520 dev_err(jrdev, "unable to map source\n");
1521 return ERR_PTR(-ENOMEM);
1522 }
1523 } else {
1524 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1525 DMA_TO_DEVICE);
1526 if (unlikely(!mapped_src_nents)) {
1527 dev_err(jrdev, "unable to map source\n");
1528 return ERR_PTR(-ENOMEM);
1529 }
1530
1531 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1532 DMA_FROM_DEVICE);
1533 if (unlikely(!mapped_dst_nents)) {
1534 dev_err(jrdev, "unable to map destination\n");
1535 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1536 return ERR_PTR(-ENOMEM);
1537 }
1538 }
1539
1540 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
1541 if (dma_mapping_error(jrdev, iv_dma)) {
1542 dev_err(jrdev, "unable to map IV\n");
1543 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1544 0, 0, 0);
1545 return ERR_PTR(-ENOMEM);
1546 }
1547
1548 if (mapped_src_nents == 1 &&
1549 iv_dma + ivsize == sg_dma_address(req->src)) {
1550 in_contig = true;
1551 sec4_sg_ents = 0;
1552 } else {
1553 in_contig = false;
1554 sec4_sg_ents = 1 + mapped_src_nents;
1555 }
1556 dst_sg_idx = sec4_sg_ents;
1557 sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0;
1558 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1559
1560 /* allocate space for base edesc and hw desc commands, link tables */
1561 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1562 GFP_DMA | flags);
1563 if (!edesc) {
1564 dev_err(jrdev, "could not allocate extended descriptor\n");
1565 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1566 iv_dma, ivsize, 0, 0);
1567 return ERR_PTR(-ENOMEM);
1568 }
1569
1570 edesc->src_nents = src_nents;
1571 edesc->dst_nents = dst_nents;
1572 edesc->sec4_sg_bytes = sec4_sg_bytes;
1573 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1574 desc_bytes;
1575
1576 if (!in_contig) {
1577 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
1578 sg_to_sec4_sg_last(req->src, mapped_src_nents,
1579 edesc->sec4_sg + 1, 0);
1580 }
1581
1582 if (mapped_dst_nents > 1) {
1583 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1584 edesc->sec4_sg + dst_sg_idx, 0);
1585 }
1586
1587 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1588 sec4_sg_bytes, DMA_TO_DEVICE);
1589 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1590 dev_err(jrdev, "unable to map S/G table\n");
1591 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1592 iv_dma, ivsize, 0, 0);
1593 kfree(edesc);
1594 return ERR_PTR(-ENOMEM);
1595 }
1596
1597 edesc->iv_dma = iv_dma;
1598
1599 #ifdef DEBUG
1600 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
1601 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1602 sec4_sg_bytes, 1);
1603 #endif
1604
1605 *iv_contig_out = in_contig;
1606 return edesc;
1607 }
1608
1609 static int ablkcipher_encrypt(struct ablkcipher_request *req)
1610 {
1611 struct ablkcipher_edesc *edesc;
1612 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1613 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1614 struct device *jrdev = ctx->jrdev;
1615 bool iv_contig;
1616 u32 *desc;
1617 int ret = 0;
1618
1619 /* allocate extended descriptor */
1620 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1621 CAAM_CMD_SZ, &iv_contig);
1622 if (IS_ERR(edesc))
1623 return PTR_ERR(edesc);
1624
1625 /* Create and submit job descriptor*/
1626 init_ablkcipher_job(ctx->sh_desc_enc,
1627 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
1628 #ifdef DEBUG
1629 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1630 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1631 desc_bytes(edesc->hw_desc), 1);
1632 #endif
1633 desc = edesc->hw_desc;
1634 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1635
1636 if (!ret) {
1637 ret = -EINPROGRESS;
1638 } else {
1639 ablkcipher_unmap(jrdev, edesc, req);
1640 kfree(edesc);
1641 }
1642
1643 return ret;
1644 }
1645
1646 static int ablkcipher_decrypt(struct ablkcipher_request *req)
1647 {
1648 struct ablkcipher_edesc *edesc;
1649 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1650 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1651 struct device *jrdev = ctx->jrdev;
1652 bool iv_contig;
1653 u32 *desc;
1654 int ret = 0;
1655
1656 /* allocate extended descriptor */
1657 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
1658 CAAM_CMD_SZ, &iv_contig);
1659 if (IS_ERR(edesc))
1660 return PTR_ERR(edesc);
1661
1662 /* Create and submit job descriptor*/
1663 init_ablkcipher_job(ctx->sh_desc_dec,
1664 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
1665 desc = edesc->hw_desc;
1666 #ifdef DEBUG
1667 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
1668 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1669 desc_bytes(edesc->hw_desc), 1);
1670 #endif
1671
1672 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
1673 if (!ret) {
1674 ret = -EINPROGRESS;
1675 } else {
1676 ablkcipher_unmap(jrdev, edesc, req);
1677 kfree(edesc);
1678 }
1679
1680 return ret;
1681 }
1682
1683 /*
1684 * allocate and map the ablkcipher extended descriptor
1685 * for ablkcipher givencrypt
1686 */
1687 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
1688 struct skcipher_givcrypt_request *greq,
1689 int desc_bytes,
1690 bool *iv_contig_out)
1691 {
1692 struct ablkcipher_request *req = &greq->creq;
1693 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1694 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1695 struct device *jrdev = ctx->jrdev;
1696 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1697 GFP_KERNEL : GFP_ATOMIC;
1698 int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents;
1699 struct ablkcipher_edesc *edesc;
1700 dma_addr_t iv_dma = 0;
1701 bool out_contig;
1702 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1703 int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes;
1704
1705 src_nents = sg_nents_for_len(req->src, req->nbytes);
1706 if (unlikely(src_nents < 0)) {
1707 dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n",
1708 req->nbytes);
1709 return ERR_PTR(src_nents);
1710 }
1711
1712 if (likely(req->src == req->dst)) {
1713 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1714 DMA_BIDIRECTIONAL);
1715 if (unlikely(!mapped_src_nents)) {
1716 dev_err(jrdev, "unable to map source\n");
1717 return ERR_PTR(-ENOMEM);
1718 }
1719
1720 dst_nents = src_nents;
1721 mapped_dst_nents = src_nents;
1722 } else {
1723 mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents,
1724 DMA_TO_DEVICE);
1725 if (unlikely(!mapped_src_nents)) {
1726 dev_err(jrdev, "unable to map source\n");
1727 return ERR_PTR(-ENOMEM);
1728 }
1729
1730 dst_nents = sg_nents_for_len(req->dst, req->nbytes);
1731 if (unlikely(dst_nents < 0)) {
1732 dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n",
1733 req->nbytes);
1734 return ERR_PTR(dst_nents);
1735 }
1736
1737 mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents,
1738 DMA_FROM_DEVICE);
1739 if (unlikely(!mapped_dst_nents)) {
1740 dev_err(jrdev, "unable to map destination\n");
1741 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1742 return ERR_PTR(-ENOMEM);
1743 }
1744 }
1745
1746 /*
1747 * Check if iv can be contiguous with source and destination.
1748 * If so, include it. If not, create scatterlist.
1749 */
1750 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
1751 if (dma_mapping_error(jrdev, iv_dma)) {
1752 dev_err(jrdev, "unable to map IV\n");
1753 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0,
1754 0, 0, 0);
1755 return ERR_PTR(-ENOMEM);
1756 }
1757
1758 sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0;
1759 dst_sg_idx = sec4_sg_ents;
1760 if (mapped_dst_nents == 1 &&
1761 iv_dma + ivsize == sg_dma_address(req->dst)) {
1762 out_contig = true;
1763 } else {
1764 out_contig = false;
1765 sec4_sg_ents += 1 + mapped_dst_nents;
1766 }
1767
1768 /* allocate space for base edesc and hw desc commands, link tables */
1769 sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry);
1770 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
1771 GFP_DMA | flags);
1772 if (!edesc) {
1773 dev_err(jrdev, "could not allocate extended descriptor\n");
1774 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1775 iv_dma, ivsize, 0, 0);
1776 return ERR_PTR(-ENOMEM);
1777 }
1778
1779 edesc->src_nents = src_nents;
1780 edesc->dst_nents = dst_nents;
1781 edesc->sec4_sg_bytes = sec4_sg_bytes;
1782 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
1783 desc_bytes;
1784
1785 if (mapped_src_nents > 1)
1786 sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg,
1787 0);
1788
1789 if (!out_contig) {
1790 dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx,
1791 iv_dma, ivsize, 0);
1792 sg_to_sec4_sg_last(req->dst, mapped_dst_nents,
1793 edesc->sec4_sg + dst_sg_idx + 1, 0);
1794 }
1795
1796 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1797 sec4_sg_bytes, DMA_TO_DEVICE);
1798 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1799 dev_err(jrdev, "unable to map S/G table\n");
1800 caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents,
1801 iv_dma, ivsize, 0, 0);
1802 kfree(edesc);
1803 return ERR_PTR(-ENOMEM);
1804 }
1805 edesc->iv_dma = iv_dma;
1806
1807 #ifdef DEBUG
1808 print_hex_dump(KERN_ERR,
1809 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
1810 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
1811 sec4_sg_bytes, 1);
1812 #endif
1813
1814 *iv_contig_out = out_contig;
1815 return edesc;
1816 }
1817
1818 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
1819 {
1820 struct ablkcipher_request *req = &creq->creq;
1821 struct ablkcipher_edesc *edesc;
1822 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1823 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1824 struct device *jrdev = ctx->jrdev;
1825 bool iv_contig = false;
1826 u32 *desc;
1827 int ret = 0;
1828
1829 /* allocate extended descriptor */
1830 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
1831 CAAM_CMD_SZ, &iv_contig);
1832 if (IS_ERR(edesc))
1833 return PTR_ERR(edesc);
1834
1835 /* Create and submit job descriptor*/
1836 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
1837 edesc, req, iv_contig);
1838 #ifdef DEBUG
1839 print_hex_dump(KERN_ERR,
1840 "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
1841 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
1842 desc_bytes(edesc->hw_desc), 1);
1843 #endif
1844 desc = edesc->hw_desc;
1845 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
1846
1847 if (!ret) {
1848 ret = -EINPROGRESS;
1849 } else {
1850 ablkcipher_unmap(jrdev, edesc, req);
1851 kfree(edesc);
1852 }
1853
1854 return ret;
1855 }
1856
1857 #define template_aead template_u.aead
1858 #define template_ablkcipher template_u.ablkcipher
1859 struct caam_alg_template {
1860 char name[CRYPTO_MAX_ALG_NAME];
1861 char driver_name[CRYPTO_MAX_ALG_NAME];
1862 unsigned int blocksize;
1863 u32 type;
1864 union {
1865 struct ablkcipher_alg ablkcipher;
1866 } template_u;
1867 u32 class1_alg_type;
1868 u32 class2_alg_type;
1869 };
1870
1871 static struct caam_alg_template driver_algs[] = {
1872 /* ablkcipher descriptor */
1873 {
1874 .name = "cbc(aes)",
1875 .driver_name = "cbc-aes-caam",
1876 .blocksize = AES_BLOCK_SIZE,
1877 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
1878 .template_ablkcipher = {
1879 .setkey = ablkcipher_setkey,
1880 .encrypt = ablkcipher_encrypt,
1881 .decrypt = ablkcipher_decrypt,
1882 .givencrypt = ablkcipher_givencrypt,
1883 .geniv = "<built-in>",
1884 .min_keysize = AES_MIN_KEY_SIZE,
1885 .max_keysize = AES_MAX_KEY_SIZE,
1886 .ivsize = AES_BLOCK_SIZE,
1887 },
1888 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
1889 },
1890 {
1891 .name = "cbc(des3_ede)",
1892 .driver_name = "cbc-3des-caam",
1893 .blocksize = DES3_EDE_BLOCK_SIZE,
1894 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
1895 .template_ablkcipher = {
1896 .setkey = ablkcipher_setkey,
1897 .encrypt = ablkcipher_encrypt,
1898 .decrypt = ablkcipher_decrypt,
1899 .givencrypt = ablkcipher_givencrypt,
1900 .geniv = "<built-in>",
1901 .min_keysize = DES3_EDE_KEY_SIZE,
1902 .max_keysize = DES3_EDE_KEY_SIZE,
1903 .ivsize = DES3_EDE_BLOCK_SIZE,
1904 },
1905 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
1906 },
1907 {
1908 .name = "cbc(des)",
1909 .driver_name = "cbc-des-caam",
1910 .blocksize = DES_BLOCK_SIZE,
1911 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
1912 .template_ablkcipher = {
1913 .setkey = ablkcipher_setkey,
1914 .encrypt = ablkcipher_encrypt,
1915 .decrypt = ablkcipher_decrypt,
1916 .givencrypt = ablkcipher_givencrypt,
1917 .geniv = "<built-in>",
1918 .min_keysize = DES_KEY_SIZE,
1919 .max_keysize = DES_KEY_SIZE,
1920 .ivsize = DES_BLOCK_SIZE,
1921 },
1922 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
1923 },
1924 {
1925 .name = "ctr(aes)",
1926 .driver_name = "ctr-aes-caam",
1927 .blocksize = 1,
1928 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
1929 .template_ablkcipher = {
1930 .setkey = ablkcipher_setkey,
1931 .encrypt = ablkcipher_encrypt,
1932 .decrypt = ablkcipher_decrypt,
1933 .geniv = "chainiv",
1934 .min_keysize = AES_MIN_KEY_SIZE,
1935 .max_keysize = AES_MAX_KEY_SIZE,
1936 .ivsize = AES_BLOCK_SIZE,
1937 },
1938 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
1939 },
1940 {
1941 .name = "rfc3686(ctr(aes))",
1942 .driver_name = "rfc3686-ctr-aes-caam",
1943 .blocksize = 1,
1944 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
1945 .template_ablkcipher = {
1946 .setkey = ablkcipher_setkey,
1947 .encrypt = ablkcipher_encrypt,
1948 .decrypt = ablkcipher_decrypt,
1949 .givencrypt = ablkcipher_givencrypt,
1950 .geniv = "<built-in>",
1951 .min_keysize = AES_MIN_KEY_SIZE +
1952 CTR_RFC3686_NONCE_SIZE,
1953 .max_keysize = AES_MAX_KEY_SIZE +
1954 CTR_RFC3686_NONCE_SIZE,
1955 .ivsize = CTR_RFC3686_IV_SIZE,
1956 },
1957 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
1958 },
1959 {
1960 .name = "xts(aes)",
1961 .driver_name = "xts-aes-caam",
1962 .blocksize = AES_BLOCK_SIZE,
1963 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
1964 .template_ablkcipher = {
1965 .setkey = xts_ablkcipher_setkey,
1966 .encrypt = ablkcipher_encrypt,
1967 .decrypt = ablkcipher_decrypt,
1968 .geniv = "eseqiv",
1969 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1970 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1971 .ivsize = AES_BLOCK_SIZE,
1972 },
1973 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
1974 },
1975 };
1976
1977 static struct caam_aead_alg driver_aeads[] = {
1978 {
1979 .aead = {
1980 .base = {
1981 .cra_name = "rfc4106(gcm(aes))",
1982 .cra_driver_name = "rfc4106-gcm-aes-caam",
1983 .cra_blocksize = 1,
1984 },
1985 .setkey = rfc4106_setkey,
1986 .setauthsize = rfc4106_setauthsize,
1987 .encrypt = ipsec_gcm_encrypt,
1988 .decrypt = ipsec_gcm_decrypt,
1989 .ivsize = GCM_RFC4106_IV_SIZE,
1990 .maxauthsize = AES_BLOCK_SIZE,
1991 },
1992 .caam = {
1993 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
1994 },
1995 },
1996 {
1997 .aead = {
1998 .base = {
1999 .cra_name = "rfc4543(gcm(aes))",
2000 .cra_driver_name = "rfc4543-gcm-aes-caam",
2001 .cra_blocksize = 1,
2002 },
2003 .setkey = rfc4543_setkey,
2004 .setauthsize = rfc4543_setauthsize,
2005 .encrypt = ipsec_gcm_encrypt,
2006 .decrypt = ipsec_gcm_decrypt,
2007 .ivsize = GCM_RFC4543_IV_SIZE,
2008 .maxauthsize = AES_BLOCK_SIZE,
2009 },
2010 .caam = {
2011 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2012 },
2013 },
2014 /* Galois Counter Mode */
2015 {
2016 .aead = {
2017 .base = {
2018 .cra_name = "gcm(aes)",
2019 .cra_driver_name = "gcm-aes-caam",
2020 .cra_blocksize = 1,
2021 },
2022 .setkey = gcm_setkey,
2023 .setauthsize = gcm_setauthsize,
2024 .encrypt = gcm_encrypt,
2025 .decrypt = gcm_decrypt,
2026 .ivsize = GCM_AES_IV_SIZE,
2027 .maxauthsize = AES_BLOCK_SIZE,
2028 },
2029 .caam = {
2030 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
2031 },
2032 },
2033 /* single-pass ipsec_esp descriptor */
2034 {
2035 .aead = {
2036 .base = {
2037 .cra_name = "authenc(hmac(md5),"
2038 "ecb(cipher_null))",
2039 .cra_driver_name = "authenc-hmac-md5-"
2040 "ecb-cipher_null-caam",
2041 .cra_blocksize = NULL_BLOCK_SIZE,
2042 },
2043 .setkey = aead_setkey,
2044 .setauthsize = aead_setauthsize,
2045 .encrypt = aead_encrypt,
2046 .decrypt = aead_decrypt,
2047 .ivsize = NULL_IV_SIZE,
2048 .maxauthsize = MD5_DIGEST_SIZE,
2049 },
2050 .caam = {
2051 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2052 OP_ALG_AAI_HMAC_PRECOMP,
2053 },
2054 },
2055 {
2056 .aead = {
2057 .base = {
2058 .cra_name = "authenc(hmac(sha1),"
2059 "ecb(cipher_null))",
2060 .cra_driver_name = "authenc-hmac-sha1-"
2061 "ecb-cipher_null-caam",
2062 .cra_blocksize = NULL_BLOCK_SIZE,
2063 },
2064 .setkey = aead_setkey,
2065 .setauthsize = aead_setauthsize,
2066 .encrypt = aead_encrypt,
2067 .decrypt = aead_decrypt,
2068 .ivsize = NULL_IV_SIZE,
2069 .maxauthsize = SHA1_DIGEST_SIZE,
2070 },
2071 .caam = {
2072 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2073 OP_ALG_AAI_HMAC_PRECOMP,
2074 },
2075 },
2076 {
2077 .aead = {
2078 .base = {
2079 .cra_name = "authenc(hmac(sha224),"
2080 "ecb(cipher_null))",
2081 .cra_driver_name = "authenc-hmac-sha224-"
2082 "ecb-cipher_null-caam",
2083 .cra_blocksize = NULL_BLOCK_SIZE,
2084 },
2085 .setkey = aead_setkey,
2086 .setauthsize = aead_setauthsize,
2087 .encrypt = aead_encrypt,
2088 .decrypt = aead_decrypt,
2089 .ivsize = NULL_IV_SIZE,
2090 .maxauthsize = SHA224_DIGEST_SIZE,
2091 },
2092 .caam = {
2093 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2094 OP_ALG_AAI_HMAC_PRECOMP,
2095 },
2096 },
2097 {
2098 .aead = {
2099 .base = {
2100 .cra_name = "authenc(hmac(sha256),"
2101 "ecb(cipher_null))",
2102 .cra_driver_name = "authenc-hmac-sha256-"
2103 "ecb-cipher_null-caam",
2104 .cra_blocksize = NULL_BLOCK_SIZE,
2105 },
2106 .setkey = aead_setkey,
2107 .setauthsize = aead_setauthsize,
2108 .encrypt = aead_encrypt,
2109 .decrypt = aead_decrypt,
2110 .ivsize = NULL_IV_SIZE,
2111 .maxauthsize = SHA256_DIGEST_SIZE,
2112 },
2113 .caam = {
2114 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2115 OP_ALG_AAI_HMAC_PRECOMP,
2116 },
2117 },
2118 {
2119 .aead = {
2120 .base = {
2121 .cra_name = "authenc(hmac(sha384),"
2122 "ecb(cipher_null))",
2123 .cra_driver_name = "authenc-hmac-sha384-"
2124 "ecb-cipher_null-caam",
2125 .cra_blocksize = NULL_BLOCK_SIZE,
2126 },
2127 .setkey = aead_setkey,
2128 .setauthsize = aead_setauthsize,
2129 .encrypt = aead_encrypt,
2130 .decrypt = aead_decrypt,
2131 .ivsize = NULL_IV_SIZE,
2132 .maxauthsize = SHA384_DIGEST_SIZE,
2133 },
2134 .caam = {
2135 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2136 OP_ALG_AAI_HMAC_PRECOMP,
2137 },
2138 },
2139 {
2140 .aead = {
2141 .base = {
2142 .cra_name = "authenc(hmac(sha512),"
2143 "ecb(cipher_null))",
2144 .cra_driver_name = "authenc-hmac-sha512-"
2145 "ecb-cipher_null-caam",
2146 .cra_blocksize = NULL_BLOCK_SIZE,
2147 },
2148 .setkey = aead_setkey,
2149 .setauthsize = aead_setauthsize,
2150 .encrypt = aead_encrypt,
2151 .decrypt = aead_decrypt,
2152 .ivsize = NULL_IV_SIZE,
2153 .maxauthsize = SHA512_DIGEST_SIZE,
2154 },
2155 .caam = {
2156 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2157 OP_ALG_AAI_HMAC_PRECOMP,
2158 },
2159 },
2160 {
2161 .aead = {
2162 .base = {
2163 .cra_name = "authenc(hmac(md5),cbc(aes))",
2164 .cra_driver_name = "authenc-hmac-md5-"
2165 "cbc-aes-caam",
2166 .cra_blocksize = AES_BLOCK_SIZE,
2167 },
2168 .setkey = aead_setkey,
2169 .setauthsize = aead_setauthsize,
2170 .encrypt = aead_encrypt,
2171 .decrypt = aead_decrypt,
2172 .ivsize = AES_BLOCK_SIZE,
2173 .maxauthsize = MD5_DIGEST_SIZE,
2174 },
2175 .caam = {
2176 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2177 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2178 OP_ALG_AAI_HMAC_PRECOMP,
2179 },
2180 },
2181 {
2182 .aead = {
2183 .base = {
2184 .cra_name = "echainiv(authenc(hmac(md5),"
2185 "cbc(aes)))",
2186 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2187 "cbc-aes-caam",
2188 .cra_blocksize = AES_BLOCK_SIZE,
2189 },
2190 .setkey = aead_setkey,
2191 .setauthsize = aead_setauthsize,
2192 .encrypt = aead_encrypt,
2193 .decrypt = aead_decrypt,
2194 .ivsize = AES_BLOCK_SIZE,
2195 .maxauthsize = MD5_DIGEST_SIZE,
2196 },
2197 .caam = {
2198 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2199 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2200 OP_ALG_AAI_HMAC_PRECOMP,
2201 .geniv = true,
2202 },
2203 },
2204 {
2205 .aead = {
2206 .base = {
2207 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2208 .cra_driver_name = "authenc-hmac-sha1-"
2209 "cbc-aes-caam",
2210 .cra_blocksize = AES_BLOCK_SIZE,
2211 },
2212 .setkey = aead_setkey,
2213 .setauthsize = aead_setauthsize,
2214 .encrypt = aead_encrypt,
2215 .decrypt = aead_decrypt,
2216 .ivsize = AES_BLOCK_SIZE,
2217 .maxauthsize = SHA1_DIGEST_SIZE,
2218 },
2219 .caam = {
2220 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2221 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2222 OP_ALG_AAI_HMAC_PRECOMP,
2223 },
2224 },
2225 {
2226 .aead = {
2227 .base = {
2228 .cra_name = "echainiv(authenc(hmac(sha1),"
2229 "cbc(aes)))",
2230 .cra_driver_name = "echainiv-authenc-"
2231 "hmac-sha1-cbc-aes-caam",
2232 .cra_blocksize = AES_BLOCK_SIZE,
2233 },
2234 .setkey = aead_setkey,
2235 .setauthsize = aead_setauthsize,
2236 .encrypt = aead_encrypt,
2237 .decrypt = aead_decrypt,
2238 .ivsize = AES_BLOCK_SIZE,
2239 .maxauthsize = SHA1_DIGEST_SIZE,
2240 },
2241 .caam = {
2242 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2243 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2244 OP_ALG_AAI_HMAC_PRECOMP,
2245 .geniv = true,
2246 },
2247 },
2248 {
2249 .aead = {
2250 .base = {
2251 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2252 .cra_driver_name = "authenc-hmac-sha224-"
2253 "cbc-aes-caam",
2254 .cra_blocksize = AES_BLOCK_SIZE,
2255 },
2256 .setkey = aead_setkey,
2257 .setauthsize = aead_setauthsize,
2258 .encrypt = aead_encrypt,
2259 .decrypt = aead_decrypt,
2260 .ivsize = AES_BLOCK_SIZE,
2261 .maxauthsize = SHA224_DIGEST_SIZE,
2262 },
2263 .caam = {
2264 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2265 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2266 OP_ALG_AAI_HMAC_PRECOMP,
2267 },
2268 },
2269 {
2270 .aead = {
2271 .base = {
2272 .cra_name = "echainiv(authenc(hmac(sha224),"
2273 "cbc(aes)))",
2274 .cra_driver_name = "echainiv-authenc-"
2275 "hmac-sha224-cbc-aes-caam",
2276 .cra_blocksize = AES_BLOCK_SIZE,
2277 },
2278 .setkey = aead_setkey,
2279 .setauthsize = aead_setauthsize,
2280 .encrypt = aead_encrypt,
2281 .decrypt = aead_decrypt,
2282 .ivsize = AES_BLOCK_SIZE,
2283 .maxauthsize = SHA224_DIGEST_SIZE,
2284 },
2285 .caam = {
2286 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2287 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2288 OP_ALG_AAI_HMAC_PRECOMP,
2289 .geniv = true,
2290 },
2291 },
2292 {
2293 .aead = {
2294 .base = {
2295 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2296 .cra_driver_name = "authenc-hmac-sha256-"
2297 "cbc-aes-caam",
2298 .cra_blocksize = AES_BLOCK_SIZE,
2299 },
2300 .setkey = aead_setkey,
2301 .setauthsize = aead_setauthsize,
2302 .encrypt = aead_encrypt,
2303 .decrypt = aead_decrypt,
2304 .ivsize = AES_BLOCK_SIZE,
2305 .maxauthsize = SHA256_DIGEST_SIZE,
2306 },
2307 .caam = {
2308 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2309 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2310 OP_ALG_AAI_HMAC_PRECOMP,
2311 },
2312 },
2313 {
2314 .aead = {
2315 .base = {
2316 .cra_name = "echainiv(authenc(hmac(sha256),"
2317 "cbc(aes)))",
2318 .cra_driver_name = "echainiv-authenc-"
2319 "hmac-sha256-cbc-aes-caam",
2320 .cra_blocksize = AES_BLOCK_SIZE,
2321 },
2322 .setkey = aead_setkey,
2323 .setauthsize = aead_setauthsize,
2324 .encrypt = aead_encrypt,
2325 .decrypt = aead_decrypt,
2326 .ivsize = AES_BLOCK_SIZE,
2327 .maxauthsize = SHA256_DIGEST_SIZE,
2328 },
2329 .caam = {
2330 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2331 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2332 OP_ALG_AAI_HMAC_PRECOMP,
2333 .geniv = true,
2334 },
2335 },
2336 {
2337 .aead = {
2338 .base = {
2339 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2340 .cra_driver_name = "authenc-hmac-sha384-"
2341 "cbc-aes-caam",
2342 .cra_blocksize = AES_BLOCK_SIZE,
2343 },
2344 .setkey = aead_setkey,
2345 .setauthsize = aead_setauthsize,
2346 .encrypt = aead_encrypt,
2347 .decrypt = aead_decrypt,
2348 .ivsize = AES_BLOCK_SIZE,
2349 .maxauthsize = SHA384_DIGEST_SIZE,
2350 },
2351 .caam = {
2352 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2353 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2354 OP_ALG_AAI_HMAC_PRECOMP,
2355 },
2356 },
2357 {
2358 .aead = {
2359 .base = {
2360 .cra_name = "echainiv(authenc(hmac(sha384),"
2361 "cbc(aes)))",
2362 .cra_driver_name = "echainiv-authenc-"
2363 "hmac-sha384-cbc-aes-caam",
2364 .cra_blocksize = AES_BLOCK_SIZE,
2365 },
2366 .setkey = aead_setkey,
2367 .setauthsize = aead_setauthsize,
2368 .encrypt = aead_encrypt,
2369 .decrypt = aead_decrypt,
2370 .ivsize = AES_BLOCK_SIZE,
2371 .maxauthsize = SHA384_DIGEST_SIZE,
2372 },
2373 .caam = {
2374 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2375 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2376 OP_ALG_AAI_HMAC_PRECOMP,
2377 .geniv = true,
2378 },
2379 },
2380 {
2381 .aead = {
2382 .base = {
2383 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2384 .cra_driver_name = "authenc-hmac-sha512-"
2385 "cbc-aes-caam",
2386 .cra_blocksize = AES_BLOCK_SIZE,
2387 },
2388 .setkey = aead_setkey,
2389 .setauthsize = aead_setauthsize,
2390 .encrypt = aead_encrypt,
2391 .decrypt = aead_decrypt,
2392 .ivsize = AES_BLOCK_SIZE,
2393 .maxauthsize = SHA512_DIGEST_SIZE,
2394 },
2395 .caam = {
2396 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2397 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2398 OP_ALG_AAI_HMAC_PRECOMP,
2399 },
2400 },
2401 {
2402 .aead = {
2403 .base = {
2404 .cra_name = "echainiv(authenc(hmac(sha512),"
2405 "cbc(aes)))",
2406 .cra_driver_name = "echainiv-authenc-"
2407 "hmac-sha512-cbc-aes-caam",
2408 .cra_blocksize = AES_BLOCK_SIZE,
2409 },
2410 .setkey = aead_setkey,
2411 .setauthsize = aead_setauthsize,
2412 .encrypt = aead_encrypt,
2413 .decrypt = aead_decrypt,
2414 .ivsize = AES_BLOCK_SIZE,
2415 .maxauthsize = SHA512_DIGEST_SIZE,
2416 },
2417 .caam = {
2418 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2419 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2420 OP_ALG_AAI_HMAC_PRECOMP,
2421 .geniv = true,
2422 },
2423 },
2424 {
2425 .aead = {
2426 .base = {
2427 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2428 .cra_driver_name = "authenc-hmac-md5-"
2429 "cbc-des3_ede-caam",
2430 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2431 },
2432 .setkey = aead_setkey,
2433 .setauthsize = aead_setauthsize,
2434 .encrypt = aead_encrypt,
2435 .decrypt = aead_decrypt,
2436 .ivsize = DES3_EDE_BLOCK_SIZE,
2437 .maxauthsize = MD5_DIGEST_SIZE,
2438 },
2439 .caam = {
2440 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2441 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2442 OP_ALG_AAI_HMAC_PRECOMP,
2443 }
2444 },
2445 {
2446 .aead = {
2447 .base = {
2448 .cra_name = "echainiv(authenc(hmac(md5),"
2449 "cbc(des3_ede)))",
2450 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2451 "cbc-des3_ede-caam",
2452 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2453 },
2454 .setkey = aead_setkey,
2455 .setauthsize = aead_setauthsize,
2456 .encrypt = aead_encrypt,
2457 .decrypt = aead_decrypt,
2458 .ivsize = DES3_EDE_BLOCK_SIZE,
2459 .maxauthsize = MD5_DIGEST_SIZE,
2460 },
2461 .caam = {
2462 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2463 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2464 OP_ALG_AAI_HMAC_PRECOMP,
2465 .geniv = true,
2466 }
2467 },
2468 {
2469 .aead = {
2470 .base = {
2471 .cra_name = "authenc(hmac(sha1),"
2472 "cbc(des3_ede))",
2473 .cra_driver_name = "authenc-hmac-sha1-"
2474 "cbc-des3_ede-caam",
2475 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2476 },
2477 .setkey = aead_setkey,
2478 .setauthsize = aead_setauthsize,
2479 .encrypt = aead_encrypt,
2480 .decrypt = aead_decrypt,
2481 .ivsize = DES3_EDE_BLOCK_SIZE,
2482 .maxauthsize = SHA1_DIGEST_SIZE,
2483 },
2484 .caam = {
2485 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2486 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2487 OP_ALG_AAI_HMAC_PRECOMP,
2488 },
2489 },
2490 {
2491 .aead = {
2492 .base = {
2493 .cra_name = "echainiv(authenc(hmac(sha1),"
2494 "cbc(des3_ede)))",
2495 .cra_driver_name = "echainiv-authenc-"
2496 "hmac-sha1-"
2497 "cbc-des3_ede-caam",
2498 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2499 },
2500 .setkey = aead_setkey,
2501 .setauthsize = aead_setauthsize,
2502 .encrypt = aead_encrypt,
2503 .decrypt = aead_decrypt,
2504 .ivsize = DES3_EDE_BLOCK_SIZE,
2505 .maxauthsize = SHA1_DIGEST_SIZE,
2506 },
2507 .caam = {
2508 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2509 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2510 OP_ALG_AAI_HMAC_PRECOMP,
2511 .geniv = true,
2512 },
2513 },
2514 {
2515 .aead = {
2516 .base = {
2517 .cra_name = "authenc(hmac(sha224),"
2518 "cbc(des3_ede))",
2519 .cra_driver_name = "authenc-hmac-sha224-"
2520 "cbc-des3_ede-caam",
2521 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2522 },
2523 .setkey = aead_setkey,
2524 .setauthsize = aead_setauthsize,
2525 .encrypt = aead_encrypt,
2526 .decrypt = aead_decrypt,
2527 .ivsize = DES3_EDE_BLOCK_SIZE,
2528 .maxauthsize = SHA224_DIGEST_SIZE,
2529 },
2530 .caam = {
2531 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2532 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2533 OP_ALG_AAI_HMAC_PRECOMP,
2534 },
2535 },
2536 {
2537 .aead = {
2538 .base = {
2539 .cra_name = "echainiv(authenc(hmac(sha224),"
2540 "cbc(des3_ede)))",
2541 .cra_driver_name = "echainiv-authenc-"
2542 "hmac-sha224-"
2543 "cbc-des3_ede-caam",
2544 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2545 },
2546 .setkey = aead_setkey,
2547 .setauthsize = aead_setauthsize,
2548 .encrypt = aead_encrypt,
2549 .decrypt = aead_decrypt,
2550 .ivsize = DES3_EDE_BLOCK_SIZE,
2551 .maxauthsize = SHA224_DIGEST_SIZE,
2552 },
2553 .caam = {
2554 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2555 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2556 OP_ALG_AAI_HMAC_PRECOMP,
2557 .geniv = true,
2558 },
2559 },
2560 {
2561 .aead = {
2562 .base = {
2563 .cra_name = "authenc(hmac(sha256),"
2564 "cbc(des3_ede))",
2565 .cra_driver_name = "authenc-hmac-sha256-"
2566 "cbc-des3_ede-caam",
2567 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2568 },
2569 .setkey = aead_setkey,
2570 .setauthsize = aead_setauthsize,
2571 .encrypt = aead_encrypt,
2572 .decrypt = aead_decrypt,
2573 .ivsize = DES3_EDE_BLOCK_SIZE,
2574 .maxauthsize = SHA256_DIGEST_SIZE,
2575 },
2576 .caam = {
2577 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2578 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2579 OP_ALG_AAI_HMAC_PRECOMP,
2580 },
2581 },
2582 {
2583 .aead = {
2584 .base = {
2585 .cra_name = "echainiv(authenc(hmac(sha256),"
2586 "cbc(des3_ede)))",
2587 .cra_driver_name = "echainiv-authenc-"
2588 "hmac-sha256-"
2589 "cbc-des3_ede-caam",
2590 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2591 },
2592 .setkey = aead_setkey,
2593 .setauthsize = aead_setauthsize,
2594 .encrypt = aead_encrypt,
2595 .decrypt = aead_decrypt,
2596 .ivsize = DES3_EDE_BLOCK_SIZE,
2597 .maxauthsize = SHA256_DIGEST_SIZE,
2598 },
2599 .caam = {
2600 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2601 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2602 OP_ALG_AAI_HMAC_PRECOMP,
2603 .geniv = true,
2604 },
2605 },
2606 {
2607 .aead = {
2608 .base = {
2609 .cra_name = "authenc(hmac(sha384),"
2610 "cbc(des3_ede))",
2611 .cra_driver_name = "authenc-hmac-sha384-"
2612 "cbc-des3_ede-caam",
2613 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2614 },
2615 .setkey = aead_setkey,
2616 .setauthsize = aead_setauthsize,
2617 .encrypt = aead_encrypt,
2618 .decrypt = aead_decrypt,
2619 .ivsize = DES3_EDE_BLOCK_SIZE,
2620 .maxauthsize = SHA384_DIGEST_SIZE,
2621 },
2622 .caam = {
2623 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2624 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2625 OP_ALG_AAI_HMAC_PRECOMP,
2626 },
2627 },
2628 {
2629 .aead = {
2630 .base = {
2631 .cra_name = "echainiv(authenc(hmac(sha384),"
2632 "cbc(des3_ede)))",
2633 .cra_driver_name = "echainiv-authenc-"
2634 "hmac-sha384-"
2635 "cbc-des3_ede-caam",
2636 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2637 },
2638 .setkey = aead_setkey,
2639 .setauthsize = aead_setauthsize,
2640 .encrypt = aead_encrypt,
2641 .decrypt = aead_decrypt,
2642 .ivsize = DES3_EDE_BLOCK_SIZE,
2643 .maxauthsize = SHA384_DIGEST_SIZE,
2644 },
2645 .caam = {
2646 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2647 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2648 OP_ALG_AAI_HMAC_PRECOMP,
2649 .geniv = true,
2650 },
2651 },
2652 {
2653 .aead = {
2654 .base = {
2655 .cra_name = "authenc(hmac(sha512),"
2656 "cbc(des3_ede))",
2657 .cra_driver_name = "authenc-hmac-sha512-"
2658 "cbc-des3_ede-caam",
2659 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2660 },
2661 .setkey = aead_setkey,
2662 .setauthsize = aead_setauthsize,
2663 .encrypt = aead_encrypt,
2664 .decrypt = aead_decrypt,
2665 .ivsize = DES3_EDE_BLOCK_SIZE,
2666 .maxauthsize = SHA512_DIGEST_SIZE,
2667 },
2668 .caam = {
2669 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2670 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2671 OP_ALG_AAI_HMAC_PRECOMP,
2672 },
2673 },
2674 {
2675 .aead = {
2676 .base = {
2677 .cra_name = "echainiv(authenc(hmac(sha512),"
2678 "cbc(des3_ede)))",
2679 .cra_driver_name = "echainiv-authenc-"
2680 "hmac-sha512-"
2681 "cbc-des3_ede-caam",
2682 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2683 },
2684 .setkey = aead_setkey,
2685 .setauthsize = aead_setauthsize,
2686 .encrypt = aead_encrypt,
2687 .decrypt = aead_decrypt,
2688 .ivsize = DES3_EDE_BLOCK_SIZE,
2689 .maxauthsize = SHA512_DIGEST_SIZE,
2690 },
2691 .caam = {
2692 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2693 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2694 OP_ALG_AAI_HMAC_PRECOMP,
2695 .geniv = true,
2696 },
2697 },
2698 {
2699 .aead = {
2700 .base = {
2701 .cra_name = "authenc(hmac(md5),cbc(des))",
2702 .cra_driver_name = "authenc-hmac-md5-"
2703 "cbc-des-caam",
2704 .cra_blocksize = DES_BLOCK_SIZE,
2705 },
2706 .setkey = aead_setkey,
2707 .setauthsize = aead_setauthsize,
2708 .encrypt = aead_encrypt,
2709 .decrypt = aead_decrypt,
2710 .ivsize = DES_BLOCK_SIZE,
2711 .maxauthsize = MD5_DIGEST_SIZE,
2712 },
2713 .caam = {
2714 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2715 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2716 OP_ALG_AAI_HMAC_PRECOMP,
2717 },
2718 },
2719 {
2720 .aead = {
2721 .base = {
2722 .cra_name = "echainiv(authenc(hmac(md5),"
2723 "cbc(des)))",
2724 .cra_driver_name = "echainiv-authenc-hmac-md5-"
2725 "cbc-des-caam",
2726 .cra_blocksize = DES_BLOCK_SIZE,
2727 },
2728 .setkey = aead_setkey,
2729 .setauthsize = aead_setauthsize,
2730 .encrypt = aead_encrypt,
2731 .decrypt = aead_decrypt,
2732 .ivsize = DES_BLOCK_SIZE,
2733 .maxauthsize = MD5_DIGEST_SIZE,
2734 },
2735 .caam = {
2736 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2737 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2738 OP_ALG_AAI_HMAC_PRECOMP,
2739 .geniv = true,
2740 },
2741 },
2742 {
2743 .aead = {
2744 .base = {
2745 .cra_name = "authenc(hmac(sha1),cbc(des))",
2746 .cra_driver_name = "authenc-hmac-sha1-"
2747 "cbc-des-caam",
2748 .cra_blocksize = DES_BLOCK_SIZE,
2749 },
2750 .setkey = aead_setkey,
2751 .setauthsize = aead_setauthsize,
2752 .encrypt = aead_encrypt,
2753 .decrypt = aead_decrypt,
2754 .ivsize = DES_BLOCK_SIZE,
2755 .maxauthsize = SHA1_DIGEST_SIZE,
2756 },
2757 .caam = {
2758 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2759 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2760 OP_ALG_AAI_HMAC_PRECOMP,
2761 },
2762 },
2763 {
2764 .aead = {
2765 .base = {
2766 .cra_name = "echainiv(authenc(hmac(sha1),"
2767 "cbc(des)))",
2768 .cra_driver_name = "echainiv-authenc-"
2769 "hmac-sha1-cbc-des-caam",
2770 .cra_blocksize = DES_BLOCK_SIZE,
2771 },
2772 .setkey = aead_setkey,
2773 .setauthsize = aead_setauthsize,
2774 .encrypt = aead_encrypt,
2775 .decrypt = aead_decrypt,
2776 .ivsize = DES_BLOCK_SIZE,
2777 .maxauthsize = SHA1_DIGEST_SIZE,
2778 },
2779 .caam = {
2780 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2781 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
2782 OP_ALG_AAI_HMAC_PRECOMP,
2783 .geniv = true,
2784 },
2785 },
2786 {
2787 .aead = {
2788 .base = {
2789 .cra_name = "authenc(hmac(sha224),cbc(des))",
2790 .cra_driver_name = "authenc-hmac-sha224-"
2791 "cbc-des-caam",
2792 .cra_blocksize = DES_BLOCK_SIZE,
2793 },
2794 .setkey = aead_setkey,
2795 .setauthsize = aead_setauthsize,
2796 .encrypt = aead_encrypt,
2797 .decrypt = aead_decrypt,
2798 .ivsize = DES_BLOCK_SIZE,
2799 .maxauthsize = SHA224_DIGEST_SIZE,
2800 },
2801 .caam = {
2802 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2803 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2804 OP_ALG_AAI_HMAC_PRECOMP,
2805 },
2806 },
2807 {
2808 .aead = {
2809 .base = {
2810 .cra_name = "echainiv(authenc(hmac(sha224),"
2811 "cbc(des)))",
2812 .cra_driver_name = "echainiv-authenc-"
2813 "hmac-sha224-cbc-des-caam",
2814 .cra_blocksize = DES_BLOCK_SIZE,
2815 },
2816 .setkey = aead_setkey,
2817 .setauthsize = aead_setauthsize,
2818 .encrypt = aead_encrypt,
2819 .decrypt = aead_decrypt,
2820 .ivsize = DES_BLOCK_SIZE,
2821 .maxauthsize = SHA224_DIGEST_SIZE,
2822 },
2823 .caam = {
2824 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2825 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
2826 OP_ALG_AAI_HMAC_PRECOMP,
2827 .geniv = true,
2828 },
2829 },
2830 {
2831 .aead = {
2832 .base = {
2833 .cra_name = "authenc(hmac(sha256),cbc(des))",
2834 .cra_driver_name = "authenc-hmac-sha256-"
2835 "cbc-des-caam",
2836 .cra_blocksize = DES_BLOCK_SIZE,
2837 },
2838 .setkey = aead_setkey,
2839 .setauthsize = aead_setauthsize,
2840 .encrypt = aead_encrypt,
2841 .decrypt = aead_decrypt,
2842 .ivsize = DES_BLOCK_SIZE,
2843 .maxauthsize = SHA256_DIGEST_SIZE,
2844 },
2845 .caam = {
2846 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2847 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2848 OP_ALG_AAI_HMAC_PRECOMP,
2849 },
2850 },
2851 {
2852 .aead = {
2853 .base = {
2854 .cra_name = "echainiv(authenc(hmac(sha256),"
2855 "cbc(des)))",
2856 .cra_driver_name = "echainiv-authenc-"
2857 "hmac-sha256-cbc-des-caam",
2858 .cra_blocksize = DES_BLOCK_SIZE,
2859 },
2860 .setkey = aead_setkey,
2861 .setauthsize = aead_setauthsize,
2862 .encrypt = aead_encrypt,
2863 .decrypt = aead_decrypt,
2864 .ivsize = DES_BLOCK_SIZE,
2865 .maxauthsize = SHA256_DIGEST_SIZE,
2866 },
2867 .caam = {
2868 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2869 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
2870 OP_ALG_AAI_HMAC_PRECOMP,
2871 .geniv = true,
2872 },
2873 },
2874 {
2875 .aead = {
2876 .base = {
2877 .cra_name = "authenc(hmac(sha384),cbc(des))",
2878 .cra_driver_name = "authenc-hmac-sha384-"
2879 "cbc-des-caam",
2880 .cra_blocksize = DES_BLOCK_SIZE,
2881 },
2882 .setkey = aead_setkey,
2883 .setauthsize = aead_setauthsize,
2884 .encrypt = aead_encrypt,
2885 .decrypt = aead_decrypt,
2886 .ivsize = DES_BLOCK_SIZE,
2887 .maxauthsize = SHA384_DIGEST_SIZE,
2888 },
2889 .caam = {
2890 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2891 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2892 OP_ALG_AAI_HMAC_PRECOMP,
2893 },
2894 },
2895 {
2896 .aead = {
2897 .base = {
2898 .cra_name = "echainiv(authenc(hmac(sha384),"
2899 "cbc(des)))",
2900 .cra_driver_name = "echainiv-authenc-"
2901 "hmac-sha384-cbc-des-caam",
2902 .cra_blocksize = DES_BLOCK_SIZE,
2903 },
2904 .setkey = aead_setkey,
2905 .setauthsize = aead_setauthsize,
2906 .encrypt = aead_encrypt,
2907 .decrypt = aead_decrypt,
2908 .ivsize = DES_BLOCK_SIZE,
2909 .maxauthsize = SHA384_DIGEST_SIZE,
2910 },
2911 .caam = {
2912 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2913 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
2914 OP_ALG_AAI_HMAC_PRECOMP,
2915 .geniv = true,
2916 },
2917 },
2918 {
2919 .aead = {
2920 .base = {
2921 .cra_name = "authenc(hmac(sha512),cbc(des))",
2922 .cra_driver_name = "authenc-hmac-sha512-"
2923 "cbc-des-caam",
2924 .cra_blocksize = DES_BLOCK_SIZE,
2925 },
2926 .setkey = aead_setkey,
2927 .setauthsize = aead_setauthsize,
2928 .encrypt = aead_encrypt,
2929 .decrypt = aead_decrypt,
2930 .ivsize = DES_BLOCK_SIZE,
2931 .maxauthsize = SHA512_DIGEST_SIZE,
2932 },
2933 .caam = {
2934 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2935 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2936 OP_ALG_AAI_HMAC_PRECOMP,
2937 },
2938 },
2939 {
2940 .aead = {
2941 .base = {
2942 .cra_name = "echainiv(authenc(hmac(sha512),"
2943 "cbc(des)))",
2944 .cra_driver_name = "echainiv-authenc-"
2945 "hmac-sha512-cbc-des-caam",
2946 .cra_blocksize = DES_BLOCK_SIZE,
2947 },
2948 .setkey = aead_setkey,
2949 .setauthsize = aead_setauthsize,
2950 .encrypt = aead_encrypt,
2951 .decrypt = aead_decrypt,
2952 .ivsize = DES_BLOCK_SIZE,
2953 .maxauthsize = SHA512_DIGEST_SIZE,
2954 },
2955 .caam = {
2956 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2957 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
2958 OP_ALG_AAI_HMAC_PRECOMP,
2959 .geniv = true,
2960 },
2961 },
2962 {
2963 .aead = {
2964 .base = {
2965 .cra_name = "authenc(hmac(md5),"
2966 "rfc3686(ctr(aes)))",
2967 .cra_driver_name = "authenc-hmac-md5-"
2968 "rfc3686-ctr-aes-caam",
2969 .cra_blocksize = 1,
2970 },
2971 .setkey = aead_setkey,
2972 .setauthsize = aead_setauthsize,
2973 .encrypt = aead_encrypt,
2974 .decrypt = aead_decrypt,
2975 .ivsize = CTR_RFC3686_IV_SIZE,
2976 .maxauthsize = MD5_DIGEST_SIZE,
2977 },
2978 .caam = {
2979 .class1_alg_type = OP_ALG_ALGSEL_AES |
2980 OP_ALG_AAI_CTR_MOD128,
2981 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
2982 OP_ALG_AAI_HMAC_PRECOMP,
2983 .rfc3686 = true,
2984 },
2985 },
2986 {
2987 .aead = {
2988 .base = {
2989 .cra_name = "seqiv(authenc("
2990 "hmac(md5),rfc3686(ctr(aes))))",
2991 .cra_driver_name = "seqiv-authenc-hmac-md5-"
2992 "rfc3686-ctr-aes-caam",
2993 .cra_blocksize = 1,
2994 },
2995 .setkey = aead_setkey,
2996 .setauthsize = aead_setauthsize,
2997 .encrypt = aead_encrypt,
2998 .decrypt = aead_decrypt,
2999 .ivsize = CTR_RFC3686_IV_SIZE,
3000 .maxauthsize = MD5_DIGEST_SIZE,
3001 },
3002 .caam = {
3003 .class1_alg_type = OP_ALG_ALGSEL_AES |
3004 OP_ALG_AAI_CTR_MOD128,
3005 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3006 OP_ALG_AAI_HMAC_PRECOMP,
3007 .rfc3686 = true,
3008 .geniv = true,
3009 },
3010 },
3011 {
3012 .aead = {
3013 .base = {
3014 .cra_name = "authenc(hmac(sha1),"
3015 "rfc3686(ctr(aes)))",
3016 .cra_driver_name = "authenc-hmac-sha1-"
3017 "rfc3686-ctr-aes-caam",
3018 .cra_blocksize = 1,
3019 },
3020 .setkey = aead_setkey,
3021 .setauthsize = aead_setauthsize,
3022 .encrypt = aead_encrypt,
3023 .decrypt = aead_decrypt,
3024 .ivsize = CTR_RFC3686_IV_SIZE,
3025 .maxauthsize = SHA1_DIGEST_SIZE,
3026 },
3027 .caam = {
3028 .class1_alg_type = OP_ALG_ALGSEL_AES |
3029 OP_ALG_AAI_CTR_MOD128,
3030 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3031 OP_ALG_AAI_HMAC_PRECOMP,
3032 .rfc3686 = true,
3033 },
3034 },
3035 {
3036 .aead = {
3037 .base = {
3038 .cra_name = "seqiv(authenc("
3039 "hmac(sha1),rfc3686(ctr(aes))))",
3040 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
3041 "rfc3686-ctr-aes-caam",
3042 .cra_blocksize = 1,
3043 },
3044 .setkey = aead_setkey,
3045 .setauthsize = aead_setauthsize,
3046 .encrypt = aead_encrypt,
3047 .decrypt = aead_decrypt,
3048 .ivsize = CTR_RFC3686_IV_SIZE,
3049 .maxauthsize = SHA1_DIGEST_SIZE,
3050 },
3051 .caam = {
3052 .class1_alg_type = OP_ALG_ALGSEL_AES |
3053 OP_ALG_AAI_CTR_MOD128,
3054 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3055 OP_ALG_AAI_HMAC_PRECOMP,
3056 .rfc3686 = true,
3057 .geniv = true,
3058 },
3059 },
3060 {
3061 .aead = {
3062 .base = {
3063 .cra_name = "authenc(hmac(sha224),"
3064 "rfc3686(ctr(aes)))",
3065 .cra_driver_name = "authenc-hmac-sha224-"
3066 "rfc3686-ctr-aes-caam",
3067 .cra_blocksize = 1,
3068 },
3069 .setkey = aead_setkey,
3070 .setauthsize = aead_setauthsize,
3071 .encrypt = aead_encrypt,
3072 .decrypt = aead_decrypt,
3073 .ivsize = CTR_RFC3686_IV_SIZE,
3074 .maxauthsize = SHA224_DIGEST_SIZE,
3075 },
3076 .caam = {
3077 .class1_alg_type = OP_ALG_ALGSEL_AES |
3078 OP_ALG_AAI_CTR_MOD128,
3079 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3080 OP_ALG_AAI_HMAC_PRECOMP,
3081 .rfc3686 = true,
3082 },
3083 },
3084 {
3085 .aead = {
3086 .base = {
3087 .cra_name = "seqiv(authenc("
3088 "hmac(sha224),rfc3686(ctr(aes))))",
3089 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
3090 "rfc3686-ctr-aes-caam",
3091 .cra_blocksize = 1,
3092 },
3093 .setkey = aead_setkey,
3094 .setauthsize = aead_setauthsize,
3095 .encrypt = aead_encrypt,
3096 .decrypt = aead_decrypt,
3097 .ivsize = CTR_RFC3686_IV_SIZE,
3098 .maxauthsize = SHA224_DIGEST_SIZE,
3099 },
3100 .caam = {
3101 .class1_alg_type = OP_ALG_ALGSEL_AES |
3102 OP_ALG_AAI_CTR_MOD128,
3103 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3104 OP_ALG_AAI_HMAC_PRECOMP,
3105 .rfc3686 = true,
3106 .geniv = true,
3107 },
3108 },
3109 {
3110 .aead = {
3111 .base = {
3112 .cra_name = "authenc(hmac(sha256),"
3113 "rfc3686(ctr(aes)))",
3114 .cra_driver_name = "authenc-hmac-sha256-"
3115 "rfc3686-ctr-aes-caam",
3116 .cra_blocksize = 1,
3117 },
3118 .setkey = aead_setkey,
3119 .setauthsize = aead_setauthsize,
3120 .encrypt = aead_encrypt,
3121 .decrypt = aead_decrypt,
3122 .ivsize = CTR_RFC3686_IV_SIZE,
3123 .maxauthsize = SHA256_DIGEST_SIZE,
3124 },
3125 .caam = {
3126 .class1_alg_type = OP_ALG_ALGSEL_AES |
3127 OP_ALG_AAI_CTR_MOD128,
3128 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3129 OP_ALG_AAI_HMAC_PRECOMP,
3130 .rfc3686 = true,
3131 },
3132 },
3133 {
3134 .aead = {
3135 .base = {
3136 .cra_name = "seqiv(authenc(hmac(sha256),"
3137 "rfc3686(ctr(aes))))",
3138 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
3139 "rfc3686-ctr-aes-caam",
3140 .cra_blocksize = 1,
3141 },
3142 .setkey = aead_setkey,
3143 .setauthsize = aead_setauthsize,
3144 .encrypt = aead_encrypt,
3145 .decrypt = aead_decrypt,
3146 .ivsize = CTR_RFC3686_IV_SIZE,
3147 .maxauthsize = SHA256_DIGEST_SIZE,
3148 },
3149 .caam = {
3150 .class1_alg_type = OP_ALG_ALGSEL_AES |
3151 OP_ALG_AAI_CTR_MOD128,
3152 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3153 OP_ALG_AAI_HMAC_PRECOMP,
3154 .rfc3686 = true,
3155 .geniv = true,
3156 },
3157 },
3158 {
3159 .aead = {
3160 .base = {
3161 .cra_name = "authenc(hmac(sha384),"
3162 "rfc3686(ctr(aes)))",
3163 .cra_driver_name = "authenc-hmac-sha384-"
3164 "rfc3686-ctr-aes-caam",
3165 .cra_blocksize = 1,
3166 },
3167 .setkey = aead_setkey,
3168 .setauthsize = aead_setauthsize,
3169 .encrypt = aead_encrypt,
3170 .decrypt = aead_decrypt,
3171 .ivsize = CTR_RFC3686_IV_SIZE,
3172 .maxauthsize = SHA384_DIGEST_SIZE,
3173 },
3174 .caam = {
3175 .class1_alg_type = OP_ALG_ALGSEL_AES |
3176 OP_ALG_AAI_CTR_MOD128,
3177 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3178 OP_ALG_AAI_HMAC_PRECOMP,
3179 .rfc3686 = true,
3180 },
3181 },
3182 {
3183 .aead = {
3184 .base = {
3185 .cra_name = "seqiv(authenc(hmac(sha384),"
3186 "rfc3686(ctr(aes))))",
3187 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
3188 "rfc3686-ctr-aes-caam",
3189 .cra_blocksize = 1,
3190 },
3191 .setkey = aead_setkey,
3192 .setauthsize = aead_setauthsize,
3193 .encrypt = aead_encrypt,
3194 .decrypt = aead_decrypt,
3195 .ivsize = CTR_RFC3686_IV_SIZE,
3196 .maxauthsize = SHA384_DIGEST_SIZE,
3197 },
3198 .caam = {
3199 .class1_alg_type = OP_ALG_ALGSEL_AES |
3200 OP_ALG_AAI_CTR_MOD128,
3201 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3202 OP_ALG_AAI_HMAC_PRECOMP,
3203 .rfc3686 = true,
3204 .geniv = true,
3205 },
3206 },
3207 {
3208 .aead = {
3209 .base = {
3210 .cra_name = "authenc(hmac(sha512),"
3211 "rfc3686(ctr(aes)))",
3212 .cra_driver_name = "authenc-hmac-sha512-"
3213 "rfc3686-ctr-aes-caam",
3214 .cra_blocksize = 1,
3215 },
3216 .setkey = aead_setkey,
3217 .setauthsize = aead_setauthsize,
3218 .encrypt = aead_encrypt,
3219 .decrypt = aead_decrypt,
3220 .ivsize = CTR_RFC3686_IV_SIZE,
3221 .maxauthsize = SHA512_DIGEST_SIZE,
3222 },
3223 .caam = {
3224 .class1_alg_type = OP_ALG_ALGSEL_AES |
3225 OP_ALG_AAI_CTR_MOD128,
3226 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3227 OP_ALG_AAI_HMAC_PRECOMP,
3228 .rfc3686 = true,
3229 },
3230 },
3231 {
3232 .aead = {
3233 .base = {
3234 .cra_name = "seqiv(authenc(hmac(sha512),"
3235 "rfc3686(ctr(aes))))",
3236 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
3237 "rfc3686-ctr-aes-caam",
3238 .cra_blocksize = 1,
3239 },
3240 .setkey = aead_setkey,
3241 .setauthsize = aead_setauthsize,
3242 .encrypt = aead_encrypt,
3243 .decrypt = aead_decrypt,
3244 .ivsize = CTR_RFC3686_IV_SIZE,
3245 .maxauthsize = SHA512_DIGEST_SIZE,
3246 },
3247 .caam = {
3248 .class1_alg_type = OP_ALG_ALGSEL_AES |
3249 OP_ALG_AAI_CTR_MOD128,
3250 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3251 OP_ALG_AAI_HMAC_PRECOMP,
3252 .rfc3686 = true,
3253 .geniv = true,
3254 },
3255 },
3256 };
3257
3258 struct caam_crypto_alg {
3259 struct crypto_alg crypto_alg;
3260 struct list_head entry;
3261 struct caam_alg_entry caam;
3262 };
3263
3264 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam,
3265 bool uses_dkp)
3266 {
3267 dma_addr_t dma_addr;
3268 struct caam_drv_private *priv;
3269
3270 ctx->jrdev = caam_jr_alloc();
3271 if (IS_ERR(ctx->jrdev)) {
3272 pr_err("Job Ring Device allocation for transform failed\n");
3273 return PTR_ERR(ctx->jrdev);
3274 }
3275
3276 priv = dev_get_drvdata(ctx->jrdev->parent);
3277 if (priv->era >= 6 && uses_dkp)
3278 ctx->dir = DMA_BIDIRECTIONAL;
3279 else
3280 ctx->dir = DMA_TO_DEVICE;
3281
3282 dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc,
3283 offsetof(struct caam_ctx,
3284 sh_desc_enc_dma),
3285 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3286 if (dma_mapping_error(ctx->jrdev, dma_addr)) {
3287 dev_err(ctx->jrdev, "unable to map key, shared descriptors\n");
3288 caam_jr_free(ctx->jrdev);
3289 return -ENOMEM;
3290 }
3291
3292 ctx->sh_desc_enc_dma = dma_addr;
3293 ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx,
3294 sh_desc_dec);
3295 ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx,
3296 sh_desc_givenc);
3297 ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key);
3298
3299 /* copy descriptor header template value */
3300 ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
3301 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
3302
3303 return 0;
3304 }
3305
3306 static int caam_cra_init(struct crypto_tfm *tfm)
3307 {
3308 struct crypto_alg *alg = tfm->__crt_alg;
3309 struct caam_crypto_alg *caam_alg =
3310 container_of(alg, struct caam_crypto_alg, crypto_alg);
3311 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
3312
3313 return caam_init_common(ctx, &caam_alg->caam, false);
3314 }
3315
3316 static int caam_aead_init(struct crypto_aead *tfm)
3317 {
3318 struct aead_alg *alg = crypto_aead_alg(tfm);
3319 struct caam_aead_alg *caam_alg =
3320 container_of(alg, struct caam_aead_alg, aead);
3321 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
3322
3323 return caam_init_common(ctx, &caam_alg->caam,
3324 alg->setkey == aead_setkey);
3325 }
3326
3327 static void caam_exit_common(struct caam_ctx *ctx)
3328 {
3329 dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma,
3330 offsetof(struct caam_ctx, sh_desc_enc_dma),
3331 ctx->dir, DMA_ATTR_SKIP_CPU_SYNC);
3332 caam_jr_free(ctx->jrdev);
3333 }
3334
3335 static void caam_cra_exit(struct crypto_tfm *tfm)
3336 {
3337 caam_exit_common(crypto_tfm_ctx(tfm));
3338 }
3339
3340 static void caam_aead_exit(struct crypto_aead *tfm)
3341 {
3342 caam_exit_common(crypto_aead_ctx(tfm));
3343 }
3344
3345 static void __exit caam_algapi_exit(void)
3346 {
3347
3348 struct caam_crypto_alg *t_alg, *n;
3349 int i;
3350
3351 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3352 struct caam_aead_alg *t_alg = driver_aeads + i;
3353
3354 if (t_alg->registered)
3355 crypto_unregister_aead(&t_alg->aead);
3356 }
3357
3358 if (!alg_list.next)
3359 return;
3360
3361 list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
3362 crypto_unregister_alg(&t_alg->crypto_alg);
3363 list_del(&t_alg->entry);
3364 kfree(t_alg);
3365 }
3366 }
3367
3368 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
3369 *template)
3370 {
3371 struct caam_crypto_alg *t_alg;
3372 struct crypto_alg *alg;
3373
3374 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
3375 if (!t_alg) {
3376 pr_err("failed to allocate t_alg\n");
3377 return ERR_PTR(-ENOMEM);
3378 }
3379
3380 alg = &t_alg->crypto_alg;
3381
3382 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
3383 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
3384 template->driver_name);
3385 alg->cra_module = THIS_MODULE;
3386 alg->cra_init = caam_cra_init;
3387 alg->cra_exit = caam_cra_exit;
3388 alg->cra_priority = CAAM_CRA_PRIORITY;
3389 alg->cra_blocksize = template->blocksize;
3390 alg->cra_alignmask = 0;
3391 alg->cra_ctxsize = sizeof(struct caam_ctx);
3392 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
3393 template->type;
3394 switch (template->type) {
3395 case CRYPTO_ALG_TYPE_GIVCIPHER:
3396 alg->cra_type = &crypto_givcipher_type;
3397 alg->cra_ablkcipher = template->template_ablkcipher;
3398 break;
3399 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3400 alg->cra_type = &crypto_ablkcipher_type;
3401 alg->cra_ablkcipher = template->template_ablkcipher;
3402 break;
3403 }
3404
3405 t_alg->caam.class1_alg_type = template->class1_alg_type;
3406 t_alg->caam.class2_alg_type = template->class2_alg_type;
3407
3408 return t_alg;
3409 }
3410
3411 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
3412 {
3413 struct aead_alg *alg = &t_alg->aead;
3414
3415 alg->base.cra_module = THIS_MODULE;
3416 alg->base.cra_priority = CAAM_CRA_PRIORITY;
3417 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
3418 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
3419
3420 alg->init = caam_aead_init;
3421 alg->exit = caam_aead_exit;
3422 }
3423
3424 static int __init caam_algapi_init(void)
3425 {
3426 struct device_node *dev_node;
3427 struct platform_device *pdev;
3428 struct device *ctrldev;
3429 struct caam_drv_private *priv;
3430 int i = 0, err = 0;
3431 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
3432 unsigned int md_limit = SHA512_DIGEST_SIZE;
3433 bool registered = false;
3434
3435 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
3436 if (!dev_node) {
3437 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
3438 if (!dev_node)
3439 return -ENODEV;
3440 }
3441
3442 pdev = of_find_device_by_node(dev_node);
3443 if (!pdev) {
3444 of_node_put(dev_node);
3445 return -ENODEV;
3446 }
3447
3448 ctrldev = &pdev->dev;
3449 priv = dev_get_drvdata(ctrldev);
3450 of_node_put(dev_node);
3451
3452 /*
3453 * If priv is NULL, it's probably because the caam driver wasn't
3454 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
3455 */
3456 if (!priv)
3457 return -ENODEV;
3458
3459
3460 INIT_LIST_HEAD(&alg_list);
3461
3462 /*
3463 * Register crypto algorithms the device supports.
3464 * First, detect presence and attributes of DES, AES, and MD blocks.
3465 */
3466 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
3467 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
3468 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
3469 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
3470 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
3471
3472 /* If MD is present, limit digest size based on LP256 */
3473 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
3474 md_limit = SHA256_DIGEST_SIZE;
3475
3476 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3477 struct caam_crypto_alg *t_alg;
3478 struct caam_alg_template *alg = driver_algs + i;
3479 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
3480
3481 /* Skip DES algorithms if not supported by device */
3482 if (!des_inst &&
3483 ((alg_sel == OP_ALG_ALGSEL_3DES) ||
3484 (alg_sel == OP_ALG_ALGSEL_DES)))
3485 continue;
3486
3487 /* Skip AES algorithms if not supported by device */
3488 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
3489 continue;
3490
3491 /*
3492 * Check support for AES modes not available
3493 * on LP devices.
3494 */
3495 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
3496 if ((alg->class1_alg_type & OP_ALG_AAI_MASK) ==
3497 OP_ALG_AAI_XTS)
3498 continue;
3499
3500 t_alg = caam_alg_alloc(alg);
3501 if (IS_ERR(t_alg)) {
3502 err = PTR_ERR(t_alg);
3503 pr_warn("%s alg allocation failed\n", alg->driver_name);
3504 continue;
3505 }
3506
3507 err = crypto_register_alg(&t_alg->crypto_alg);
3508 if (err) {
3509 pr_warn("%s alg registration failed\n",
3510 t_alg->crypto_alg.cra_driver_name);
3511 kfree(t_alg);
3512 continue;
3513 }
3514
3515 list_add_tail(&t_alg->entry, &alg_list);
3516 registered = true;
3517 }
3518
3519 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
3520 struct caam_aead_alg *t_alg = driver_aeads + i;
3521 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
3522 OP_ALG_ALGSEL_MASK;
3523 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
3524 OP_ALG_ALGSEL_MASK;
3525 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
3526
3527 /* Skip DES algorithms if not supported by device */
3528 if (!des_inst &&
3529 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
3530 (c1_alg_sel == OP_ALG_ALGSEL_DES)))
3531 continue;
3532
3533 /* Skip AES algorithms if not supported by device */
3534 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
3535 continue;
3536
3537 /*
3538 * Check support for AES algorithms not available
3539 * on LP devices.
3540 */
3541 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
3542 if (alg_aai == OP_ALG_AAI_GCM)
3543 continue;
3544
3545 /*
3546 * Skip algorithms requiring message digests
3547 * if MD or MD size is not supported by device.
3548 */
3549 if (c2_alg_sel &&
3550 (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
3551 continue;
3552
3553 caam_aead_alg_init(t_alg);
3554
3555 err = crypto_register_aead(&t_alg->aead);
3556 if (err) {
3557 pr_warn("%s alg registration failed\n",
3558 t_alg->aead.base.cra_driver_name);
3559 continue;
3560 }
3561
3562 t_alg->registered = true;
3563 registered = true;
3564 }
3565
3566 if (registered)
3567 pr_info("caam algorithms registered in /proc/crypto\n");
3568
3569 return err;
3570 }
3571
3572 module_init(caam_algapi_init);
3573 module_exit(caam_algapi_exit);
3574
3575 MODULE_LICENSE("GPL");
3576 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
3577 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
CRIS linux kernel tree