?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 5.1.15
[linux/fpc-iii.git] / drivers / crypto / caam / caamalg_desc.c
blob1e1a376edc2f1d00231e3a551eac81b1ba6e05db
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Shared descriptors for aead, skcipher algorithms
4 *
5 * Copyright 2016-2019 NXP
6 */
7
8 #include "compat.h"
9 #include "desc_constr.h"
10 #include "caamalg_desc.h"
11
12 /*
13 * For aead functions, read payload and write payload,
14 * both of which are specified in req->src and req->dst
15 */
16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17 {
18 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21 }
22
23 /* Set DK bit in class 1 operation if shared */
24 static inline void append_dec_op1(u32 *desc, u32 type)
25 {
26 u32 *jump_cmd, *uncond_jump_cmd;
27
28 /* DK bit is valid only for AES */
29 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 append_operation(desc, type | OP_ALG_AS_INITFINAL |
31 OP_ALG_DECRYPT);
32 return;
33 }
34
35 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 append_operation(desc, type | OP_ALG_AS_INITFINAL |
37 OP_ALG_DECRYPT);
38 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
39 set_jump_tgt_here(desc, jump_cmd);
40 append_operation(desc, type | OP_ALG_AS_INITFINAL |
41 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
42 set_jump_tgt_here(desc, uncond_jump_cmd);
43 }
44
45 /**
46 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
47 * (non-protocol) with no (null) encryption.
48 * @desc: pointer to buffer used for descriptor construction
49 * @adata: pointer to authentication transform definitions.
50 * A split key is required for SEC Era < 6; the size of the split key
51 * is specified in this case. Valid algorithm values - one of
52 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
53 * with OP_ALG_AAI_HMAC_PRECOMP.
54 * @icvsize: integrity check value (ICV) size (truncated or full)
55 * @era: SEC Era
56 */
57 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
58 unsigned int icvsize, int era)
59 {
60 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
61
62 init_sh_desc(desc, HDR_SHARE_SERIAL);
63
64 /* Skip if already shared */
65 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
66 JUMP_COND_SHRD);
67 if (era < 6) {
68 if (adata->key_inline)
69 append_key_as_imm(desc, adata->key_virt,
70 adata->keylen_pad, adata->keylen,
71 CLASS_2 | KEY_DEST_MDHA_SPLIT |
72 KEY_ENC);
73 else
74 append_key(desc, adata->key_dma, adata->keylen,
75 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
76 } else {
77 append_proto_dkp(desc, adata);
78 }
79 set_jump_tgt_here(desc, key_jump_cmd);
80
81 /* assoclen + cryptlen = seqinlen */
82 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
83
84 /* Prepare to read and write cryptlen + assoclen bytes */
85 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
86 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
87
88 /*
89 * MOVE_LEN opcode is not available in all SEC HW revisions,
90 * thus need to do some magic, i.e. self-patch the descriptor
91 * buffer.
92 */
93 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
94 MOVE_DEST_MATH3 |
95 (0x6 << MOVE_LEN_SHIFT));
96 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
97 MOVE_DEST_DESCBUF |
98 MOVE_WAITCOMP |
99 (0x8 << MOVE_LEN_SHIFT));
100
101 /* Class 2 operation */
102 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
103 OP_ALG_ENCRYPT);
104
105 /* Read and write cryptlen bytes */
106 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
107
108 set_move_tgt_here(desc, read_move_cmd);
109 set_move_tgt_here(desc, write_move_cmd);
110 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
111 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
112 MOVE_AUX_LS);
113
114 /* Write ICV */
115 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
116 LDST_SRCDST_BYTE_CONTEXT);
117
118 #ifdef DEBUG
119 print_hex_dump(KERN_ERR,
120 "aead null enc shdesc@" __stringify(__LINE__)": ",
121 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
122 #endif
123 }
124 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
125
126 /**
127 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
128 * (non-protocol) with no (null) decryption.
129 * @desc: pointer to buffer used for descriptor construction
130 * @adata: pointer to authentication transform definitions.
131 * A split key is required for SEC Era < 6; the size of the split key
132 * is specified in this case. Valid algorithm values - one of
133 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
134 * with OP_ALG_AAI_HMAC_PRECOMP.
135 * @icvsize: integrity check value (ICV) size (truncated or full)
136 * @era: SEC Era
137 */
138 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
139 unsigned int icvsize, int era)
140 {
141 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
142
143 init_sh_desc(desc, HDR_SHARE_SERIAL);
144
145 /* Skip if already shared */
146 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
147 JUMP_COND_SHRD);
148 if (era < 6) {
149 if (adata->key_inline)
150 append_key_as_imm(desc, adata->key_virt,
151 adata->keylen_pad, adata->keylen,
152 CLASS_2 | KEY_DEST_MDHA_SPLIT |
153 KEY_ENC);
154 else
155 append_key(desc, adata->key_dma, adata->keylen,
156 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
157 } else {
158 append_proto_dkp(desc, adata);
159 }
160 set_jump_tgt_here(desc, key_jump_cmd);
161
162 /* Class 2 operation */
163 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
164 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
165
166 /* assoclen + cryptlen = seqoutlen */
167 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
168
169 /* Prepare to read and write cryptlen + assoclen bytes */
170 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
171 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
172
173 /*
174 * MOVE_LEN opcode is not available in all SEC HW revisions,
175 * thus need to do some magic, i.e. self-patch the descriptor
176 * buffer.
177 */
178 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
179 MOVE_DEST_MATH2 |
180 (0x6 << MOVE_LEN_SHIFT));
181 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
182 MOVE_DEST_DESCBUF |
183 MOVE_WAITCOMP |
184 (0x8 << MOVE_LEN_SHIFT));
185
186 /* Read and write cryptlen bytes */
187 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
188
189 /*
190 * Insert a NOP here, since we need at least 4 instructions between
191 * code patching the descriptor buffer and the location being patched.
192 */
193 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
194 set_jump_tgt_here(desc, jump_cmd);
195
196 set_move_tgt_here(desc, read_move_cmd);
197 set_move_tgt_here(desc, write_move_cmd);
198 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
199 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
200 MOVE_AUX_LS);
201 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
202
203 /* Load ICV */
204 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
205 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
206
207 #ifdef DEBUG
208 print_hex_dump(KERN_ERR,
209 "aead null dec shdesc@" __stringify(__LINE__)": ",
210 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
211 #endif
212 }
213 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
214
215 static void init_sh_desc_key_aead(u32 * const desc,
216 struct alginfo * const cdata,
217 struct alginfo * const adata,
218 const bool is_rfc3686, u32 *nonce, int era)
219 {
220 u32 *key_jump_cmd;
221 unsigned int enckeylen = cdata->keylen;
222
223 /* Note: Context registers are saved. */
224 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
225
226 /* Skip if already shared */
227 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
228 JUMP_COND_SHRD);
229
230 /*
231 * RFC3686 specific:
232 * | key = {AUTH_KEY, ENC_KEY, NONCE}
233 * | enckeylen = encryption key size + nonce size
234 */
235 if (is_rfc3686)
236 enckeylen -= CTR_RFC3686_NONCE_SIZE;
237
238 if (era < 6) {
239 if (adata->key_inline)
240 append_key_as_imm(desc, adata->key_virt,
241 adata->keylen_pad, adata->keylen,
242 CLASS_2 | KEY_DEST_MDHA_SPLIT |
243 KEY_ENC);
244 else
245 append_key(desc, adata->key_dma, adata->keylen,
246 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
247 } else {
248 append_proto_dkp(desc, adata);
249 }
250
251 if (cdata->key_inline)
252 append_key_as_imm(desc, cdata->key_virt, enckeylen,
253 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
254 else
255 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
256 KEY_DEST_CLASS_REG);
257
258 /* Load Counter into CONTEXT1 reg */
259 if (is_rfc3686) {
260 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
261 LDST_CLASS_IND_CCB |
262 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
263 append_move(desc,
264 MOVE_SRC_OUTFIFO |
265 MOVE_DEST_CLASS1CTX |
266 (16 << MOVE_OFFSET_SHIFT) |
267 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
268 }
269
270 set_jump_tgt_here(desc, key_jump_cmd);
271 }
272
273 /**
274 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
275 * (non-protocol).
276 * @desc: pointer to buffer used for descriptor construction
277 * @cdata: pointer to block cipher transform definitions
278 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
279 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
280 * @adata: pointer to authentication transform definitions.
281 * A split key is required for SEC Era < 6; the size of the split key
282 * is specified in this case. Valid algorithm values - one of
283 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
284 * with OP_ALG_AAI_HMAC_PRECOMP.
285 * @ivsize: initialization vector size
286 * @icvsize: integrity check value (ICV) size (truncated or full)
287 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
288 * @nonce: pointer to rfc3686 nonce
289 * @ctx1_iv_off: IV offset in CONTEXT1 register
290 * @is_qi: true when called from caam/qi
291 * @era: SEC Era
292 */
293 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
294 struct alginfo *adata, unsigned int ivsize,
295 unsigned int icvsize, const bool is_rfc3686,
296 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
297 int era)
298 {
299 /* Note: Context registers are saved. */
300 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
301
302 /* Class 2 operation */
303 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
304 OP_ALG_ENCRYPT);
305
306 if (is_qi) {
307 u32 *wait_load_cmd;
308
309 /* REG3 = assoclen */
310 append_seq_load(desc, 4, LDST_CLASS_DECO |
311 LDST_SRCDST_WORD_DECO_MATH3 |
312 (4 << LDST_OFFSET_SHIFT));
313
314 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
315 JUMP_COND_CALM | JUMP_COND_NCP |
316 JUMP_COND_NOP | JUMP_COND_NIP |
317 JUMP_COND_NIFP);
318 set_jump_tgt_here(desc, wait_load_cmd);
319
320 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
321 LDST_SRCDST_BYTE_CONTEXT |
322 (ctx1_iv_off << LDST_OFFSET_SHIFT));
323 }
324
325 /* Read and write assoclen bytes */
326 if (is_qi || era < 3) {
327 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
328 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
329 } else {
330 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
331 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
332 }
333
334 /* Skip assoc data */
335 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
336
337 /* read assoc before reading payload */
338 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
339 FIFOLDST_VLF);
340
341 /* Load Counter into CONTEXT1 reg */
342 if (is_rfc3686)
343 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
344 LDST_SRCDST_BYTE_CONTEXT |
345 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
346 LDST_OFFSET_SHIFT));
347
348 /* Class 1 operation */
349 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
350 OP_ALG_ENCRYPT);
351
352 /* Read and write cryptlen bytes */
353 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
354 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
355 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
356
357 /* Write ICV */
358 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
359 LDST_SRCDST_BYTE_CONTEXT);
360
361 #ifdef DEBUG
362 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
363 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
364 #endif
365 }
366 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
367
368 /**
369 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
370 * (non-protocol).
371 * @desc: pointer to buffer used for descriptor construction
372 * @cdata: pointer to block cipher transform definitions
373 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
374 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
375 * @adata: pointer to authentication transform definitions.
376 * A split key is required for SEC Era < 6; the size of the split key
377 * is specified in this case. Valid algorithm values - one of
378 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
379 * with OP_ALG_AAI_HMAC_PRECOMP.
380 * @ivsize: initialization vector size
381 * @icvsize: integrity check value (ICV) size (truncated or full)
382 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
383 * @nonce: pointer to rfc3686 nonce
384 * @ctx1_iv_off: IV offset in CONTEXT1 register
385 * @is_qi: true when called from caam/qi
386 * @era: SEC Era
387 */
388 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
389 struct alginfo *adata, unsigned int ivsize,
390 unsigned int icvsize, const bool geniv,
391 const bool is_rfc3686, u32 *nonce,
392 const u32 ctx1_iv_off, const bool is_qi, int era)
393 {
394 /* Note: Context registers are saved. */
395 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
396
397 /* Class 2 operation */
398 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
399 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
400
401 if (is_qi) {
402 u32 *wait_load_cmd;
403
404 /* REG3 = assoclen */
405 append_seq_load(desc, 4, LDST_CLASS_DECO |
406 LDST_SRCDST_WORD_DECO_MATH3 |
407 (4 << LDST_OFFSET_SHIFT));
408
409 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
410 JUMP_COND_CALM | JUMP_COND_NCP |
411 JUMP_COND_NOP | JUMP_COND_NIP |
412 JUMP_COND_NIFP);
413 set_jump_tgt_here(desc, wait_load_cmd);
414
415 if (!geniv)
416 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
417 LDST_SRCDST_BYTE_CONTEXT |
418 (ctx1_iv_off << LDST_OFFSET_SHIFT));
419 }
420
421 /* Read and write assoclen bytes */
422 if (is_qi || era < 3) {
423 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
424 if (geniv)
425 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
426 ivsize);
427 else
428 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
429 CAAM_CMD_SZ);
430 } else {
431 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
432 if (geniv)
433 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
434 ivsize);
435 else
436 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
437 CAAM_CMD_SZ);
438 }
439
440 /* Skip assoc data */
441 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
442
443 /* read assoc before reading payload */
444 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
445 KEY_VLF);
446
447 if (geniv) {
448 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
449 LDST_SRCDST_BYTE_CONTEXT |
450 (ctx1_iv_off << LDST_OFFSET_SHIFT));
451 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
452 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
453 }
454
455 /* Load Counter into CONTEXT1 reg */
456 if (is_rfc3686)
457 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
458 LDST_SRCDST_BYTE_CONTEXT |
459 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
460 LDST_OFFSET_SHIFT));
461
462 /* Choose operation */
463 if (ctx1_iv_off)
464 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
465 OP_ALG_DECRYPT);
466 else
467 append_dec_op1(desc, cdata->algtype);
468
469 /* Read and write cryptlen bytes */
470 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
471 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
472 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
473
474 /* Load ICV */
475 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
476 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
477
478 #ifdef DEBUG
479 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
480 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
481 #endif
482 }
483 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
484
485 /**
486 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
487 * (non-protocol) with HW-generated initialization
488 * vector.
489 * @desc: pointer to buffer used for descriptor construction
490 * @cdata: pointer to block cipher transform definitions
491 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
492 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
493 * @adata: pointer to authentication transform definitions.
494 * A split key is required for SEC Era < 6; the size of the split key
495 * is specified in this case. Valid algorithm values - one of
496 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
497 * with OP_ALG_AAI_HMAC_PRECOMP.
498 * @ivsize: initialization vector size
499 * @icvsize: integrity check value (ICV) size (truncated or full)
500 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
501 * @nonce: pointer to rfc3686 nonce
502 * @ctx1_iv_off: IV offset in CONTEXT1 register
503 * @is_qi: true when called from caam/qi
504 * @era: SEC Era
505 */
506 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
507 struct alginfo *adata, unsigned int ivsize,
508 unsigned int icvsize, const bool is_rfc3686,
509 u32 *nonce, const u32 ctx1_iv_off,
510 const bool is_qi, int era)
511 {
512 u32 geniv, moveiv;
513
514 /* Note: Context registers are saved. */
515 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
516
517 if (is_qi) {
518 u32 *wait_load_cmd;
519
520 /* REG3 = assoclen */
521 append_seq_load(desc, 4, LDST_CLASS_DECO |
522 LDST_SRCDST_WORD_DECO_MATH3 |
523 (4 << LDST_OFFSET_SHIFT));
524
525 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
526 JUMP_COND_CALM | JUMP_COND_NCP |
527 JUMP_COND_NOP | JUMP_COND_NIP |
528 JUMP_COND_NIFP);
529 set_jump_tgt_here(desc, wait_load_cmd);
530 }
531
532 if (is_rfc3686) {
533 if (is_qi)
534 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
535 LDST_SRCDST_BYTE_CONTEXT |
536 (ctx1_iv_off << LDST_OFFSET_SHIFT));
537
538 goto copy_iv;
539 }
540
541 /* Generate IV */
542 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
543 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
544 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
545 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
546 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
547 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
548 append_move(desc, MOVE_WAITCOMP |
549 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
550 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
551 (ivsize << MOVE_LEN_SHIFT));
552 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
553
554 copy_iv:
555 /* Copy IV to class 1 context */
556 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
557 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
558 (ivsize << MOVE_LEN_SHIFT));
559
560 /* Return to encryption */
561 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
562 OP_ALG_ENCRYPT);
563
564 /* Read and write assoclen bytes */
565 if (is_qi || era < 3) {
566 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
567 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
568 } else {
569 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
570 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
571 }
572
573 /* Skip assoc data */
574 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
575
576 /* read assoc before reading payload */
577 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
578 KEY_VLF);
579
580 /* Copy iv from outfifo to class 2 fifo */
581 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
582 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
583 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
584 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
585 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
586 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
587
588 /* Load Counter into CONTEXT1 reg */
589 if (is_rfc3686)
590 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
591 LDST_SRCDST_BYTE_CONTEXT |
592 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
593 LDST_OFFSET_SHIFT));
594
595 /* Class 1 operation */
596 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
597 OP_ALG_ENCRYPT);
598
599 /* Will write ivsize + cryptlen */
600 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
601
602 /* Not need to reload iv */
603 append_seq_fifo_load(desc, ivsize,
604 FIFOLD_CLASS_SKIP);
605
606 /* Will read cryptlen */
607 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
608 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
609 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
610 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
611
612 /* Write ICV */
613 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
614 LDST_SRCDST_BYTE_CONTEXT);
615
616 #ifdef DEBUG
617 print_hex_dump(KERN_ERR,
618 "aead givenc shdesc@" __stringify(__LINE__)": ",
619 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
620 #endif
621 }
622 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
623
624 /**
625 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
626 * @desc: pointer to buffer used for descriptor construction
627 * @cdata: pointer to block cipher transform definitions
628 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
629 * @ivsize: initialization vector size
630 * @icvsize: integrity check value (ICV) size (truncated or full)
631 * @is_qi: true when called from caam/qi
632 */
633 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
634 unsigned int ivsize, unsigned int icvsize,
635 const bool is_qi)
636 {
637 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
638 *zero_assoc_jump_cmd2;
639
640 init_sh_desc(desc, HDR_SHARE_SERIAL);
641
642 /* skip key loading if they are loaded due to sharing */
643 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
644 JUMP_COND_SHRD);
645 if (cdata->key_inline)
646 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
647 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
648 else
649 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
650 KEY_DEST_CLASS_REG);
651 set_jump_tgt_here(desc, key_jump_cmd);
652
653 /* class 1 operation */
654 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
655 OP_ALG_ENCRYPT);
656
657 if (is_qi) {
658 u32 *wait_load_cmd;
659
660 /* REG3 = assoclen */
661 append_seq_load(desc, 4, LDST_CLASS_DECO |
662 LDST_SRCDST_WORD_DECO_MATH3 |
663 (4 << LDST_OFFSET_SHIFT));
664
665 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
666 JUMP_COND_CALM | JUMP_COND_NCP |
667 JUMP_COND_NOP | JUMP_COND_NIP |
668 JUMP_COND_NIFP);
669 set_jump_tgt_here(desc, wait_load_cmd);
670
671 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
672 ivsize);
673 } else {
674 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
675 CAAM_CMD_SZ);
676 }
677
678 /* if assoclen + cryptlen is ZERO, skip to ICV write */
679 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
680 JUMP_COND_MATH_Z);
681
682 if (is_qi)
683 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
684 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
685
686 /* if assoclen is ZERO, skip reading the assoc data */
687 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
688 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
689 JUMP_COND_MATH_Z);
690
691 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
692
693 /* skip assoc data */
694 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
695
696 /* cryptlen = seqinlen - assoclen */
697 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
698
699 /* if cryptlen is ZERO jump to zero-payload commands */
700 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
701 JUMP_COND_MATH_Z);
702
703 /* read assoc data */
704 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
705 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
706 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
707
708 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
709
710 /* write encrypted data */
711 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
712
713 /* read payload data */
714 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
715 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
716
717 /* jump to ICV writing */
718 if (is_qi)
719 append_jump(desc, JUMP_TEST_ALL | 4);
720 else
721 append_jump(desc, JUMP_TEST_ALL | 2);
722
723 /* zero-payload commands */
724 set_jump_tgt_here(desc, zero_payload_jump_cmd);
725
726 /* read assoc data */
727 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
728 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
729 if (is_qi)
730 /* jump to ICV writing */
731 append_jump(desc, JUMP_TEST_ALL | 2);
732
733 /* There is no input data */
734 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
735
736 if (is_qi)
737 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
738 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
739 FIFOLD_TYPE_LAST1);
740
741 /* write ICV */
742 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
743 LDST_SRCDST_BYTE_CONTEXT);
744
745 #ifdef DEBUG
746 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
747 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
748 #endif
749 }
750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
751
752 /**
753 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
754 * @desc: pointer to buffer used for descriptor construction
755 * @cdata: pointer to block cipher transform definitions
756 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757 * @ivsize: initialization vector size
758 * @icvsize: integrity check value (ICV) size (truncated or full)
759 * @is_qi: true when called from caam/qi
760 */
761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762 unsigned int ivsize, unsigned int icvsize,
763 const bool is_qi)
764 {
765 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
766
767 init_sh_desc(desc, HDR_SHARE_SERIAL);
768
769 /* skip key loading if they are loaded due to sharing */
770 key_jump_cmd = append_jump(desc, JUMP_JSL |
771 JUMP_TEST_ALL | JUMP_COND_SHRD);
772 if (cdata->key_inline)
773 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
774 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
775 else
776 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
777 KEY_DEST_CLASS_REG);
778 set_jump_tgt_here(desc, key_jump_cmd);
779
780 /* class 1 operation */
781 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
782 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
783
784 if (is_qi) {
785 u32 *wait_load_cmd;
786
787 /* REG3 = assoclen */
788 append_seq_load(desc, 4, LDST_CLASS_DECO |
789 LDST_SRCDST_WORD_DECO_MATH3 |
790 (4 << LDST_OFFSET_SHIFT));
791
792 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793 JUMP_COND_CALM | JUMP_COND_NCP |
794 JUMP_COND_NOP | JUMP_COND_NIP |
795 JUMP_COND_NIFP);
796 set_jump_tgt_here(desc, wait_load_cmd);
797
798 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
799 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
800 }
801
802 /* if assoclen is ZERO, skip reading the assoc data */
803 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
805 JUMP_COND_MATH_Z);
806
807 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
808
809 /* skip assoc data */
810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
811
812 /* read assoc data */
813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
815
816 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
817
818 /* cryptlen = seqoutlen - assoclen */
819 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
820
821 /* jump to zero-payload command if cryptlen is zero */
822 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
823 JUMP_COND_MATH_Z);
824
825 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
826
827 /* store encrypted data */
828 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
829
830 /* read payload data */
831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
833
834 /* zero-payload command */
835 set_jump_tgt_here(desc, zero_payload_jump_cmd);
836
837 /* read ICV */
838 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
839 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
840
841 #ifdef DEBUG
842 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
843 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
844 #endif
845 }
846 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
847
848 /**
849 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
850 * (non-protocol).
851 * @desc: pointer to buffer used for descriptor construction
852 * @cdata: pointer to block cipher transform definitions
853 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
854 * @ivsize: initialization vector size
855 * @icvsize: integrity check value (ICV) size (truncated or full)
856 * @is_qi: true when called from caam/qi
857 */
858 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
859 unsigned int ivsize, unsigned int icvsize,
860 const bool is_qi)
861 {
862 u32 *key_jump_cmd;
863
864 init_sh_desc(desc, HDR_SHARE_SERIAL);
865
866 /* Skip key loading if it is loaded due to sharing */
867 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
868 JUMP_COND_SHRD);
869 if (cdata->key_inline)
870 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
871 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
872 else
873 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
874 KEY_DEST_CLASS_REG);
875 set_jump_tgt_here(desc, key_jump_cmd);
876
877 /* Class 1 operation */
878 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
879 OP_ALG_ENCRYPT);
880
881 if (is_qi) {
882 u32 *wait_load_cmd;
883
884 /* REG3 = assoclen */
885 append_seq_load(desc, 4, LDST_CLASS_DECO |
886 LDST_SRCDST_WORD_DECO_MATH3 |
887 (4 << LDST_OFFSET_SHIFT));
888
889 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
890 JUMP_COND_CALM | JUMP_COND_NCP |
891 JUMP_COND_NOP | JUMP_COND_NIP |
892 JUMP_COND_NIFP);
893 set_jump_tgt_here(desc, wait_load_cmd);
894
895 /* Read salt and IV */
896 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
897 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
898 FIFOLD_TYPE_IV);
899 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
900 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
901 }
902
903 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
904 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
905
906 /* Read assoc data */
907 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
908 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
909
910 /* Skip IV */
911 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
912
913 /* Will read cryptlen bytes */
914 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
915
916 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
917 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
918
919 /* Skip assoc data */
920 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
921
922 /* cryptlen = seqoutlen - assoclen */
923 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
924
925 /* Write encrypted data */
926 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
927
928 /* Read payload data */
929 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
930 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
931
932 /* Write ICV */
933 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
934 LDST_SRCDST_BYTE_CONTEXT);
935
936 #ifdef DEBUG
937 print_hex_dump(KERN_ERR,
938 "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
939 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
940 #endif
941 }
942 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
943
944 /**
945 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
946 * (non-protocol).
947 * @desc: pointer to buffer used for descriptor construction
948 * @cdata: pointer to block cipher transform definitions
949 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
950 * @ivsize: initialization vector size
951 * @icvsize: integrity check value (ICV) size (truncated or full)
952 * @is_qi: true when called from caam/qi
953 */
954 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
955 unsigned int ivsize, unsigned int icvsize,
956 const bool is_qi)
957 {
958 u32 *key_jump_cmd;
959
960 init_sh_desc(desc, HDR_SHARE_SERIAL);
961
962 /* Skip key loading if it is loaded due to sharing */
963 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
964 JUMP_COND_SHRD);
965 if (cdata->key_inline)
966 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
967 cdata->keylen, CLASS_1 |
968 KEY_DEST_CLASS_REG);
969 else
970 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
971 KEY_DEST_CLASS_REG);
972 set_jump_tgt_here(desc, key_jump_cmd);
973
974 /* Class 1 operation */
975 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
976 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
977
978 if (is_qi) {
979 u32 *wait_load_cmd;
980
981 /* REG3 = assoclen */
982 append_seq_load(desc, 4, LDST_CLASS_DECO |
983 LDST_SRCDST_WORD_DECO_MATH3 |
984 (4 << LDST_OFFSET_SHIFT));
985
986 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
987 JUMP_COND_CALM | JUMP_COND_NCP |
988 JUMP_COND_NOP | JUMP_COND_NIP |
989 JUMP_COND_NIFP);
990 set_jump_tgt_here(desc, wait_load_cmd);
991
992 /* Read salt and IV */
993 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
994 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
995 FIFOLD_TYPE_IV);
996 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
997 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
998 }
999
1000 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1001 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1002
1003 /* Read assoc data */
1004 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1005 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1006
1007 /* Skip IV */
1008 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1009
1010 /* Will read cryptlen bytes */
1011 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1012
1013 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1014 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1015
1016 /* Skip assoc data */
1017 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1018
1019 /* Will write cryptlen bytes */
1020 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1021
1022 /* Store payload data */
1023 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1024
1025 /* Read encrypted data */
1026 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1027 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1028
1029 /* Read ICV */
1030 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1031 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1032
1033 #ifdef DEBUG
1034 print_hex_dump(KERN_ERR,
1035 "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1036 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1037 #endif
1038 }
1039 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1040
1041 /**
1042 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1043 * (non-protocol).
1044 * @desc: pointer to buffer used for descriptor construction
1045 * @cdata: pointer to block cipher transform definitions
1046 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1047 * @ivsize: initialization vector size
1048 * @icvsize: integrity check value (ICV) size (truncated or full)
1049 * @is_qi: true when called from caam/qi
1050 */
1051 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1052 unsigned int ivsize, unsigned int icvsize,
1053 const bool is_qi)
1054 {
1055 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1056
1057 init_sh_desc(desc, HDR_SHARE_SERIAL);
1058
1059 /* Skip key loading if it is loaded due to sharing */
1060 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1061 JUMP_COND_SHRD);
1062 if (cdata->key_inline)
1063 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1064 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1065 else
1066 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1067 KEY_DEST_CLASS_REG);
1068 set_jump_tgt_here(desc, key_jump_cmd);
1069
1070 /* Class 1 operation */
1071 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1072 OP_ALG_ENCRYPT);
1073
1074 if (is_qi) {
1075 /* assoclen is not needed, skip it */
1076 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1077
1078 /* Read salt and IV */
1079 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1080 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1081 FIFOLD_TYPE_IV);
1082 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1083 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1084 }
1085
1086 /* assoclen + cryptlen = seqinlen */
1087 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1088
1089 /*
1090 * MOVE_LEN opcode is not available in all SEC HW revisions,
1091 * thus need to do some magic, i.e. self-patch the descriptor
1092 * buffer.
1093 */
1094 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1095 (0x6 << MOVE_LEN_SHIFT));
1096 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1097 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1098
1099 /* Will read assoclen + cryptlen bytes */
1100 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1101
1102 /* Will write assoclen + cryptlen bytes */
1103 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1104
1105 /* Read and write assoclen + cryptlen bytes */
1106 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1107
1108 set_move_tgt_here(desc, read_move_cmd);
1109 set_move_tgt_here(desc, write_move_cmd);
1110 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1111 /* Move payload data to OFIFO */
1112 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1113
1114 /* Write ICV */
1115 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1116 LDST_SRCDST_BYTE_CONTEXT);
1117
1118 #ifdef DEBUG
1119 print_hex_dump(KERN_ERR,
1120 "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1121 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1122 #endif
1123 }
1124 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1125
1126 /**
1127 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1128 * (non-protocol).
1129 * @desc: pointer to buffer used for descriptor construction
1130 * @cdata: pointer to block cipher transform definitions
1131 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1132 * @ivsize: initialization vector size
1133 * @icvsize: integrity check value (ICV) size (truncated or full)
1134 * @is_qi: true when called from caam/qi
1135 */
1136 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1137 unsigned int ivsize, unsigned int icvsize,
1138 const bool is_qi)
1139 {
1140 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1141
1142 init_sh_desc(desc, HDR_SHARE_SERIAL);
1143
1144 /* Skip key loading if it is loaded due to sharing */
1145 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1146 JUMP_COND_SHRD);
1147 if (cdata->key_inline)
1148 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1149 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1150 else
1151 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1152 KEY_DEST_CLASS_REG);
1153 set_jump_tgt_here(desc, key_jump_cmd);
1154
1155 /* Class 1 operation */
1156 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1157 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1158
1159 if (is_qi) {
1160 /* assoclen is not needed, skip it */
1161 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1162
1163 /* Read salt and IV */
1164 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1165 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1166 FIFOLD_TYPE_IV);
1167 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1168 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1169 }
1170
1171 /* assoclen + cryptlen = seqoutlen */
1172 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1173
1174 /*
1175 * MOVE_LEN opcode is not available in all SEC HW revisions,
1176 * thus need to do some magic, i.e. self-patch the descriptor
1177 * buffer.
1178 */
1179 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1180 (0x6 << MOVE_LEN_SHIFT));
1181 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1182 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1183
1184 /* Will read assoclen + cryptlen bytes */
1185 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1186
1187 /* Will write assoclen + cryptlen bytes */
1188 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1189
1190 /* Store payload data */
1191 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1192
1193 /* In-snoop assoclen + cryptlen data */
1194 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1195 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1196
1197 set_move_tgt_here(desc, read_move_cmd);
1198 set_move_tgt_here(desc, write_move_cmd);
1199 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1200 /* Move payload data to OFIFO */
1201 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1202 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1203
1204 /* Read ICV */
1205 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1206 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1207
1208 #ifdef DEBUG
1209 print_hex_dump(KERN_ERR,
1210 "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1211 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1212 #endif
1213 }
1214 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1215
1216 /**
1217 * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1218 * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1219 * descriptor (non-protocol).
1220 * @desc: pointer to buffer used for descriptor construction
1221 * @cdata: pointer to block cipher transform definitions
1222 * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1223 * OP_ALG_AAI_AEAD.
1224 * @adata: pointer to authentication transform definitions
1225 * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1226 * OP_ALG_AAI_AEAD.
1227 * @ivsize: initialization vector size
1228 * @icvsize: integrity check value (ICV) size (truncated or full)
1229 * @encap: true if encapsulation, false if decapsulation
1230 * @is_qi: true when called from caam/qi
1231 */
1232 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1233 struct alginfo *adata, unsigned int ivsize,
1234 unsigned int icvsize, const bool encap,
1235 const bool is_qi)
1236 {
1237 u32 *key_jump_cmd, *wait_cmd;
1238 u32 nfifo;
1239 const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1240
1241 /* Note: Context registers are saved. */
1242 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1243
1244 /* skip key loading if they are loaded due to sharing */
1245 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1246 JUMP_COND_SHRD);
1247
1248 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1249 CLASS_1 | KEY_DEST_CLASS_REG);
1250
1251 /* For IPsec load the salt from keymat in the context register */
1252 if (is_ipsec)
1253 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1254 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1255 4 << LDST_OFFSET_SHIFT);
1256
1257 set_jump_tgt_here(desc, key_jump_cmd);
1258
1259 /* Class 2 and 1 operations: Poly & ChaCha */
1260 if (encap) {
1261 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1262 OP_ALG_ENCRYPT);
1263 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1264 OP_ALG_ENCRYPT);
1265 } else {
1266 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1267 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1268 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1269 OP_ALG_DECRYPT);
1270 }
1271
1272 if (is_qi) {
1273 u32 *wait_load_cmd;
1274 u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1275
1276 /* REG3 = assoclen */
1277 append_seq_load(desc, 4, LDST_CLASS_DECO |
1278 LDST_SRCDST_WORD_DECO_MATH3 |
1279 4 << LDST_OFFSET_SHIFT);
1280
1281 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1282 JUMP_COND_CALM | JUMP_COND_NCP |
1283 JUMP_COND_NOP | JUMP_COND_NIP |
1284 JUMP_COND_NIFP);
1285 set_jump_tgt_here(desc, wait_load_cmd);
1286
1287 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1288 LDST_SRCDST_BYTE_CONTEXT |
1289 ctx1_iv_off << LDST_OFFSET_SHIFT);
1290 }
1291
1292 /*
1293 * MAGIC with NFIFO
1294 * Read associated data from the input and send them to class1 and
1295 * class2 alignment blocks. From class1 send data to output fifo and
1296 * then write it to memory since we don't need to encrypt AD.
1297 */
1298 nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1299 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1300 append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1301 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1302
1303 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1304 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1305 append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1306 FIFOLD_CLASS_CLASS1 | LDST_VLF);
1307 append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1308 MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1309 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1310
1311 /* IPsec - copy IV at the output */
1312 if (is_ipsec)
1313 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1314 0x2 << 25);
1315
1316 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1317 JUMP_COND_NOP | JUMP_TEST_ALL);
1318 set_jump_tgt_here(desc, wait_cmd);
1319
1320 if (encap) {
1321 /* Read and write cryptlen bytes */
1322 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1323 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1324 CAAM_CMD_SZ);
1325 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1326
1327 /* Write ICV */
1328 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1329 LDST_SRCDST_BYTE_CONTEXT);
1330 } else {
1331 /* Read and write cryptlen bytes */
1332 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1333 CAAM_CMD_SZ);
1334 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1335 CAAM_CMD_SZ);
1336 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1337
1338 /* Load ICV for verification */
1339 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1340 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1341 }
1342
1343 print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1344 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1345 1);
1346 }
1347 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1348
1349 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1350 static inline void skcipher_append_src_dst(u32 *desc)
1351 {
1352 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1353 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1354 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1355 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1356 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1357 }
1358
1359 /**
1360 * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1361 * @desc: pointer to buffer used for descriptor construction
1362 * @cdata: pointer to block cipher transform definitions
1363 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1364 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1365 * - OP_ALG_ALGSEL_CHACHA20
1366 * @ivsize: initialization vector size
1367 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1368 * @ctx1_iv_off: IV offset in CONTEXT1 register
1369 */
1370 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1371 unsigned int ivsize, const bool is_rfc3686,
1372 const u32 ctx1_iv_off)
1373 {
1374 u32 *key_jump_cmd;
1375
1376 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1377 /* Skip if already shared */
1378 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1379 JUMP_COND_SHRD);
1380
1381 /* Load class1 key only */
1382 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1383 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1384
1385 /* Load nonce into CONTEXT1 reg */
1386 if (is_rfc3686) {
1387 const u8 *nonce = cdata->key_virt + cdata->keylen;
1388
1389 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1390 LDST_CLASS_IND_CCB |
1391 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1392 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1393 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1394 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1395 }
1396
1397 set_jump_tgt_here(desc, key_jump_cmd);
1398
1399 /* Load IV, if there is one */
1400 if (ivsize)
1401 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1402 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1403 LDST_OFFSET_SHIFT));
1404
1405 /* Load counter into CONTEXT1 reg */
1406 if (is_rfc3686)
1407 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1408 LDST_SRCDST_BYTE_CONTEXT |
1409 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1410 LDST_OFFSET_SHIFT));
1411
1412 /* Load operation */
1413 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1414 OP_ALG_ENCRYPT);
1415
1416 /* Perform operation */
1417 skcipher_append_src_dst(desc);
1418
1419 #ifdef DEBUG
1420 print_hex_dump(KERN_ERR,
1421 "skcipher enc shdesc@" __stringify(__LINE__)": ",
1422 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1423 #endif
1424 }
1425 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1426
1427 /**
1428 * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1429 * @desc: pointer to buffer used for descriptor construction
1430 * @cdata: pointer to block cipher transform definitions
1431 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1432 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1433 * - OP_ALG_ALGSEL_CHACHA20
1434 * @ivsize: initialization vector size
1435 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1436 * @ctx1_iv_off: IV offset in CONTEXT1 register
1437 */
1438 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1439 unsigned int ivsize, const bool is_rfc3686,
1440 const u32 ctx1_iv_off)
1441 {
1442 u32 *key_jump_cmd;
1443
1444 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1445 /* Skip if already shared */
1446 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1447 JUMP_COND_SHRD);
1448
1449 /* Load class1 key only */
1450 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1451 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1452
1453 /* Load nonce into CONTEXT1 reg */
1454 if (is_rfc3686) {
1455 const u8 *nonce = cdata->key_virt + cdata->keylen;
1456
1457 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1458 LDST_CLASS_IND_CCB |
1459 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1460 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1461 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1462 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1463 }
1464
1465 set_jump_tgt_here(desc, key_jump_cmd);
1466
1467 /* Load IV, if there is one */
1468 if (ivsize)
1469 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1470 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1471 LDST_OFFSET_SHIFT));
1472
1473 /* Load counter into CONTEXT1 reg */
1474 if (is_rfc3686)
1475 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1476 LDST_SRCDST_BYTE_CONTEXT |
1477 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1478 LDST_OFFSET_SHIFT));
1479
1480 /* Choose operation */
1481 if (ctx1_iv_off)
1482 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1483 OP_ALG_DECRYPT);
1484 else
1485 append_dec_op1(desc, cdata->algtype);
1486
1487 /* Perform operation */
1488 skcipher_append_src_dst(desc);
1489
1490 #ifdef DEBUG
1491 print_hex_dump(KERN_ERR,
1492 "skcipher dec shdesc@" __stringify(__LINE__)": ",
1493 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1494 #endif
1495 }
1496 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1497
1498 /**
1499 * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1500 * @desc: pointer to buffer used for descriptor construction
1501 * @cdata: pointer to block cipher transform definitions
1502 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1503 */
1504 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1505 {
1506 __be64 sector_size = cpu_to_be64(512);
1507 u32 *key_jump_cmd;
1508
1509 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1510 /* Skip if already shared */
1511 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1512 JUMP_COND_SHRD);
1513
1514 /* Load class1 keys only */
1515 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1516 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1517
1518 /* Load sector size with index 40 bytes (0x28) */
1519 append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1520 LDST_SRCDST_BYTE_CONTEXT |
1521 (0x28 << LDST_OFFSET_SHIFT));
1522
1523 set_jump_tgt_here(desc, key_jump_cmd);
1524
1525 /*
1526 * create sequence for loading the sector index
1527 * Upper 8B of IV - will be used as sector index
1528 * Lower 8B of IV - will be discarded
1529 */
1530 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1531 (0x20 << LDST_OFFSET_SHIFT));
1532 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1533
1534 /* Load operation */
1535 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1536 OP_ALG_ENCRYPT);
1537
1538 /* Perform operation */
1539 skcipher_append_src_dst(desc);
1540
1541 #ifdef DEBUG
1542 print_hex_dump(KERN_ERR,
1543 "xts skcipher enc shdesc@" __stringify(__LINE__) ": ",
1544 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1545 #endif
1546 }
1547 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1548
1549 /**
1550 * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1551 * @desc: pointer to buffer used for descriptor construction
1552 * @cdata: pointer to block cipher transform definitions
1553 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1554 */
1555 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1556 {
1557 __be64 sector_size = cpu_to_be64(512);
1558 u32 *key_jump_cmd;
1559
1560 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1561 /* Skip if already shared */
1562 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1563 JUMP_COND_SHRD);
1564
1565 /* Load class1 key only */
1566 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1567 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1568
1569 /* Load sector size with index 40 bytes (0x28) */
1570 append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1571 LDST_SRCDST_BYTE_CONTEXT |
1572 (0x28 << LDST_OFFSET_SHIFT));
1573
1574 set_jump_tgt_here(desc, key_jump_cmd);
1575
1576 /*
1577 * create sequence for loading the sector index
1578 * Upper 8B of IV - will be used as sector index
1579 * Lower 8B of IV - will be discarded
1580 */
1581 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1582 (0x20 << LDST_OFFSET_SHIFT));
1583 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1584
1585 /* Load operation */
1586 append_dec_op1(desc, cdata->algtype);
1587
1588 /* Perform operation */
1589 skcipher_append_src_dst(desc);
1590
1591 #ifdef DEBUG
1592 print_hex_dump(KERN_ERR,
1593 "xts skcipher dec shdesc@" __stringify(__LINE__) ": ",
1594 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1595 #endif
1596 }
1597 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1598
1599 MODULE_LICENSE("GPL");
1600 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1601 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
Linux with added FPC-III support