?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Revert "unicode: Don't special case ignorable code points"
[linux.git] / drivers / crypto / caam / caamalg_desc.c
blob7571e1ac913b1d35fb49e35f95cc1d1b7b79864f
1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3 * Shared descriptors for aead, skcipher algorithms
4 *
5 * Copyright 2016-2019 NXP
6 */
7
8 #include "compat.h"
9 #include "desc_constr.h"
10 #include "caamalg_desc.h"
11
12 /*
13 * For aead functions, read payload and write payload,
14 * both of which are specified in req->src and req->dst
15 */
16 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17 {
18 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21 }
22
23 /* Set DK bit in class 1 operation if shared */
24 static inline void append_dec_op1(u32 *desc, u32 type)
25 {
26 u32 *jump_cmd, *uncond_jump_cmd;
27
28 /* DK bit is valid only for AES */
29 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 append_operation(desc, type | OP_ALG_AS_INITFINAL |
31 OP_ALG_DECRYPT);
32 return;
33 }
34
35 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38 set_jump_tgt_here(desc, jump_cmd);
39 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
40 OP_ALG_AAI_DK);
41 set_jump_tgt_here(desc, uncond_jump_cmd);
42 }
43
44 /**
45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46 * (non-protocol) with no (null) encryption.
47 * @desc: pointer to buffer used for descriptor construction
48 * @adata: pointer to authentication transform definitions.
49 * A split key is required for SEC Era < 6; the size of the split key
50 * is specified in this case. Valid algorithm values - one of
51 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
52 * with OP_ALG_AAI_HMAC_PRECOMP.
53 * @icvsize: integrity check value (ICV) size (truncated or full)
54 * @era: SEC Era
55 */
56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57 unsigned int icvsize, int era)
58 {
59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61 init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63 /* Skip if already shared */
64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65 JUMP_COND_SHRD);
66 if (era < 6) {
67 if (adata->key_inline)
68 append_key_as_imm(desc, adata->key_virt,
69 adata->keylen_pad, adata->keylen,
70 CLASS_2 | KEY_DEST_MDHA_SPLIT |
71 KEY_ENC);
72 else
73 append_key(desc, adata->key_dma, adata->keylen,
74 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
75 } else {
76 append_proto_dkp(desc, adata);
77 }
78 set_jump_tgt_here(desc, key_jump_cmd);
79
80 /* assoclen + cryptlen = seqinlen */
81 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
82
83 /* Prepare to read and write cryptlen + assoclen bytes */
84 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
85 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
86
87 /*
88 * MOVE_LEN opcode is not available in all SEC HW revisions,
89 * thus need to do some magic, i.e. self-patch the descriptor
90 * buffer.
91 */
92 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
93 MOVE_DEST_MATH3 |
94 (0x6 << MOVE_LEN_SHIFT));
95 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
96 MOVE_DEST_DESCBUF |
97 MOVE_WAITCOMP |
98 (0x8 << MOVE_LEN_SHIFT));
99
100 /* Class 2 operation */
101 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
102 OP_ALG_ENCRYPT);
103
104 /* Read and write cryptlen bytes */
105 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
106
107 set_move_tgt_here(desc, read_move_cmd);
108 set_move_tgt_here(desc, write_move_cmd);
109 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
110 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
111 MOVE_AUX_LS);
112
113 /* Write ICV */
114 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
115 LDST_SRCDST_BYTE_CONTEXT);
116
117 print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
118 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
119 1);
120 }
121 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
122
123 /**
124 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
125 * (non-protocol) with no (null) decryption.
126 * @desc: pointer to buffer used for descriptor construction
127 * @adata: pointer to authentication transform definitions.
128 * A split key is required for SEC Era < 6; the size of the split key
129 * is specified in this case. Valid algorithm values - one of
130 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
131 * with OP_ALG_AAI_HMAC_PRECOMP.
132 * @icvsize: integrity check value (ICV) size (truncated or full)
133 * @era: SEC Era
134 */
135 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
136 unsigned int icvsize, int era)
137 {
138 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
139
140 init_sh_desc(desc, HDR_SHARE_SERIAL);
141
142 /* Skip if already shared */
143 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
144 JUMP_COND_SHRD);
145 if (era < 6) {
146 if (adata->key_inline)
147 append_key_as_imm(desc, adata->key_virt,
148 adata->keylen_pad, adata->keylen,
149 CLASS_2 | KEY_DEST_MDHA_SPLIT |
150 KEY_ENC);
151 else
152 append_key(desc, adata->key_dma, adata->keylen,
153 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
154 } else {
155 append_proto_dkp(desc, adata);
156 }
157 set_jump_tgt_here(desc, key_jump_cmd);
158
159 /* Class 2 operation */
160 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
161 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
162
163 /* assoclen + cryptlen = seqoutlen */
164 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
165
166 /* Prepare to read and write cryptlen + assoclen bytes */
167 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
168 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
169
170 /*
171 * MOVE_LEN opcode is not available in all SEC HW revisions,
172 * thus need to do some magic, i.e. self-patch the descriptor
173 * buffer.
174 */
175 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
176 MOVE_DEST_MATH2 |
177 (0x6 << MOVE_LEN_SHIFT));
178 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
179 MOVE_DEST_DESCBUF |
180 MOVE_WAITCOMP |
181 (0x8 << MOVE_LEN_SHIFT));
182
183 /* Read and write cryptlen bytes */
184 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
185
186 /*
187 * Insert a NOP here, since we need at least 4 instructions between
188 * code patching the descriptor buffer and the location being patched.
189 */
190 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
191 set_jump_tgt_here(desc, jump_cmd);
192
193 set_move_tgt_here(desc, read_move_cmd);
194 set_move_tgt_here(desc, write_move_cmd);
195 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
196 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
197 MOVE_AUX_LS);
198 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
199
200 /* Load ICV */
201 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
202 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
203
204 print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
205 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
206 1);
207 }
208 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
209
210 static void init_sh_desc_key_aead(u32 * const desc,
211 struct alginfo * const cdata,
212 struct alginfo * const adata,
213 const bool is_rfc3686, u32 *nonce, int era)
214 {
215 u32 *key_jump_cmd;
216 unsigned int enckeylen = cdata->keylen;
217
218 /* Note: Context registers are saved. */
219 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
220
221 /* Skip if already shared */
222 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
223 JUMP_COND_SHRD);
224
225 /*
226 * RFC3686 specific:
227 * | key = {AUTH_KEY, ENC_KEY, NONCE}
228 * | enckeylen = encryption key size + nonce size
229 */
230 if (is_rfc3686)
231 enckeylen -= CTR_RFC3686_NONCE_SIZE;
232
233 if (era < 6) {
234 if (adata->key_inline)
235 append_key_as_imm(desc, adata->key_virt,
236 adata->keylen_pad, adata->keylen,
237 CLASS_2 | KEY_DEST_MDHA_SPLIT |
238 KEY_ENC);
239 else
240 append_key(desc, adata->key_dma, adata->keylen,
241 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
242 } else {
243 append_proto_dkp(desc, adata);
244 }
245
246 if (cdata->key_inline)
247 append_key_as_imm(desc, cdata->key_virt, enckeylen,
248 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
249 else
250 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
251 KEY_DEST_CLASS_REG);
252
253 /* Load Counter into CONTEXT1 reg */
254 if (is_rfc3686) {
255 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
256 LDST_CLASS_IND_CCB |
257 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
258 append_move(desc,
259 MOVE_SRC_OUTFIFO |
260 MOVE_DEST_CLASS1CTX |
261 (16 << MOVE_OFFSET_SHIFT) |
262 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
263 }
264
265 set_jump_tgt_here(desc, key_jump_cmd);
266 }
267
268 /**
269 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
270 * (non-protocol).
271 * @desc: pointer to buffer used for descriptor construction
272 * @cdata: pointer to block cipher transform definitions
273 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
274 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
275 * @adata: pointer to authentication transform definitions.
276 * A split key is required for SEC Era < 6; the size of the split key
277 * is specified in this case. Valid algorithm values - one of
278 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
279 * with OP_ALG_AAI_HMAC_PRECOMP.
280 * @ivsize: initialization vector size
281 * @icvsize: integrity check value (ICV) size (truncated or full)
282 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
283 * @nonce: pointer to rfc3686 nonce
284 * @ctx1_iv_off: IV offset in CONTEXT1 register
285 * @is_qi: true when called from caam/qi
286 * @era: SEC Era
287 */
288 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
289 struct alginfo *adata, unsigned int ivsize,
290 unsigned int icvsize, const bool is_rfc3686,
291 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
292 int era)
293 {
294 /* Note: Context registers are saved. */
295 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
296
297 /* Class 2 operation */
298 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
299 OP_ALG_ENCRYPT);
300
301 if (is_qi) {
302 u32 *wait_load_cmd;
303
304 /* REG3 = assoclen */
305 append_seq_load(desc, 4, LDST_CLASS_DECO |
306 LDST_SRCDST_WORD_DECO_MATH3 |
307 (4 << LDST_OFFSET_SHIFT));
308
309 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
310 JUMP_COND_CALM | JUMP_COND_NCP |
311 JUMP_COND_NOP | JUMP_COND_NIP |
312 JUMP_COND_NIFP);
313 set_jump_tgt_here(desc, wait_load_cmd);
314
315 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
316 LDST_SRCDST_BYTE_CONTEXT |
317 (ctx1_iv_off << LDST_OFFSET_SHIFT));
318 }
319
320 /* Read and write assoclen bytes */
321 if (is_qi || era < 3) {
322 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
323 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
324 } else {
325 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
326 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
327 }
328
329 /* Skip assoc data */
330 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
331
332 /* read assoc before reading payload */
333 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
334 FIFOLDST_VLF);
335
336 /* Load Counter into CONTEXT1 reg */
337 if (is_rfc3686)
338 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
339 LDST_SRCDST_BYTE_CONTEXT |
340 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
341 LDST_OFFSET_SHIFT));
342
343 /* Class 1 operation */
344 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
345 OP_ALG_ENCRYPT);
346
347 /* Read and write cryptlen bytes */
348 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
349 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
350 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
351
352 /* Write ICV */
353 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
354 LDST_SRCDST_BYTE_CONTEXT);
355
356 print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
357 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
358 1);
359 }
360 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
361
362 /**
363 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
364 * (non-protocol).
365 * @desc: pointer to buffer used for descriptor construction
366 * @cdata: pointer to block cipher transform definitions
367 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
368 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
369 * @adata: pointer to authentication transform definitions.
370 * A split key is required for SEC Era < 6; the size of the split key
371 * is specified in this case. Valid algorithm values - one of
372 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
373 * with OP_ALG_AAI_HMAC_PRECOMP.
374 * @ivsize: initialization vector size
375 * @icvsize: integrity check value (ICV) size (truncated or full)
376 * @geniv: whether to generate Encrypted Chain IV
377 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
378 * @nonce: pointer to rfc3686 nonce
379 * @ctx1_iv_off: IV offset in CONTEXT1 register
380 * @is_qi: true when called from caam/qi
381 * @era: SEC Era
382 */
383 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
384 struct alginfo *adata, unsigned int ivsize,
385 unsigned int icvsize, const bool geniv,
386 const bool is_rfc3686, u32 *nonce,
387 const u32 ctx1_iv_off, const bool is_qi, int era)
388 {
389 /* Note: Context registers are saved. */
390 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
391
392 /* Class 2 operation */
393 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
394 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
395
396 if (is_qi) {
397 u32 *wait_load_cmd;
398
399 /* REG3 = assoclen */
400 append_seq_load(desc, 4, LDST_CLASS_DECO |
401 LDST_SRCDST_WORD_DECO_MATH3 |
402 (4 << LDST_OFFSET_SHIFT));
403
404 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
405 JUMP_COND_CALM | JUMP_COND_NCP |
406 JUMP_COND_NOP | JUMP_COND_NIP |
407 JUMP_COND_NIFP);
408 set_jump_tgt_here(desc, wait_load_cmd);
409
410 if (!geniv)
411 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
412 LDST_SRCDST_BYTE_CONTEXT |
413 (ctx1_iv_off << LDST_OFFSET_SHIFT));
414 }
415
416 /* Read and write assoclen bytes */
417 if (is_qi || era < 3) {
418 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
419 if (geniv)
420 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
421 ivsize);
422 else
423 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
424 CAAM_CMD_SZ);
425 } else {
426 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
427 if (geniv)
428 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
429 ivsize);
430 else
431 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
432 CAAM_CMD_SZ);
433 }
434
435 /* Skip assoc data */
436 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
437
438 /* read assoc before reading payload */
439 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
440 KEY_VLF);
441
442 if (geniv) {
443 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
444 LDST_SRCDST_BYTE_CONTEXT |
445 (ctx1_iv_off << LDST_OFFSET_SHIFT));
446 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
447 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
448 }
449
450 /* Load Counter into CONTEXT1 reg */
451 if (is_rfc3686)
452 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
453 LDST_SRCDST_BYTE_CONTEXT |
454 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
455 LDST_OFFSET_SHIFT));
456
457 /* Choose operation */
458 if (ctx1_iv_off)
459 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
460 OP_ALG_DECRYPT);
461 else
462 append_dec_op1(desc, cdata->algtype);
463
464 /* Read and write cryptlen bytes */
465 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
466 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
467 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
468
469 /* Load ICV */
470 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
471 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
472
473 print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
474 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
475 1);
476 }
477 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
478
479 /**
480 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
481 * (non-protocol) with HW-generated initialization
482 * vector.
483 * @desc: pointer to buffer used for descriptor construction
484 * @cdata: pointer to block cipher transform definitions
485 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
486 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
487 * @adata: pointer to authentication transform definitions.
488 * A split key is required for SEC Era < 6; the size of the split key
489 * is specified in this case. Valid algorithm values - one of
490 * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed
491 * with OP_ALG_AAI_HMAC_PRECOMP.
492 * @ivsize: initialization vector size
493 * @icvsize: integrity check value (ICV) size (truncated or full)
494 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
495 * @nonce: pointer to rfc3686 nonce
496 * @ctx1_iv_off: IV offset in CONTEXT1 register
497 * @is_qi: true when called from caam/qi
498 * @era: SEC Era
499 */
500 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
501 struct alginfo *adata, unsigned int ivsize,
502 unsigned int icvsize, const bool is_rfc3686,
503 u32 *nonce, const u32 ctx1_iv_off,
504 const bool is_qi, int era)
505 {
506 u32 geniv, moveiv;
507 u32 *wait_cmd;
508
509 /* Note: Context registers are saved. */
510 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
511
512 if (is_qi) {
513 u32 *wait_load_cmd;
514
515 /* REG3 = assoclen */
516 append_seq_load(desc, 4, LDST_CLASS_DECO |
517 LDST_SRCDST_WORD_DECO_MATH3 |
518 (4 << LDST_OFFSET_SHIFT));
519
520 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
521 JUMP_COND_CALM | JUMP_COND_NCP |
522 JUMP_COND_NOP | JUMP_COND_NIP |
523 JUMP_COND_NIFP);
524 set_jump_tgt_here(desc, wait_load_cmd);
525 }
526
527 if (is_rfc3686) {
528 if (is_qi)
529 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
530 LDST_SRCDST_BYTE_CONTEXT |
531 (ctx1_iv_off << LDST_OFFSET_SHIFT));
532
533 goto copy_iv;
534 }
535
536 /* Generate IV */
537 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
538 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
539 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
540 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
541 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
542 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
543 append_move(desc, MOVE_WAITCOMP |
544 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
545 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
546 (ivsize << MOVE_LEN_SHIFT));
547 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
548
549 copy_iv:
550 /* Copy IV to class 1 context */
551 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
552 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
553 (ivsize << MOVE_LEN_SHIFT));
554
555 /* Return to encryption */
556 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
557 OP_ALG_ENCRYPT);
558
559 /* Read and write assoclen bytes */
560 if (is_qi || era < 3) {
561 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
562 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
563 } else {
564 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
565 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
566 }
567
568 /* Skip assoc data */
569 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
570
571 /* read assoc before reading payload */
572 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
573 KEY_VLF);
574
575 /* Copy iv from outfifo to class 2 fifo */
576 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
577 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
578 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
579 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
580 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
581 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
582
583 /* Load Counter into CONTEXT1 reg */
584 if (is_rfc3686)
585 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
586 LDST_SRCDST_BYTE_CONTEXT |
587 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
588 LDST_OFFSET_SHIFT));
589
590 /* Class 1 operation */
591 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
592 OP_ALG_ENCRYPT);
593
594 /* Will write ivsize + cryptlen */
595 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
596
597 /* Not need to reload iv */
598 append_seq_fifo_load(desc, ivsize,
599 FIFOLD_CLASS_SKIP);
600
601 /* Will read cryptlen */
602 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
603
604 /*
605 * Wait for IV transfer (ofifo -> class2) to finish before starting
606 * ciphertext transfer (ofifo -> external memory).
607 */
608 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
609 set_jump_tgt_here(desc, wait_cmd);
610
611 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
612 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
613 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
614
615 /* Write ICV */
616 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
617 LDST_SRCDST_BYTE_CONTEXT);
618
619 print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
620 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
621 1);
622 }
623 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
624
625 /**
626 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
627 * @desc: pointer to buffer used for descriptor construction
628 * @cdata: pointer to block cipher transform definitions
629 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
630 * @ivsize: initialization vector size
631 * @icvsize: integrity check value (ICV) size (truncated or full)
632 * @is_qi: true when called from caam/qi
633 */
634 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
635 unsigned int ivsize, unsigned int icvsize,
636 const bool is_qi)
637 {
638 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
639 *zero_assoc_jump_cmd2;
640
641 init_sh_desc(desc, HDR_SHARE_SERIAL);
642
643 /* skip key loading if they are loaded due to sharing */
644 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
645 JUMP_COND_SHRD);
646 if (cdata->key_inline)
647 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
648 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
649 else
650 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
651 KEY_DEST_CLASS_REG);
652 set_jump_tgt_here(desc, key_jump_cmd);
653
654 /* class 1 operation */
655 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
656 OP_ALG_ENCRYPT);
657
658 if (is_qi) {
659 u32 *wait_load_cmd;
660
661 /* REG3 = assoclen */
662 append_seq_load(desc, 4, LDST_CLASS_DECO |
663 LDST_SRCDST_WORD_DECO_MATH3 |
664 (4 << LDST_OFFSET_SHIFT));
665
666 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
667 JUMP_COND_CALM | JUMP_COND_NCP |
668 JUMP_COND_NOP | JUMP_COND_NIP |
669 JUMP_COND_NIFP);
670 set_jump_tgt_here(desc, wait_load_cmd);
671
672 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
673 ivsize);
674 } else {
675 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
676 CAAM_CMD_SZ);
677 }
678
679 /* if assoclen + cryptlen is ZERO, skip to ICV write */
680 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
681 JUMP_COND_MATH_Z);
682
683 if (is_qi)
684 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
685 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
686
687 /* if assoclen is ZERO, skip reading the assoc data */
688 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
689 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
690 JUMP_COND_MATH_Z);
691
692 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
693
694 /* skip assoc data */
695 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
696
697 /* cryptlen = seqinlen - assoclen */
698 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
699
700 /* if cryptlen is ZERO jump to zero-payload commands */
701 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
702 JUMP_COND_MATH_Z);
703
704 /* read assoc data */
705 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
706 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
707 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
708
709 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
710
711 /* write encrypted data */
712 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
713
714 /* read payload data */
715 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
716 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
717
718 /* jump to ICV writing */
719 if (is_qi)
720 append_jump(desc, JUMP_TEST_ALL | 4);
721 else
722 append_jump(desc, JUMP_TEST_ALL | 2);
723
724 /* zero-payload commands */
725 set_jump_tgt_here(desc, zero_payload_jump_cmd);
726
727 /* read assoc data */
728 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
729 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
730 if (is_qi)
731 /* jump to ICV writing */
732 append_jump(desc, JUMP_TEST_ALL | 2);
733
734 /* There is no input data */
735 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
736
737 if (is_qi)
738 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
739 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
740 FIFOLD_TYPE_LAST1);
741
742 /* write ICV */
743 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
744 LDST_SRCDST_BYTE_CONTEXT);
745
746 print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
747 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
748 1);
749 }
750 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
751
752 /**
753 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
754 * @desc: pointer to buffer used for descriptor construction
755 * @cdata: pointer to block cipher transform definitions
756 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
757 * @ivsize: initialization vector size
758 * @icvsize: integrity check value (ICV) size (truncated or full)
759 * @is_qi: true when called from caam/qi
760 */
761 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762 unsigned int ivsize, unsigned int icvsize,
763 const bool is_qi)
764 {
765 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
766
767 init_sh_desc(desc, HDR_SHARE_SERIAL);
768
769 /* skip key loading if they are loaded due to sharing */
770 key_jump_cmd = append_jump(desc, JUMP_JSL |
771 JUMP_TEST_ALL | JUMP_COND_SHRD);
772 if (cdata->key_inline)
773 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
774 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
775 else
776 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
777 KEY_DEST_CLASS_REG);
778 set_jump_tgt_here(desc, key_jump_cmd);
779
780 /* class 1 operation */
781 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
782 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
783
784 if (is_qi) {
785 u32 *wait_load_cmd;
786
787 /* REG3 = assoclen */
788 append_seq_load(desc, 4, LDST_CLASS_DECO |
789 LDST_SRCDST_WORD_DECO_MATH3 |
790 (4 << LDST_OFFSET_SHIFT));
791
792 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793 JUMP_COND_CALM | JUMP_COND_NCP |
794 JUMP_COND_NOP | JUMP_COND_NIP |
795 JUMP_COND_NIFP);
796 set_jump_tgt_here(desc, wait_load_cmd);
797
798 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
799 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
800 }
801
802 /* if assoclen is ZERO, skip reading the assoc data */
803 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
805 JUMP_COND_MATH_Z);
806
807 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
808
809 /* skip assoc data */
810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
811
812 /* read assoc data */
813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
815
816 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
817
818 /* cryptlen = seqoutlen - assoclen */
819 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
820
821 /* jump to zero-payload command if cryptlen is zero */
822 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
823 JUMP_COND_MATH_Z);
824
825 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
826
827 /* store encrypted data */
828 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
829
830 /* read payload data */
831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
833
834 /* zero-payload command */
835 set_jump_tgt_here(desc, zero_payload_jump_cmd);
836
837 /* read ICV */
838 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
839 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
840
841 print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
842 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
843 1);
844 }
845 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
846
847 /**
848 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
849 * (non-protocol).
850 * @desc: pointer to buffer used for descriptor construction
851 * @cdata: pointer to block cipher transform definitions
852 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
853 * @ivsize: initialization vector size
854 * @icvsize: integrity check value (ICV) size (truncated or full)
855 * @is_qi: true when called from caam/qi
856 *
857 * Input sequence: AAD | PTXT
858 * Output sequence: AAD | CTXT | ICV
859 * AAD length (assoclen), which includes the IV length, is available in Math3.
860 */
861 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
862 unsigned int ivsize, unsigned int icvsize,
863 const bool is_qi)
864 {
865 u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
866 init_sh_desc(desc, HDR_SHARE_SERIAL);
867
868 /* Skip key loading if it is loaded due to sharing */
869 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
870 JUMP_COND_SHRD);
871 if (cdata->key_inline)
872 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
873 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
874 else
875 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
876 KEY_DEST_CLASS_REG);
877 set_jump_tgt_here(desc, key_jump_cmd);
878
879 /* Class 1 operation */
880 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
881 OP_ALG_ENCRYPT);
882
883 if (is_qi) {
884 u32 *wait_load_cmd;
885
886 /* REG3 = assoclen */
887 append_seq_load(desc, 4, LDST_CLASS_DECO |
888 LDST_SRCDST_WORD_DECO_MATH3 |
889 (4 << LDST_OFFSET_SHIFT));
890
891 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
892 JUMP_COND_CALM | JUMP_COND_NCP |
893 JUMP_COND_NOP | JUMP_COND_NIP |
894 JUMP_COND_NIFP);
895 set_jump_tgt_here(desc, wait_load_cmd);
896
897 /* Read salt and IV */
898 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
899 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
900 FIFOLD_TYPE_IV);
901 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
902 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
903 }
904
905 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
906 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
907
908 /* Skip AAD */
909 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
910
911 /* Read cryptlen and set this value into VARSEQOUTLEN */
912 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
913
914 /* If cryptlen is ZERO jump to AAD command */
915 zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
916 JUMP_COND_MATH_Z);
917
918 /* Read AAD data */
919 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
920 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
921
922 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
923 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
924
925 /* Skip IV */
926 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
927 append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
928
929 /* Write encrypted data */
930 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
931
932 /* Read payload data */
933 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
934 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
935
936 /* Jump instructions to avoid double reading of AAD */
937 skip_instructions = append_jump(desc, JUMP_TEST_ALL);
938
939 /* There is no input data, cryptlen = 0 */
940 set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
941
942 /* Read AAD */
943 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
944 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
945
946 set_jump_tgt_here(desc, skip_instructions);
947
948 /* Write ICV */
949 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
950 LDST_SRCDST_BYTE_CONTEXT);
951
952 print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
953 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
954 1);
955 }
956 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
957
958 /**
959 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
960 * (non-protocol).
961 * @desc: pointer to buffer used for descriptor construction
962 * @cdata: pointer to block cipher transform definitions
963 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
964 * @ivsize: initialization vector size
965 * @icvsize: integrity check value (ICV) size (truncated or full)
966 * @is_qi: true when called from caam/qi
967 */
968 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
969 unsigned int ivsize, unsigned int icvsize,
970 const bool is_qi)
971 {
972 u32 *key_jump_cmd;
973
974 init_sh_desc(desc, HDR_SHARE_SERIAL);
975
976 /* Skip key loading if it is loaded due to sharing */
977 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
978 JUMP_COND_SHRD);
979 if (cdata->key_inline)
980 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
981 cdata->keylen, CLASS_1 |
982 KEY_DEST_CLASS_REG);
983 else
984 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
985 KEY_DEST_CLASS_REG);
986 set_jump_tgt_here(desc, key_jump_cmd);
987
988 /* Class 1 operation */
989 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
990 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
991
992 if (is_qi) {
993 u32 *wait_load_cmd;
994
995 /* REG3 = assoclen */
996 append_seq_load(desc, 4, LDST_CLASS_DECO |
997 LDST_SRCDST_WORD_DECO_MATH3 |
998 (4 << LDST_OFFSET_SHIFT));
999
1000 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1001 JUMP_COND_CALM | JUMP_COND_NCP |
1002 JUMP_COND_NOP | JUMP_COND_NIP |
1003 JUMP_COND_NIFP);
1004 set_jump_tgt_here(desc, wait_load_cmd);
1005
1006 /* Read salt and IV */
1007 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1008 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1009 FIFOLD_TYPE_IV);
1010 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1011 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1012 }
1013
1014 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1015 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1016
1017 /* Read assoc data */
1018 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1019 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1020
1021 /* Skip IV */
1022 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1023
1024 /* Will read cryptlen bytes */
1025 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1026
1027 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1028 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1029
1030 /* Skip assoc data */
1031 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1032
1033 /* Will write cryptlen bytes */
1034 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1035
1036 /* Store payload data */
1037 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1038
1039 /* Read encrypted data */
1040 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1041 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1042
1043 /* Read ICV */
1044 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1045 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1046
1047 print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1048 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1049 1);
1050 }
1051 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1052
1053 /**
1054 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
1055 * (non-protocol).
1056 * @desc: pointer to buffer used for descriptor construction
1057 * @cdata: pointer to block cipher transform definitions
1058 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1059 * @ivsize: initialization vector size
1060 * @icvsize: integrity check value (ICV) size (truncated or full)
1061 * @is_qi: true when called from caam/qi
1062 */
1063 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1064 unsigned int ivsize, unsigned int icvsize,
1065 const bool is_qi)
1066 {
1067 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1068
1069 init_sh_desc(desc, HDR_SHARE_SERIAL);
1070
1071 /* Skip key loading if it is loaded due to sharing */
1072 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1073 JUMP_COND_SHRD);
1074 if (cdata->key_inline)
1075 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1076 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 else
1078 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1079 KEY_DEST_CLASS_REG);
1080 set_jump_tgt_here(desc, key_jump_cmd);
1081
1082 /* Class 1 operation */
1083 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1084 OP_ALG_ENCRYPT);
1085
1086 if (is_qi) {
1087 /* assoclen is not needed, skip it */
1088 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1089
1090 /* Read salt and IV */
1091 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1092 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1093 FIFOLD_TYPE_IV);
1094 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1095 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1096 }
1097
1098 /* assoclen + cryptlen = seqinlen */
1099 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1100
1101 /*
1102 * MOVE_LEN opcode is not available in all SEC HW revisions,
1103 * thus need to do some magic, i.e. self-patch the descriptor
1104 * buffer.
1105 */
1106 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1107 (0x6 << MOVE_LEN_SHIFT));
1108 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1109 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1110
1111 /* Will read assoclen + cryptlen bytes */
1112 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1113
1114 /* Will write assoclen + cryptlen bytes */
1115 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1116
1117 /* Read and write assoclen + cryptlen bytes */
1118 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1119
1120 set_move_tgt_here(desc, read_move_cmd);
1121 set_move_tgt_here(desc, write_move_cmd);
1122 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1123 /* Move payload data to OFIFO */
1124 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1125
1126 /* Write ICV */
1127 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1128 LDST_SRCDST_BYTE_CONTEXT);
1129
1130 print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1131 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132 1);
1133 }
1134 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1135
1136 /**
1137 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
1138 * (non-protocol).
1139 * @desc: pointer to buffer used for descriptor construction
1140 * @cdata: pointer to block cipher transform definitions
1141 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
1142 * @ivsize: initialization vector size
1143 * @icvsize: integrity check value (ICV) size (truncated or full)
1144 * @is_qi: true when called from caam/qi
1145 */
1146 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1147 unsigned int ivsize, unsigned int icvsize,
1148 const bool is_qi)
1149 {
1150 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1151
1152 init_sh_desc(desc, HDR_SHARE_SERIAL);
1153
1154 /* Skip key loading if it is loaded due to sharing */
1155 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1156 JUMP_COND_SHRD);
1157 if (cdata->key_inline)
1158 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1159 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1160 else
1161 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1162 KEY_DEST_CLASS_REG);
1163 set_jump_tgt_here(desc, key_jump_cmd);
1164
1165 /* Class 1 operation */
1166 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1167 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1168
1169 if (is_qi) {
1170 /* assoclen is not needed, skip it */
1171 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1172
1173 /* Read salt and IV */
1174 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1175 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1176 FIFOLD_TYPE_IV);
1177 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1178 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1179 }
1180
1181 /* assoclen + cryptlen = seqoutlen */
1182 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1183
1184 /*
1185 * MOVE_LEN opcode is not available in all SEC HW revisions,
1186 * thus need to do some magic, i.e. self-patch the descriptor
1187 * buffer.
1188 */
1189 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1190 (0x6 << MOVE_LEN_SHIFT));
1191 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1192 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1193
1194 /* Will read assoclen + cryptlen bytes */
1195 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1196
1197 /* Will write assoclen + cryptlen bytes */
1198 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1199
1200 /* Store payload data */
1201 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1202
1203 /* In-snoop assoclen + cryptlen data */
1204 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1205 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1206
1207 set_move_tgt_here(desc, read_move_cmd);
1208 set_move_tgt_here(desc, write_move_cmd);
1209 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1210 /* Move payload data to OFIFO */
1211 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1212 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1213
1214 /* Read ICV */
1215 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1216 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1217
1218 print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1220 1);
1221 }
1222 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223
1224 /**
1225 * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and
1226 * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared
1227 * descriptor (non-protocol).
1228 * @desc: pointer to buffer used for descriptor construction
1229 * @cdata: pointer to block cipher transform definitions
1230 * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with
1231 * OP_ALG_AAI_AEAD.
1232 * @adata: pointer to authentication transform definitions
1233 * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with
1234 * OP_ALG_AAI_AEAD.
1235 * @ivsize: initialization vector size
1236 * @icvsize: integrity check value (ICV) size (truncated or full)
1237 * @encap: true if encapsulation, false if decapsulation
1238 * @is_qi: true when called from caam/qi
1239 */
1240 void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1241 struct alginfo *adata, unsigned int ivsize,
1242 unsigned int icvsize, const bool encap,
1243 const bool is_qi)
1244 {
1245 u32 *key_jump_cmd, *wait_cmd;
1246 u32 nfifo;
1247 const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1248
1249 /* Note: Context registers are saved. */
1250 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1251
1252 /* skip key loading if they are loaded due to sharing */
1253 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1254 JUMP_COND_SHRD);
1255
1256 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1257 CLASS_1 | KEY_DEST_CLASS_REG);
1258
1259 /* For IPsec load the salt from keymat in the context register */
1260 if (is_ipsec)
1261 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1262 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1263 4 << LDST_OFFSET_SHIFT);
1264
1265 set_jump_tgt_here(desc, key_jump_cmd);
1266
1267 /* Class 2 and 1 operations: Poly & ChaCha */
1268 if (encap) {
1269 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1270 OP_ALG_ENCRYPT);
1271 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1272 OP_ALG_ENCRYPT);
1273 } else {
1274 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1275 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1276 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1277 OP_ALG_DECRYPT);
1278 }
1279
1280 if (is_qi) {
1281 u32 *wait_load_cmd;
1282 u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1283
1284 /* REG3 = assoclen */
1285 append_seq_load(desc, 4, LDST_CLASS_DECO |
1286 LDST_SRCDST_WORD_DECO_MATH3 |
1287 4 << LDST_OFFSET_SHIFT);
1288
1289 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1290 JUMP_COND_CALM | JUMP_COND_NCP |
1291 JUMP_COND_NOP | JUMP_COND_NIP |
1292 JUMP_COND_NIFP);
1293 set_jump_tgt_here(desc, wait_load_cmd);
1294
1295 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1296 LDST_SRCDST_BYTE_CONTEXT |
1297 ctx1_iv_off << LDST_OFFSET_SHIFT);
1298 }
1299
1300 /*
1301 * MAGIC with NFIFO
1302 * Read associated data from the input and send them to class1 and
1303 * class2 alignment blocks. From class1 send data to output fifo and
1304 * then write it to memory since we don't need to encrypt AD.
1305 */
1306 nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1307 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1308 append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1309 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1310
1311 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1312 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1313 append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1314 FIFOLD_CLASS_CLASS1 | LDST_VLF);
1315 append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1316 MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1317 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1318
1319 /* IPsec - copy IV at the output */
1320 if (is_ipsec)
1321 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1322 0x2 << 25);
1323
1324 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1325 JUMP_COND_NOP | JUMP_TEST_ALL);
1326 set_jump_tgt_here(desc, wait_cmd);
1327
1328 if (encap) {
1329 /* Read and write cryptlen bytes */
1330 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1331 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1332 CAAM_CMD_SZ);
1333 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1334
1335 /* Write ICV */
1336 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1337 LDST_SRCDST_BYTE_CONTEXT);
1338 } else {
1339 /* Read and write cryptlen bytes */
1340 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1341 CAAM_CMD_SZ);
1342 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1343 CAAM_CMD_SZ);
1344 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1345
1346 /* Load ICV for verification */
1347 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1348 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1349 }
1350
1351 print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1352 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1353 1);
1354 }
1355 EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1356
1357 /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */
1358 static inline void skcipher_append_src_dst(u32 *desc)
1359 {
1360 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1361 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1362 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1363 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1364 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1365 }
1366
1367 /**
1368 * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor
1369 * @desc: pointer to buffer used for descriptor construction
1370 * @cdata: pointer to block cipher transform definitions
1371 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1372 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1373 * - OP_ALG_ALGSEL_CHACHA20
1374 * @ivsize: initialization vector size
1375 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1376 * @ctx1_iv_off: IV offset in CONTEXT1 register
1377 */
1378 void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1379 unsigned int ivsize, const bool is_rfc3686,
1380 const u32 ctx1_iv_off)
1381 {
1382 u32 *key_jump_cmd;
1383 u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1384 bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1385 OP_ALG_ALGSEL_CHACHA20);
1386
1387 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388 /* Skip if already shared */
1389 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390 JUMP_COND_SHRD);
1391
1392 /* Load class1 key only */
1393 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395
1396 /* Load nonce into CONTEXT1 reg */
1397 if (is_rfc3686) {
1398 const u8 *nonce = cdata->key_virt + cdata->keylen;
1399
1400 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401 LDST_CLASS_IND_CCB |
1402 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406 }
1407
1408 set_jump_tgt_here(desc, key_jump_cmd);
1409
1410 /* Load IV, if there is one */
1411 if (ivsize)
1412 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1413 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1414 LDST_OFFSET_SHIFT));
1415
1416 /* Load counter into CONTEXT1 reg */
1417 if (is_rfc3686)
1418 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1419 LDST_SRCDST_BYTE_CONTEXT |
1420 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1421 LDST_OFFSET_SHIFT));
1422
1423 /* Load operation */
1424 if (is_chacha20)
1425 options |= OP_ALG_AS_FINALIZE;
1426 append_operation(desc, options);
1427
1428 /* Perform operation */
1429 skcipher_append_src_dst(desc);
1430
1431 /* Store IV */
1432 if (!is_chacha20 && ivsize)
1433 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1434 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1435 LDST_OFFSET_SHIFT));
1436
1437 print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1438 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1439 1);
1440 }
1441 EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1442
1443 /**
1444 * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor
1445 * @desc: pointer to buffer used for descriptor construction
1446 * @cdata: pointer to block cipher transform definitions
1447 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1448 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128
1449 * - OP_ALG_ALGSEL_CHACHA20
1450 * @ivsize: initialization vector size
1451 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1452 * @ctx1_iv_off: IV offset in CONTEXT1 register
1453 */
1454 void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1455 unsigned int ivsize, const bool is_rfc3686,
1456 const u32 ctx1_iv_off)
1457 {
1458 u32 *key_jump_cmd;
1459 bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1460 OP_ALG_ALGSEL_CHACHA20);
1461
1462 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1463 /* Skip if already shared */
1464 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1465 JUMP_COND_SHRD);
1466
1467 /* Load class1 key only */
1468 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1469 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1470
1471 /* Load nonce into CONTEXT1 reg */
1472 if (is_rfc3686) {
1473 const u8 *nonce = cdata->key_virt + cdata->keylen;
1474
1475 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1476 LDST_CLASS_IND_CCB |
1477 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1478 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1479 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1480 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1481 }
1482
1483 set_jump_tgt_here(desc, key_jump_cmd);
1484
1485 /* Load IV, if there is one */
1486 if (ivsize)
1487 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1488 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1489 LDST_OFFSET_SHIFT));
1490
1491 /* Load counter into CONTEXT1 reg */
1492 if (is_rfc3686)
1493 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1494 LDST_SRCDST_BYTE_CONTEXT |
1495 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1496 LDST_OFFSET_SHIFT));
1497
1498 /* Choose operation */
1499 if (ctx1_iv_off)
1500 append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1501 OP_ALG_DECRYPT);
1502 else
1503 append_dec_op1(desc, cdata->algtype);
1504
1505 /* Perform operation */
1506 skcipher_append_src_dst(desc);
1507
1508 /* Store IV */
1509 if (!is_chacha20 && ivsize)
1510 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1511 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1512 LDST_OFFSET_SHIFT));
1513
1514 print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1515 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1516 1);
1517 }
1518 EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1519
1520 /**
1521 * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor
1522 * @desc: pointer to buffer used for descriptor construction
1523 * @cdata: pointer to block cipher transform definitions
1524 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1525 */
1526 void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1527 {
1528 /*
1529 * Set sector size to a big value, practically disabling
1530 * sector size segmentation in xts implementation. We cannot
1531 * take full advantage of this HW feature with existing
1532 * crypto API / dm-crypt SW architecture.
1533 */
1534 __be64 sector_size = cpu_to_be64(BIT(15));
1535 u32 *key_jump_cmd;
1536
1537 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1538 /* Skip if already shared */
1539 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1540 JUMP_COND_SHRD);
1541
1542 /* Load class1 keys only */
1543 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1544 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1545
1546 /* Load sector size with index 40 bytes (0x28) */
1547 append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1548 LDST_SRCDST_BYTE_CONTEXT |
1549 (0x28 << LDST_OFFSET_SHIFT));
1550
1551 set_jump_tgt_here(desc, key_jump_cmd);
1552
1553 /*
1554 * create sequence for loading the sector index / 16B tweak value
1555 * Lower 8B of IV - sector index / tweak lower half
1556 * Upper 8B of IV - upper half of 16B tweak
1557 */
1558 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1559 (0x20 << LDST_OFFSET_SHIFT));
1560 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1561 (0x30 << LDST_OFFSET_SHIFT));
1562
1563 /* Load operation */
1564 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1565 OP_ALG_ENCRYPT);
1566
1567 /* Perform operation */
1568 skcipher_append_src_dst(desc);
1569
1570 /* Store lower 8B and upper 8B of IV */
1571 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1572 (0x20 << LDST_OFFSET_SHIFT));
1573 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1574 (0x30 << LDST_OFFSET_SHIFT));
1575
1576 print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1577 ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1578 desc, desc_bytes(desc), 1);
1579 }
1580 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1581
1582 /**
1583 * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor
1584 * @desc: pointer to buffer used for descriptor construction
1585 * @cdata: pointer to block cipher transform definitions
1586 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1587 */
1588 void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1589 {
1590 /*
1591 * Set sector size to a big value, practically disabling
1592 * sector size segmentation in xts implementation. We cannot
1593 * take full advantage of this HW feature with existing
1594 * crypto API / dm-crypt SW architecture.
1595 */
1596 __be64 sector_size = cpu_to_be64(BIT(15));
1597 u32 *key_jump_cmd;
1598
1599 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1600 /* Skip if already shared */
1601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1602 JUMP_COND_SHRD);
1603
1604 /* Load class1 key only */
1605 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1606 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1607
1608 /* Load sector size with index 40 bytes (0x28) */
1609 append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1610 LDST_SRCDST_BYTE_CONTEXT |
1611 (0x28 << LDST_OFFSET_SHIFT));
1612
1613 set_jump_tgt_here(desc, key_jump_cmd);
1614
1615 /*
1616 * create sequence for loading the sector index / 16B tweak value
1617 * Lower 8B of IV - sector index / tweak lower half
1618 * Upper 8B of IV - upper half of 16B tweak
1619 */
1620 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1621 (0x20 << LDST_OFFSET_SHIFT));
1622 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1623 (0x30 << LDST_OFFSET_SHIFT));
1624 /* Load operation */
1625 append_dec_op1(desc, cdata->algtype);
1626
1627 /* Perform operation */
1628 skcipher_append_src_dst(desc);
1629
1630 /* Store lower 8B and upper 8B of IV */
1631 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1632 (0x20 << LDST_OFFSET_SHIFT));
1633 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1634 (0x30 << LDST_OFFSET_SHIFT));
1635
1636 print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1637 ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1638 desc_bytes(desc), 1);
1639 }
1640 EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1641
1642 MODULE_LICENSE("GPL");
1643 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1644 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
Linux Kernel