?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.19.133
[linux/fpc-iii.git] / drivers / crypto / ccree / cc_aead.c
blob57aac15a335f59a46a2b5cceaeb866058a28c8e9
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2012-2018 ARM Limited or its affiliates. */
3
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <crypto/algapi.h>
7 #include <crypto/internal/aead.h>
8 #include <crypto/authenc.h>
9 #include <crypto/des.h>
10 #include <linux/rtnetlink.h>
11 #include "cc_driver.h"
12 #include "cc_buffer_mgr.h"
13 #include "cc_aead.h"
14 #include "cc_request_mgr.h"
15 #include "cc_hash.h"
16 #include "cc_sram_mgr.h"
17
18 #define template_aead template_u.aead
19
20 #define MAX_AEAD_SETKEY_SEQ 12
21 #define MAX_AEAD_PROCESS_SEQ 23
22
23 #define MAX_HMAC_DIGEST_SIZE (SHA256_DIGEST_SIZE)
24 #define MAX_HMAC_BLOCK_SIZE (SHA256_BLOCK_SIZE)
25
26 #define AES_CCM_RFC4309_NONCE_SIZE 3
27 #define MAX_NONCE_SIZE CTR_RFC3686_NONCE_SIZE
28
29 /* Value of each ICV_CMP byte (of 8) in case of success */
30 #define ICV_VERIF_OK 0x01
31
32 struct cc_aead_handle {
33 cc_sram_addr_t sram_workspace_addr;
34 struct list_head aead_list;
35 };
36
37 struct cc_hmac_s {
38 u8 *padded_authkey;
39 u8 *ipad_opad; /* IPAD, OPAD*/
40 dma_addr_t padded_authkey_dma_addr;
41 dma_addr_t ipad_opad_dma_addr;
42 };
43
44 struct cc_xcbc_s {
45 u8 *xcbc_keys; /* K1,K2,K3 */
46 dma_addr_t xcbc_keys_dma_addr;
47 };
48
49 struct cc_aead_ctx {
50 struct cc_drvdata *drvdata;
51 u8 ctr_nonce[MAX_NONCE_SIZE]; /* used for ctr3686 iv and aes ccm */
52 u8 *enckey;
53 dma_addr_t enckey_dma_addr;
54 union {
55 struct cc_hmac_s hmac;
56 struct cc_xcbc_s xcbc;
57 } auth_state;
58 unsigned int enc_keylen;
59 unsigned int auth_keylen;
60 unsigned int authsize; /* Actual (reduced?) size of the MAC/ICv */
61 enum drv_cipher_mode cipher_mode;
62 enum cc_flow_mode flow_mode;
63 enum drv_hash_mode auth_mode;
64 };
65
66 static inline bool valid_assoclen(struct aead_request *req)
67 {
68 return ((req->assoclen == 16) || (req->assoclen == 20));
69 }
70
71 static void cc_aead_exit(struct crypto_aead *tfm)
72 {
73 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
74 struct device *dev = drvdata_to_dev(ctx->drvdata);
75
76 dev_dbg(dev, "Clearing context @%p for %s\n", crypto_aead_ctx(tfm),
77 crypto_tfm_alg_name(&tfm->base));
78
79 /* Unmap enckey buffer */
80 if (ctx->enckey) {
81 dma_free_coherent(dev, AES_MAX_KEY_SIZE, ctx->enckey,
82 ctx->enckey_dma_addr);
83 dev_dbg(dev, "Freed enckey DMA buffer enckey_dma_addr=%pad\n",
84 &ctx->enckey_dma_addr);
85 ctx->enckey_dma_addr = 0;
86 ctx->enckey = NULL;
87 }
88
89 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
90 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
91
92 if (xcbc->xcbc_keys) {
93 dma_free_coherent(dev, CC_AES_128_BIT_KEY_SIZE * 3,
94 xcbc->xcbc_keys,
95 xcbc->xcbc_keys_dma_addr);
96 }
97 dev_dbg(dev, "Freed xcbc_keys DMA buffer xcbc_keys_dma_addr=%pad\n",
98 &xcbc->xcbc_keys_dma_addr);
99 xcbc->xcbc_keys_dma_addr = 0;
100 xcbc->xcbc_keys = NULL;
101 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC auth. */
102 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
103
104 if (hmac->ipad_opad) {
105 dma_free_coherent(dev, 2 * MAX_HMAC_DIGEST_SIZE,
106 hmac->ipad_opad,
107 hmac->ipad_opad_dma_addr);
108 dev_dbg(dev, "Freed ipad_opad DMA buffer ipad_opad_dma_addr=%pad\n",
109 &hmac->ipad_opad_dma_addr);
110 hmac->ipad_opad_dma_addr = 0;
111 hmac->ipad_opad = NULL;
112 }
113 if (hmac->padded_authkey) {
114 dma_free_coherent(dev, MAX_HMAC_BLOCK_SIZE,
115 hmac->padded_authkey,
116 hmac->padded_authkey_dma_addr);
117 dev_dbg(dev, "Freed padded_authkey DMA buffer padded_authkey_dma_addr=%pad\n",
118 &hmac->padded_authkey_dma_addr);
119 hmac->padded_authkey_dma_addr = 0;
120 hmac->padded_authkey = NULL;
121 }
122 }
123 }
124
125 static int cc_aead_init(struct crypto_aead *tfm)
126 {
127 struct aead_alg *alg = crypto_aead_alg(tfm);
128 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
129 struct cc_crypto_alg *cc_alg =
130 container_of(alg, struct cc_crypto_alg, aead_alg);
131 struct device *dev = drvdata_to_dev(cc_alg->drvdata);
132
133 dev_dbg(dev, "Initializing context @%p for %s\n", ctx,
134 crypto_tfm_alg_name(&tfm->base));
135
136 /* Initialize modes in instance */
137 ctx->cipher_mode = cc_alg->cipher_mode;
138 ctx->flow_mode = cc_alg->flow_mode;
139 ctx->auth_mode = cc_alg->auth_mode;
140 ctx->drvdata = cc_alg->drvdata;
141 crypto_aead_set_reqsize(tfm, sizeof(struct aead_req_ctx));
142
143 /* Allocate key buffer, cache line aligned */
144 ctx->enckey = dma_alloc_coherent(dev, AES_MAX_KEY_SIZE,
145 &ctx->enckey_dma_addr, GFP_KERNEL);
146 if (!ctx->enckey) {
147 dev_err(dev, "Failed allocating key buffer\n");
148 goto init_failed;
149 }
150 dev_dbg(dev, "Allocated enckey buffer in context ctx->enckey=@%p\n",
151 ctx->enckey);
152
153 /* Set default authlen value */
154
155 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) { /* XCBC authetication */
156 struct cc_xcbc_s *xcbc = &ctx->auth_state.xcbc;
157 const unsigned int key_size = CC_AES_128_BIT_KEY_SIZE * 3;
158
159 /* Allocate dma-coherent buffer for XCBC's K1+K2+K3 */
160 /* (and temporary for user key - up to 256b) */
161 xcbc->xcbc_keys = dma_alloc_coherent(dev, key_size,
162 &xcbc->xcbc_keys_dma_addr,
163 GFP_KERNEL);
164 if (!xcbc->xcbc_keys) {
165 dev_err(dev, "Failed allocating buffer for XCBC keys\n");
166 goto init_failed;
167 }
168 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC authentication */
169 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
170 const unsigned int digest_size = 2 * MAX_HMAC_DIGEST_SIZE;
171 dma_addr_t *pkey_dma = &hmac->padded_authkey_dma_addr;
172
173 /* Allocate dma-coherent buffer for IPAD + OPAD */
174 hmac->ipad_opad = dma_alloc_coherent(dev, digest_size,
175 &hmac->ipad_opad_dma_addr,
176 GFP_KERNEL);
177
178 if (!hmac->ipad_opad) {
179 dev_err(dev, "Failed allocating IPAD/OPAD buffer\n");
180 goto init_failed;
181 }
182
183 dev_dbg(dev, "Allocated authkey buffer in context ctx->authkey=@%p\n",
184 hmac->ipad_opad);
185
186 hmac->padded_authkey = dma_alloc_coherent(dev,
187 MAX_HMAC_BLOCK_SIZE,
188 pkey_dma,
189 GFP_KERNEL);
190
191 if (!hmac->padded_authkey) {
192 dev_err(dev, "failed to allocate padded_authkey\n");
193 goto init_failed;
194 }
195 } else {
196 ctx->auth_state.hmac.ipad_opad = NULL;
197 ctx->auth_state.hmac.padded_authkey = NULL;
198 }
199
200 return 0;
201
202 init_failed:
203 cc_aead_exit(tfm);
204 return -ENOMEM;
205 }
206
207 static void cc_aead_complete(struct device *dev, void *cc_req, int err)
208 {
209 struct aead_request *areq = (struct aead_request *)cc_req;
210 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
211 struct crypto_aead *tfm = crypto_aead_reqtfm(cc_req);
212 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
213
214 cc_unmap_aead_request(dev, areq);
215
216 /* Restore ordinary iv pointer */
217 areq->iv = areq_ctx->backup_iv;
218
219 if (err)
220 goto done;
221
222 if (areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
223 if (memcmp(areq_ctx->mac_buf, areq_ctx->icv_virt_addr,
224 ctx->authsize) != 0) {
225 dev_dbg(dev, "Payload authentication failure, (auth-size=%d, cipher=%d)\n",
226 ctx->authsize, ctx->cipher_mode);
227 /* In case of payload authentication failure, MUST NOT
228 * revealed the decrypted message --> zero its memory.
229 */
230 cc_zero_sgl(areq->dst, areq->cryptlen);
231 err = -EBADMSG;
232 }
233 } else { /*ENCRYPT*/
234 if (areq_ctx->is_icv_fragmented) {
235 u32 skip = areq->cryptlen + areq_ctx->dst_offset;
236
237 cc_copy_sg_portion(dev, areq_ctx->mac_buf,
238 areq_ctx->dst_sgl, skip,
239 (skip + ctx->authsize),
240 CC_SG_FROM_BUF);
241 }
242
243 /* If an IV was generated, copy it back to the user provided
244 * buffer.
245 */
246 if (areq_ctx->backup_giv) {
247 if (ctx->cipher_mode == DRV_CIPHER_CTR)
248 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
249 CTR_RFC3686_NONCE_SIZE,
250 CTR_RFC3686_IV_SIZE);
251 else if (ctx->cipher_mode == DRV_CIPHER_CCM)
252 memcpy(areq_ctx->backup_giv, areq_ctx->ctr_iv +
253 CCM_BLOCK_IV_OFFSET, CCM_BLOCK_IV_SIZE);
254 }
255 }
256 done:
257 aead_request_complete(areq, err);
258 }
259
260 static unsigned int xcbc_setkey(struct cc_hw_desc *desc,
261 struct cc_aead_ctx *ctx)
262 {
263 /* Load the AES key */
264 hw_desc_init(&desc[0]);
265 /* We are using for the source/user key the same buffer
266 * as for the output keys, * because after this key loading it
267 * is not needed anymore
268 */
269 set_din_type(&desc[0], DMA_DLLI,
270 ctx->auth_state.xcbc.xcbc_keys_dma_addr, ctx->auth_keylen,
271 NS_BIT);
272 set_cipher_mode(&desc[0], DRV_CIPHER_ECB);
273 set_cipher_config0(&desc[0], DRV_CRYPTO_DIRECTION_ENCRYPT);
274 set_key_size_aes(&desc[0], ctx->auth_keylen);
275 set_flow_mode(&desc[0], S_DIN_to_AES);
276 set_setup_mode(&desc[0], SETUP_LOAD_KEY0);
277
278 hw_desc_init(&desc[1]);
279 set_din_const(&desc[1], 0x01010101, CC_AES_128_BIT_KEY_SIZE);
280 set_flow_mode(&desc[1], DIN_AES_DOUT);
281 set_dout_dlli(&desc[1], ctx->auth_state.xcbc.xcbc_keys_dma_addr,
282 AES_KEYSIZE_128, NS_BIT, 0);
283
284 hw_desc_init(&desc[2]);
285 set_din_const(&desc[2], 0x02020202, CC_AES_128_BIT_KEY_SIZE);
286 set_flow_mode(&desc[2], DIN_AES_DOUT);
287 set_dout_dlli(&desc[2], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
288 + AES_KEYSIZE_128),
289 AES_KEYSIZE_128, NS_BIT, 0);
290
291 hw_desc_init(&desc[3]);
292 set_din_const(&desc[3], 0x03030303, CC_AES_128_BIT_KEY_SIZE);
293 set_flow_mode(&desc[3], DIN_AES_DOUT);
294 set_dout_dlli(&desc[3], (ctx->auth_state.xcbc.xcbc_keys_dma_addr
295 + 2 * AES_KEYSIZE_128),
296 AES_KEYSIZE_128, NS_BIT, 0);
297
298 return 4;
299 }
300
301 static int hmac_setkey(struct cc_hw_desc *desc, struct cc_aead_ctx *ctx)
302 {
303 unsigned int hmac_pad_const[2] = { HMAC_IPAD_CONST, HMAC_OPAD_CONST };
304 unsigned int digest_ofs = 0;
305 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
306 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
307 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
308 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
309 struct cc_hmac_s *hmac = &ctx->auth_state.hmac;
310
311 unsigned int idx = 0;
312 int i;
313
314 /* calc derived HMAC key */
315 for (i = 0; i < 2; i++) {
316 /* Load hash initial state */
317 hw_desc_init(&desc[idx]);
318 set_cipher_mode(&desc[idx], hash_mode);
319 set_din_sram(&desc[idx],
320 cc_larval_digest_addr(ctx->drvdata,
321 ctx->auth_mode),
322 digest_size);
323 set_flow_mode(&desc[idx], S_DIN_to_HASH);
324 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
325 idx++;
326
327 /* Load the hash current length*/
328 hw_desc_init(&desc[idx]);
329 set_cipher_mode(&desc[idx], hash_mode);
330 set_din_const(&desc[idx], 0, ctx->drvdata->hash_len_sz);
331 set_flow_mode(&desc[idx], S_DIN_to_HASH);
332 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
333 idx++;
334
335 /* Prepare ipad key */
336 hw_desc_init(&desc[idx]);
337 set_xor_val(&desc[idx], hmac_pad_const[i]);
338 set_cipher_mode(&desc[idx], hash_mode);
339 set_flow_mode(&desc[idx], S_DIN_to_HASH);
340 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
341 idx++;
342
343 /* Perform HASH update */
344 hw_desc_init(&desc[idx]);
345 set_din_type(&desc[idx], DMA_DLLI,
346 hmac->padded_authkey_dma_addr,
347 SHA256_BLOCK_SIZE, NS_BIT);
348 set_cipher_mode(&desc[idx], hash_mode);
349 set_xor_active(&desc[idx]);
350 set_flow_mode(&desc[idx], DIN_HASH);
351 idx++;
352
353 /* Get the digset */
354 hw_desc_init(&desc[idx]);
355 set_cipher_mode(&desc[idx], hash_mode);
356 set_dout_dlli(&desc[idx],
357 (hmac->ipad_opad_dma_addr + digest_ofs),
358 digest_size, NS_BIT, 0);
359 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
360 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
361 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
362 idx++;
363
364 digest_ofs += digest_size;
365 }
366
367 return idx;
368 }
369
370 static int validate_keys_sizes(struct cc_aead_ctx *ctx)
371 {
372 struct device *dev = drvdata_to_dev(ctx->drvdata);
373
374 dev_dbg(dev, "enc_keylen=%u authkeylen=%u\n",
375 ctx->enc_keylen, ctx->auth_keylen);
376
377 switch (ctx->auth_mode) {
378 case DRV_HASH_SHA1:
379 case DRV_HASH_SHA256:
380 break;
381 case DRV_HASH_XCBC_MAC:
382 if (ctx->auth_keylen != AES_KEYSIZE_128 &&
383 ctx->auth_keylen != AES_KEYSIZE_192 &&
384 ctx->auth_keylen != AES_KEYSIZE_256)
385 return -ENOTSUPP;
386 break;
387 case DRV_HASH_NULL: /* Not authenc (e.g., CCM) - no auth_key) */
388 if (ctx->auth_keylen > 0)
389 return -EINVAL;
390 break;
391 default:
392 dev_err(dev, "Invalid auth_mode=%d\n", ctx->auth_mode);
393 return -EINVAL;
394 }
395 /* Check cipher key size */
396 if (ctx->flow_mode == S_DIN_to_DES) {
397 if (ctx->enc_keylen != DES3_EDE_KEY_SIZE) {
398 dev_err(dev, "Invalid cipher(3DES) key size: %u\n",
399 ctx->enc_keylen);
400 return -EINVAL;
401 }
402 } else { /* Default assumed to be AES ciphers */
403 if (ctx->enc_keylen != AES_KEYSIZE_128 &&
404 ctx->enc_keylen != AES_KEYSIZE_192 &&
405 ctx->enc_keylen != AES_KEYSIZE_256) {
406 dev_err(dev, "Invalid cipher(AES) key size: %u\n",
407 ctx->enc_keylen);
408 return -EINVAL;
409 }
410 }
411
412 return 0; /* All tests of keys sizes passed */
413 }
414
415 /* This function prepers the user key so it can pass to the hmac processing
416 * (copy to intenral buffer or hash in case of key longer than block
417 */
418 static int cc_get_plain_hmac_key(struct crypto_aead *tfm, const u8 *authkey,
419 unsigned int keylen)
420 {
421 dma_addr_t key_dma_addr = 0;
422 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
423 struct device *dev = drvdata_to_dev(ctx->drvdata);
424 u32 larval_addr = cc_larval_digest_addr(ctx->drvdata, ctx->auth_mode);
425 struct cc_crypto_req cc_req = {};
426 unsigned int blocksize;
427 unsigned int digestsize;
428 unsigned int hashmode;
429 unsigned int idx = 0;
430 int rc = 0;
431 u8 *key = NULL;
432 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
433 dma_addr_t padded_authkey_dma_addr =
434 ctx->auth_state.hmac.padded_authkey_dma_addr;
435
436 switch (ctx->auth_mode) { /* auth_key required and >0 */
437 case DRV_HASH_SHA1:
438 blocksize = SHA1_BLOCK_SIZE;
439 digestsize = SHA1_DIGEST_SIZE;
440 hashmode = DRV_HASH_HW_SHA1;
441 break;
442 case DRV_HASH_SHA256:
443 default:
444 blocksize = SHA256_BLOCK_SIZE;
445 digestsize = SHA256_DIGEST_SIZE;
446 hashmode = DRV_HASH_HW_SHA256;
447 }
448
449 if (keylen != 0) {
450
451 key = kmemdup(authkey, keylen, GFP_KERNEL);
452 if (!key)
453 return -ENOMEM;
454
455 key_dma_addr = dma_map_single(dev, (void *)key, keylen,
456 DMA_TO_DEVICE);
457 if (dma_mapping_error(dev, key_dma_addr)) {
458 dev_err(dev, "Mapping key va=0x%p len=%u for DMA failed\n",
459 key, keylen);
460 kzfree(key);
461 return -ENOMEM;
462 }
463 if (keylen > blocksize) {
464 /* Load hash initial state */
465 hw_desc_init(&desc[idx]);
466 set_cipher_mode(&desc[idx], hashmode);
467 set_din_sram(&desc[idx], larval_addr, digestsize);
468 set_flow_mode(&desc[idx], S_DIN_to_HASH);
469 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
470 idx++;
471
472 /* Load the hash current length*/
473 hw_desc_init(&desc[idx]);
474 set_cipher_mode(&desc[idx], hashmode);
475 set_din_const(&desc[idx], 0, ctx->drvdata->hash_len_sz);
476 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
477 set_flow_mode(&desc[idx], S_DIN_to_HASH);
478 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
479 idx++;
480
481 hw_desc_init(&desc[idx]);
482 set_din_type(&desc[idx], DMA_DLLI,
483 key_dma_addr, keylen, NS_BIT);
484 set_flow_mode(&desc[idx], DIN_HASH);
485 idx++;
486
487 /* Get hashed key */
488 hw_desc_init(&desc[idx]);
489 set_cipher_mode(&desc[idx], hashmode);
490 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
491 digestsize, NS_BIT, 0);
492 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
493 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
494 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
495 set_cipher_config0(&desc[idx],
496 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
497 idx++;
498
499 hw_desc_init(&desc[idx]);
500 set_din_const(&desc[idx], 0, (blocksize - digestsize));
501 set_flow_mode(&desc[idx], BYPASS);
502 set_dout_dlli(&desc[idx], (padded_authkey_dma_addr +
503 digestsize), (blocksize - digestsize),
504 NS_BIT, 0);
505 idx++;
506 } else {
507 hw_desc_init(&desc[idx]);
508 set_din_type(&desc[idx], DMA_DLLI, key_dma_addr,
509 keylen, NS_BIT);
510 set_flow_mode(&desc[idx], BYPASS);
511 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
512 keylen, NS_BIT, 0);
513 idx++;
514
515 if ((blocksize - keylen) != 0) {
516 hw_desc_init(&desc[idx]);
517 set_din_const(&desc[idx], 0,
518 (blocksize - keylen));
519 set_flow_mode(&desc[idx], BYPASS);
520 set_dout_dlli(&desc[idx],
521 (padded_authkey_dma_addr +
522 keylen),
523 (blocksize - keylen), NS_BIT, 0);
524 idx++;
525 }
526 }
527 } else {
528 hw_desc_init(&desc[idx]);
529 set_din_const(&desc[idx], 0, (blocksize - keylen));
530 set_flow_mode(&desc[idx], BYPASS);
531 set_dout_dlli(&desc[idx], padded_authkey_dma_addr,
532 blocksize, NS_BIT, 0);
533 idx++;
534 }
535
536 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, idx);
537 if (rc)
538 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
539
540 if (key_dma_addr)
541 dma_unmap_single(dev, key_dma_addr, keylen, DMA_TO_DEVICE);
542
543 kzfree(key);
544
545 return rc;
546 }
547
548 static int cc_aead_setkey(struct crypto_aead *tfm, const u8 *key,
549 unsigned int keylen)
550 {
551 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
552 struct cc_crypto_req cc_req = {};
553 struct cc_hw_desc desc[MAX_AEAD_SETKEY_SEQ];
554 unsigned int seq_len = 0;
555 struct device *dev = drvdata_to_dev(ctx->drvdata);
556 const u8 *enckey, *authkey;
557 int rc;
558
559 dev_dbg(dev, "Setting key in context @%p for %s. key=%p keylen=%u\n",
560 ctx, crypto_tfm_alg_name(crypto_aead_tfm(tfm)), key, keylen);
561
562 /* STAT_PHASE_0: Init and sanity checks */
563
564 if (ctx->auth_mode != DRV_HASH_NULL) { /* authenc() alg. */
565 struct crypto_authenc_keys keys;
566
567 rc = crypto_authenc_extractkeys(&keys, key, keylen);
568 if (rc)
569 goto badkey;
570 enckey = keys.enckey;
571 authkey = keys.authkey;
572 ctx->enc_keylen = keys.enckeylen;
573 ctx->auth_keylen = keys.authkeylen;
574
575 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
576 /* the nonce is stored in bytes at end of key */
577 rc = -EINVAL;
578 if (ctx->enc_keylen <
579 (AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE))
580 goto badkey;
581 /* Copy nonce from last 4 bytes in CTR key to
582 * first 4 bytes in CTR IV
583 */
584 memcpy(ctx->ctr_nonce, enckey + ctx->enc_keylen -
585 CTR_RFC3686_NONCE_SIZE, CTR_RFC3686_NONCE_SIZE);
586 /* Set CTR key size */
587 ctx->enc_keylen -= CTR_RFC3686_NONCE_SIZE;
588 }
589 } else { /* non-authenc - has just one key */
590 enckey = key;
591 authkey = NULL;
592 ctx->enc_keylen = keylen;
593 ctx->auth_keylen = 0;
594 }
595
596 rc = validate_keys_sizes(ctx);
597 if (rc)
598 goto badkey;
599
600 /* STAT_PHASE_1: Copy key to ctx */
601
602 /* Get key material */
603 memcpy(ctx->enckey, enckey, ctx->enc_keylen);
604 if (ctx->enc_keylen == 24)
605 memset(ctx->enckey + 24, 0, CC_AES_KEY_SIZE_MAX - 24);
606 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
607 memcpy(ctx->auth_state.xcbc.xcbc_keys, authkey,
608 ctx->auth_keylen);
609 } else if (ctx->auth_mode != DRV_HASH_NULL) { /* HMAC */
610 rc = cc_get_plain_hmac_key(tfm, authkey, ctx->auth_keylen);
611 if (rc)
612 goto badkey;
613 }
614
615 /* STAT_PHASE_2: Create sequence */
616
617 switch (ctx->auth_mode) {
618 case DRV_HASH_SHA1:
619 case DRV_HASH_SHA256:
620 seq_len = hmac_setkey(desc, ctx);
621 break;
622 case DRV_HASH_XCBC_MAC:
623 seq_len = xcbc_setkey(desc, ctx);
624 break;
625 case DRV_HASH_NULL: /* non-authenc modes, e.g., CCM */
626 break; /* No auth. key setup */
627 default:
628 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
629 rc = -ENOTSUPP;
630 goto badkey;
631 }
632
633 /* STAT_PHASE_3: Submit sequence to HW */
634
635 if (seq_len > 0) { /* For CCM there is no sequence to setup the key */
636 rc = cc_send_sync_request(ctx->drvdata, &cc_req, desc, seq_len);
637 if (rc) {
638 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
639 goto setkey_error;
640 }
641 }
642
643 /* Update STAT_PHASE_3 */
644 return rc;
645
646 badkey:
647 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
648
649 setkey_error:
650 return rc;
651 }
652
653 static int cc_rfc4309_ccm_setkey(struct crypto_aead *tfm, const u8 *key,
654 unsigned int keylen)
655 {
656 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
657
658 if (keylen < 3)
659 return -EINVAL;
660
661 keylen -= 3;
662 memcpy(ctx->ctr_nonce, key + keylen, 3);
663
664 return cc_aead_setkey(tfm, key, keylen);
665 }
666
667 static int cc_aead_setauthsize(struct crypto_aead *authenc,
668 unsigned int authsize)
669 {
670 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
671 struct device *dev = drvdata_to_dev(ctx->drvdata);
672
673 /* Unsupported auth. sizes */
674 if (authsize == 0 ||
675 authsize > crypto_aead_maxauthsize(authenc)) {
676 return -ENOTSUPP;
677 }
678
679 ctx->authsize = authsize;
680 dev_dbg(dev, "authlen=%d\n", ctx->authsize);
681
682 return 0;
683 }
684
685 static int cc_rfc4309_ccm_setauthsize(struct crypto_aead *authenc,
686 unsigned int authsize)
687 {
688 switch (authsize) {
689 case 8:
690 case 12:
691 case 16:
692 break;
693 default:
694 return -EINVAL;
695 }
696
697 return cc_aead_setauthsize(authenc, authsize);
698 }
699
700 static int cc_ccm_setauthsize(struct crypto_aead *authenc,
701 unsigned int authsize)
702 {
703 switch (authsize) {
704 case 4:
705 case 6:
706 case 8:
707 case 10:
708 case 12:
709 case 14:
710 case 16:
711 break;
712 default:
713 return -EINVAL;
714 }
715
716 return cc_aead_setauthsize(authenc, authsize);
717 }
718
719 static void cc_set_assoc_desc(struct aead_request *areq, unsigned int flow_mode,
720 struct cc_hw_desc desc[], unsigned int *seq_size)
721 {
722 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
723 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
724 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
725 enum cc_req_dma_buf_type assoc_dma_type = areq_ctx->assoc_buff_type;
726 unsigned int idx = *seq_size;
727 struct device *dev = drvdata_to_dev(ctx->drvdata);
728
729 switch (assoc_dma_type) {
730 case CC_DMA_BUF_DLLI:
731 dev_dbg(dev, "ASSOC buffer type DLLI\n");
732 hw_desc_init(&desc[idx]);
733 set_din_type(&desc[idx], DMA_DLLI, sg_dma_address(areq->src),
734 areq_ctx->assoclen, NS_BIT);
735 set_flow_mode(&desc[idx], flow_mode);
736 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
737 areq_ctx->cryptlen > 0)
738 set_din_not_last_indication(&desc[idx]);
739 break;
740 case CC_DMA_BUF_MLLI:
741 dev_dbg(dev, "ASSOC buffer type MLLI\n");
742 hw_desc_init(&desc[idx]);
743 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->assoc.sram_addr,
744 areq_ctx->assoc.mlli_nents, NS_BIT);
745 set_flow_mode(&desc[idx], flow_mode);
746 if (ctx->auth_mode == DRV_HASH_XCBC_MAC &&
747 areq_ctx->cryptlen > 0)
748 set_din_not_last_indication(&desc[idx]);
749 break;
750 case CC_DMA_BUF_NULL:
751 default:
752 dev_err(dev, "Invalid ASSOC buffer type\n");
753 }
754
755 *seq_size = (++idx);
756 }
757
758 static void cc_proc_authen_desc(struct aead_request *areq,
759 unsigned int flow_mode,
760 struct cc_hw_desc desc[],
761 unsigned int *seq_size, int direct)
762 {
763 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
764 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
765 unsigned int idx = *seq_size;
766 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
767 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
768 struct device *dev = drvdata_to_dev(ctx->drvdata);
769
770 switch (data_dma_type) {
771 case CC_DMA_BUF_DLLI:
772 {
773 struct scatterlist *cipher =
774 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
775 areq_ctx->dst_sgl : areq_ctx->src_sgl;
776
777 unsigned int offset =
778 (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ?
779 areq_ctx->dst_offset : areq_ctx->src_offset;
780 dev_dbg(dev, "AUTHENC: SRC/DST buffer type DLLI\n");
781 hw_desc_init(&desc[idx]);
782 set_din_type(&desc[idx], DMA_DLLI,
783 (sg_dma_address(cipher) + offset),
784 areq_ctx->cryptlen, NS_BIT);
785 set_flow_mode(&desc[idx], flow_mode);
786 break;
787 }
788 case CC_DMA_BUF_MLLI:
789 {
790 /* DOUBLE-PASS flow (as default)
791 * assoc. + iv + data -compact in one table
792 * if assoclen is ZERO only IV perform
793 */
794 cc_sram_addr_t mlli_addr = areq_ctx->assoc.sram_addr;
795 u32 mlli_nents = areq_ctx->assoc.mlli_nents;
796
797 if (areq_ctx->is_single_pass) {
798 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
799 mlli_addr = areq_ctx->dst.sram_addr;
800 mlli_nents = areq_ctx->dst.mlli_nents;
801 } else {
802 mlli_addr = areq_ctx->src.sram_addr;
803 mlli_nents = areq_ctx->src.mlli_nents;
804 }
805 }
806
807 dev_dbg(dev, "AUTHENC: SRC/DST buffer type MLLI\n");
808 hw_desc_init(&desc[idx]);
809 set_din_type(&desc[idx], DMA_MLLI, mlli_addr, mlli_nents,
810 NS_BIT);
811 set_flow_mode(&desc[idx], flow_mode);
812 break;
813 }
814 case CC_DMA_BUF_NULL:
815 default:
816 dev_err(dev, "AUTHENC: Invalid SRC/DST buffer type\n");
817 }
818
819 *seq_size = (++idx);
820 }
821
822 static void cc_proc_cipher_desc(struct aead_request *areq,
823 unsigned int flow_mode,
824 struct cc_hw_desc desc[],
825 unsigned int *seq_size)
826 {
827 unsigned int idx = *seq_size;
828 struct aead_req_ctx *areq_ctx = aead_request_ctx(areq);
829 enum cc_req_dma_buf_type data_dma_type = areq_ctx->data_buff_type;
830 struct crypto_aead *tfm = crypto_aead_reqtfm(areq);
831 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
832 struct device *dev = drvdata_to_dev(ctx->drvdata);
833
834 if (areq_ctx->cryptlen == 0)
835 return; /*null processing*/
836
837 switch (data_dma_type) {
838 case CC_DMA_BUF_DLLI:
839 dev_dbg(dev, "CIPHER: SRC/DST buffer type DLLI\n");
840 hw_desc_init(&desc[idx]);
841 set_din_type(&desc[idx], DMA_DLLI,
842 (sg_dma_address(areq_ctx->src_sgl) +
843 areq_ctx->src_offset), areq_ctx->cryptlen,
844 NS_BIT);
845 set_dout_dlli(&desc[idx],
846 (sg_dma_address(areq_ctx->dst_sgl) +
847 areq_ctx->dst_offset),
848 areq_ctx->cryptlen, NS_BIT, 0);
849 set_flow_mode(&desc[idx], flow_mode);
850 break;
851 case CC_DMA_BUF_MLLI:
852 dev_dbg(dev, "CIPHER: SRC/DST buffer type MLLI\n");
853 hw_desc_init(&desc[idx]);
854 set_din_type(&desc[idx], DMA_MLLI, areq_ctx->src.sram_addr,
855 areq_ctx->src.mlli_nents, NS_BIT);
856 set_dout_mlli(&desc[idx], areq_ctx->dst.sram_addr,
857 areq_ctx->dst.mlli_nents, NS_BIT, 0);
858 set_flow_mode(&desc[idx], flow_mode);
859 break;
860 case CC_DMA_BUF_NULL:
861 default:
862 dev_err(dev, "CIPHER: Invalid SRC/DST buffer type\n");
863 }
864
865 *seq_size = (++idx);
866 }
867
868 static void cc_proc_digest_desc(struct aead_request *req,
869 struct cc_hw_desc desc[],
870 unsigned int *seq_size)
871 {
872 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
873 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
874 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
875 unsigned int idx = *seq_size;
876 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
877 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
878 int direct = req_ctx->gen_ctx.op_type;
879
880 /* Get final ICV result */
881 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
882 hw_desc_init(&desc[idx]);
883 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
884 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
885 set_dout_dlli(&desc[idx], req_ctx->icv_dma_addr, ctx->authsize,
886 NS_BIT, 1);
887 set_queue_last_ind(ctx->drvdata, &desc[idx]);
888 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
889 set_aes_not_hash_mode(&desc[idx]);
890 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
891 } else {
892 set_cipher_config0(&desc[idx],
893 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
894 set_cipher_mode(&desc[idx], hash_mode);
895 }
896 } else { /*Decrypt*/
897 /* Get ICV out from hardware */
898 hw_desc_init(&desc[idx]);
899 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
900 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
901 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr,
902 ctx->authsize, NS_BIT, 1);
903 set_queue_last_ind(ctx->drvdata, &desc[idx]);
904 set_cipher_config0(&desc[idx],
905 HASH_DIGEST_RESULT_LITTLE_ENDIAN);
906 set_cipher_config1(&desc[idx], HASH_PADDING_DISABLED);
907 if (ctx->auth_mode == DRV_HASH_XCBC_MAC) {
908 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
909 set_aes_not_hash_mode(&desc[idx]);
910 } else {
911 set_cipher_mode(&desc[idx], hash_mode);
912 }
913 }
914
915 *seq_size = (++idx);
916 }
917
918 static void cc_set_cipher_desc(struct aead_request *req,
919 struct cc_hw_desc desc[],
920 unsigned int *seq_size)
921 {
922 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
923 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
924 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
925 unsigned int hw_iv_size = req_ctx->hw_iv_size;
926 unsigned int idx = *seq_size;
927 int direct = req_ctx->gen_ctx.op_type;
928
929 /* Setup cipher state */
930 hw_desc_init(&desc[idx]);
931 set_cipher_config0(&desc[idx], direct);
932 set_flow_mode(&desc[idx], ctx->flow_mode);
933 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gen_ctx.iv_dma_addr,
934 hw_iv_size, NS_BIT);
935 if (ctx->cipher_mode == DRV_CIPHER_CTR)
936 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
937 else
938 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
939 set_cipher_mode(&desc[idx], ctx->cipher_mode);
940 idx++;
941
942 /* Setup enc. key */
943 hw_desc_init(&desc[idx]);
944 set_cipher_config0(&desc[idx], direct);
945 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
946 set_flow_mode(&desc[idx], ctx->flow_mode);
947 if (ctx->flow_mode == S_DIN_to_AES) {
948 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
949 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
950 ctx->enc_keylen), NS_BIT);
951 set_key_size_aes(&desc[idx], ctx->enc_keylen);
952 } else {
953 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
954 ctx->enc_keylen, NS_BIT);
955 set_key_size_des(&desc[idx], ctx->enc_keylen);
956 }
957 set_cipher_mode(&desc[idx], ctx->cipher_mode);
958 idx++;
959
960 *seq_size = idx;
961 }
962
963 static void cc_proc_cipher(struct aead_request *req, struct cc_hw_desc desc[],
964 unsigned int *seq_size, unsigned int data_flow_mode)
965 {
966 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
967 int direct = req_ctx->gen_ctx.op_type;
968 unsigned int idx = *seq_size;
969
970 if (req_ctx->cryptlen == 0)
971 return; /*null processing*/
972
973 cc_set_cipher_desc(req, desc, &idx);
974 cc_proc_cipher_desc(req, data_flow_mode, desc, &idx);
975 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
976 /* We must wait for DMA to write all cipher */
977 hw_desc_init(&desc[idx]);
978 set_din_no_dma(&desc[idx], 0, 0xfffff0);
979 set_dout_no_dma(&desc[idx], 0, 0, 1);
980 idx++;
981 }
982
983 *seq_size = idx;
984 }
985
986 static void cc_set_hmac_desc(struct aead_request *req, struct cc_hw_desc desc[],
987 unsigned int *seq_size)
988 {
989 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
990 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
991 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
992 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
993 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
994 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
995 unsigned int idx = *seq_size;
996
997 /* Loading hash ipad xor key state */
998 hw_desc_init(&desc[idx]);
999 set_cipher_mode(&desc[idx], hash_mode);
1000 set_din_type(&desc[idx], DMA_DLLI,
1001 ctx->auth_state.hmac.ipad_opad_dma_addr, digest_size,
1002 NS_BIT);
1003 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1004 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1005 idx++;
1006
1007 /* Load init. digest len (64 bytes) */
1008 hw_desc_init(&desc[idx]);
1009 set_cipher_mode(&desc[idx], hash_mode);
1010 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1011 ctx->drvdata->hash_len_sz);
1012 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1013 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1014 idx++;
1015
1016 *seq_size = idx;
1017 }
1018
1019 static void cc_set_xcbc_desc(struct aead_request *req, struct cc_hw_desc desc[],
1020 unsigned int *seq_size)
1021 {
1022 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1023 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1024 unsigned int idx = *seq_size;
1025
1026 /* Loading MAC state */
1027 hw_desc_init(&desc[idx]);
1028 set_din_const(&desc[idx], 0, CC_AES_BLOCK_SIZE);
1029 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1030 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1031 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1032 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1033 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1034 set_aes_not_hash_mode(&desc[idx]);
1035 idx++;
1036
1037 /* Setup XCBC MAC K1 */
1038 hw_desc_init(&desc[idx]);
1039 set_din_type(&desc[idx], DMA_DLLI,
1040 ctx->auth_state.xcbc.xcbc_keys_dma_addr,
1041 AES_KEYSIZE_128, NS_BIT);
1042 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1043 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1044 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1045 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1046 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1047 set_aes_not_hash_mode(&desc[idx]);
1048 idx++;
1049
1050 /* Setup XCBC MAC K2 */
1051 hw_desc_init(&desc[idx]);
1052 set_din_type(&desc[idx], DMA_DLLI,
1053 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1054 AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1055 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1056 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1057 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1058 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1059 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1060 set_aes_not_hash_mode(&desc[idx]);
1061 idx++;
1062
1063 /* Setup XCBC MAC K3 */
1064 hw_desc_init(&desc[idx]);
1065 set_din_type(&desc[idx], DMA_DLLI,
1066 (ctx->auth_state.xcbc.xcbc_keys_dma_addr +
1067 2 * AES_KEYSIZE_128), AES_KEYSIZE_128, NS_BIT);
1068 set_setup_mode(&desc[idx], SETUP_LOAD_STATE2);
1069 set_cipher_mode(&desc[idx], DRV_CIPHER_XCBC_MAC);
1070 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1071 set_key_size_aes(&desc[idx], CC_AES_128_BIT_KEY_SIZE);
1072 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1073 set_aes_not_hash_mode(&desc[idx]);
1074 idx++;
1075
1076 *seq_size = idx;
1077 }
1078
1079 static void cc_proc_header_desc(struct aead_request *req,
1080 struct cc_hw_desc desc[],
1081 unsigned int *seq_size)
1082 {
1083 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1084 unsigned int idx = *seq_size;
1085
1086 /* Hash associated data */
1087 if (areq_ctx->assoclen > 0)
1088 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1089
1090 /* Hash IV */
1091 *seq_size = idx;
1092 }
1093
1094 static void cc_proc_scheme_desc(struct aead_request *req,
1095 struct cc_hw_desc desc[],
1096 unsigned int *seq_size)
1097 {
1098 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1099 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1100 struct cc_aead_handle *aead_handle = ctx->drvdata->aead_handle;
1101 unsigned int hash_mode = (ctx->auth_mode == DRV_HASH_SHA1) ?
1102 DRV_HASH_HW_SHA1 : DRV_HASH_HW_SHA256;
1103 unsigned int digest_size = (ctx->auth_mode == DRV_HASH_SHA1) ?
1104 CC_SHA1_DIGEST_SIZE : CC_SHA256_DIGEST_SIZE;
1105 unsigned int idx = *seq_size;
1106
1107 hw_desc_init(&desc[idx]);
1108 set_cipher_mode(&desc[idx], hash_mode);
1109 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1110 ctx->drvdata->hash_len_sz);
1111 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1112 set_setup_mode(&desc[idx], SETUP_WRITE_STATE1);
1113 set_cipher_do(&desc[idx], DO_PAD);
1114 idx++;
1115
1116 /* Get final ICV result */
1117 hw_desc_init(&desc[idx]);
1118 set_dout_sram(&desc[idx], aead_handle->sram_workspace_addr,
1119 digest_size);
1120 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1121 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1122 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1123 set_cipher_mode(&desc[idx], hash_mode);
1124 idx++;
1125
1126 /* Loading hash opad xor key state */
1127 hw_desc_init(&desc[idx]);
1128 set_cipher_mode(&desc[idx], hash_mode);
1129 set_din_type(&desc[idx], DMA_DLLI,
1130 (ctx->auth_state.hmac.ipad_opad_dma_addr + digest_size),
1131 digest_size, NS_BIT);
1132 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1133 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1134 idx++;
1135
1136 /* Load init. digest len (64 bytes) */
1137 hw_desc_init(&desc[idx]);
1138 set_cipher_mode(&desc[idx], hash_mode);
1139 set_din_sram(&desc[idx], cc_digest_len_addr(ctx->drvdata, hash_mode),
1140 ctx->drvdata->hash_len_sz);
1141 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1142 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1143 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1144 idx++;
1145
1146 /* Perform HASH update */
1147 hw_desc_init(&desc[idx]);
1148 set_din_sram(&desc[idx], aead_handle->sram_workspace_addr,
1149 digest_size);
1150 set_flow_mode(&desc[idx], DIN_HASH);
1151 idx++;
1152
1153 *seq_size = idx;
1154 }
1155
1156 static void cc_mlli_to_sram(struct aead_request *req,
1157 struct cc_hw_desc desc[], unsigned int *seq_size)
1158 {
1159 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1160 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1161 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1162 struct device *dev = drvdata_to_dev(ctx->drvdata);
1163
1164 if (req_ctx->assoc_buff_type == CC_DMA_BUF_MLLI ||
1165 req_ctx->data_buff_type == CC_DMA_BUF_MLLI ||
1166 !req_ctx->is_single_pass) {
1167 dev_dbg(dev, "Copy-to-sram: mlli_dma=%08x, mlli_size=%u\n",
1168 (unsigned int)ctx->drvdata->mlli_sram_addr,
1169 req_ctx->mlli_params.mlli_len);
1170 /* Copy MLLI table host-to-sram */
1171 hw_desc_init(&desc[*seq_size]);
1172 set_din_type(&desc[*seq_size], DMA_DLLI,
1173 req_ctx->mlli_params.mlli_dma_addr,
1174 req_ctx->mlli_params.mlli_len, NS_BIT);
1175 set_dout_sram(&desc[*seq_size],
1176 ctx->drvdata->mlli_sram_addr,
1177 req_ctx->mlli_params.mlli_len);
1178 set_flow_mode(&desc[*seq_size], BYPASS);
1179 (*seq_size)++;
1180 }
1181 }
1182
1183 static enum cc_flow_mode cc_get_data_flow(enum drv_crypto_direction direct,
1184 enum cc_flow_mode setup_flow_mode,
1185 bool is_single_pass)
1186 {
1187 enum cc_flow_mode data_flow_mode;
1188
1189 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1190 if (setup_flow_mode == S_DIN_to_AES)
1191 data_flow_mode = is_single_pass ?
1192 AES_to_HASH_and_DOUT : DIN_AES_DOUT;
1193 else
1194 data_flow_mode = is_single_pass ?
1195 DES_to_HASH_and_DOUT : DIN_DES_DOUT;
1196 } else { /* Decrypt */
1197 if (setup_flow_mode == S_DIN_to_AES)
1198 data_flow_mode = is_single_pass ?
1199 AES_and_HASH : DIN_AES_DOUT;
1200 else
1201 data_flow_mode = is_single_pass ?
1202 DES_and_HASH : DIN_DES_DOUT;
1203 }
1204
1205 return data_flow_mode;
1206 }
1207
1208 static void cc_hmac_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1209 unsigned int *seq_size)
1210 {
1211 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1212 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1213 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1214 int direct = req_ctx->gen_ctx.op_type;
1215 unsigned int data_flow_mode =
1216 cc_get_data_flow(direct, ctx->flow_mode,
1217 req_ctx->is_single_pass);
1218
1219 if (req_ctx->is_single_pass) {
1220 /**
1221 * Single-pass flow
1222 */
1223 cc_set_hmac_desc(req, desc, seq_size);
1224 cc_set_cipher_desc(req, desc, seq_size);
1225 cc_proc_header_desc(req, desc, seq_size);
1226 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1227 cc_proc_scheme_desc(req, desc, seq_size);
1228 cc_proc_digest_desc(req, desc, seq_size);
1229 return;
1230 }
1231
1232 /**
1233 * Double-pass flow
1234 * Fallback for unsupported single-pass modes,
1235 * i.e. using assoc. data of non-word-multiple
1236 */
1237 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1238 /* encrypt first.. */
1239 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1240 /* authenc after..*/
1241 cc_set_hmac_desc(req, desc, seq_size);
1242 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1243 cc_proc_scheme_desc(req, desc, seq_size);
1244 cc_proc_digest_desc(req, desc, seq_size);
1245
1246 } else { /*DECRYPT*/
1247 /* authenc first..*/
1248 cc_set_hmac_desc(req, desc, seq_size);
1249 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1250 cc_proc_scheme_desc(req, desc, seq_size);
1251 /* decrypt after.. */
1252 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1253 /* read the digest result with setting the completion bit
1254 * must be after the cipher operation
1255 */
1256 cc_proc_digest_desc(req, desc, seq_size);
1257 }
1258 }
1259
1260 static void
1261 cc_xcbc_authenc(struct aead_request *req, struct cc_hw_desc desc[],
1262 unsigned int *seq_size)
1263 {
1264 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1265 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1266 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1267 int direct = req_ctx->gen_ctx.op_type;
1268 unsigned int data_flow_mode =
1269 cc_get_data_flow(direct, ctx->flow_mode,
1270 req_ctx->is_single_pass);
1271
1272 if (req_ctx->is_single_pass) {
1273 /**
1274 * Single-pass flow
1275 */
1276 cc_set_xcbc_desc(req, desc, seq_size);
1277 cc_set_cipher_desc(req, desc, seq_size);
1278 cc_proc_header_desc(req, desc, seq_size);
1279 cc_proc_cipher_desc(req, data_flow_mode, desc, seq_size);
1280 cc_proc_digest_desc(req, desc, seq_size);
1281 return;
1282 }
1283
1284 /**
1285 * Double-pass flow
1286 * Fallback for unsupported single-pass modes,
1287 * i.e. using assoc. data of non-word-multiple
1288 */
1289 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) {
1290 /* encrypt first.. */
1291 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1292 /* authenc after.. */
1293 cc_set_xcbc_desc(req, desc, seq_size);
1294 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1295 cc_proc_digest_desc(req, desc, seq_size);
1296 } else { /*DECRYPT*/
1297 /* authenc first.. */
1298 cc_set_xcbc_desc(req, desc, seq_size);
1299 cc_proc_authen_desc(req, DIN_HASH, desc, seq_size, direct);
1300 /* decrypt after..*/
1301 cc_proc_cipher(req, desc, seq_size, data_flow_mode);
1302 /* read the digest result with setting the completion bit
1303 * must be after the cipher operation
1304 */
1305 cc_proc_digest_desc(req, desc, seq_size);
1306 }
1307 }
1308
1309 static int validate_data_size(struct cc_aead_ctx *ctx,
1310 enum drv_crypto_direction direct,
1311 struct aead_request *req)
1312 {
1313 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1314 struct device *dev = drvdata_to_dev(ctx->drvdata);
1315 unsigned int assoclen = areq_ctx->assoclen;
1316 unsigned int cipherlen = (direct == DRV_CRYPTO_DIRECTION_DECRYPT) ?
1317 (req->cryptlen - ctx->authsize) : req->cryptlen;
1318
1319 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT &&
1320 req->cryptlen < ctx->authsize)
1321 goto data_size_err;
1322
1323 areq_ctx->is_single_pass = true; /*defaulted to fast flow*/
1324
1325 switch (ctx->flow_mode) {
1326 case S_DIN_to_AES:
1327 if (ctx->cipher_mode == DRV_CIPHER_CBC &&
1328 !IS_ALIGNED(cipherlen, AES_BLOCK_SIZE))
1329 goto data_size_err;
1330 if (ctx->cipher_mode == DRV_CIPHER_CCM)
1331 break;
1332 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1333 if (areq_ctx->plaintext_authenticate_only)
1334 areq_ctx->is_single_pass = false;
1335 break;
1336 }
1337
1338 if (!IS_ALIGNED(assoclen, sizeof(u32)))
1339 areq_ctx->is_single_pass = false;
1340
1341 if (ctx->cipher_mode == DRV_CIPHER_CTR &&
1342 !IS_ALIGNED(cipherlen, sizeof(u32)))
1343 areq_ctx->is_single_pass = false;
1344
1345 break;
1346 case S_DIN_to_DES:
1347 if (!IS_ALIGNED(cipherlen, DES_BLOCK_SIZE))
1348 goto data_size_err;
1349 if (!IS_ALIGNED(assoclen, DES_BLOCK_SIZE))
1350 areq_ctx->is_single_pass = false;
1351 break;
1352 default:
1353 dev_err(dev, "Unexpected flow mode (%d)\n", ctx->flow_mode);
1354 goto data_size_err;
1355 }
1356
1357 return 0;
1358
1359 data_size_err:
1360 return -EINVAL;
1361 }
1362
1363 static unsigned int format_ccm_a0(u8 *pa0_buff, u32 header_size)
1364 {
1365 unsigned int len = 0;
1366
1367 if (header_size == 0)
1368 return 0;
1369
1370 if (header_size < ((1UL << 16) - (1UL << 8))) {
1371 len = 2;
1372
1373 pa0_buff[0] = (header_size >> 8) & 0xFF;
1374 pa0_buff[1] = header_size & 0xFF;
1375 } else {
1376 len = 6;
1377
1378 pa0_buff[0] = 0xFF;
1379 pa0_buff[1] = 0xFE;
1380 pa0_buff[2] = (header_size >> 24) & 0xFF;
1381 pa0_buff[3] = (header_size >> 16) & 0xFF;
1382 pa0_buff[4] = (header_size >> 8) & 0xFF;
1383 pa0_buff[5] = header_size & 0xFF;
1384 }
1385
1386 return len;
1387 }
1388
1389 static int set_msg_len(u8 *block, unsigned int msglen, unsigned int csize)
1390 {
1391 __be32 data;
1392
1393 memset(block, 0, csize);
1394 block += csize;
1395
1396 if (csize >= 4)
1397 csize = 4;
1398 else if (msglen > (1 << (8 * csize)))
1399 return -EOVERFLOW;
1400
1401 data = cpu_to_be32(msglen);
1402 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
1403
1404 return 0;
1405 }
1406
1407 static int cc_ccm(struct aead_request *req, struct cc_hw_desc desc[],
1408 unsigned int *seq_size)
1409 {
1410 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1411 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1412 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1413 unsigned int idx = *seq_size;
1414 unsigned int cipher_flow_mode;
1415 dma_addr_t mac_result;
1416
1417 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1418 cipher_flow_mode = AES_to_HASH_and_DOUT;
1419 mac_result = req_ctx->mac_buf_dma_addr;
1420 } else { /* Encrypt */
1421 cipher_flow_mode = AES_and_HASH;
1422 mac_result = req_ctx->icv_dma_addr;
1423 }
1424
1425 /* load key */
1426 hw_desc_init(&desc[idx]);
1427 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1428 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1429 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1430 ctx->enc_keylen), NS_BIT);
1431 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1432 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1433 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1434 set_flow_mode(&desc[idx], S_DIN_to_AES);
1435 idx++;
1436
1437 /* load ctr state */
1438 hw_desc_init(&desc[idx]);
1439 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1440 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1441 set_din_type(&desc[idx], DMA_DLLI,
1442 req_ctx->gen_ctx.iv_dma_addr, AES_BLOCK_SIZE, NS_BIT);
1443 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1444 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1445 set_flow_mode(&desc[idx], S_DIN_to_AES);
1446 idx++;
1447
1448 /* load MAC key */
1449 hw_desc_init(&desc[idx]);
1450 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1451 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1452 ((ctx->enc_keylen == 24) ? CC_AES_KEY_SIZE_MAX :
1453 ctx->enc_keylen), NS_BIT);
1454 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1455 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1456 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1457 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1458 set_aes_not_hash_mode(&desc[idx]);
1459 idx++;
1460
1461 /* load MAC state */
1462 hw_desc_init(&desc[idx]);
1463 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1464 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1465 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1466 AES_BLOCK_SIZE, NS_BIT);
1467 set_cipher_config0(&desc[idx], DESC_DIRECTION_ENCRYPT_ENCRYPT);
1468 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1469 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1470 set_aes_not_hash_mode(&desc[idx]);
1471 idx++;
1472
1473 /* process assoc data */
1474 if (req_ctx->assoclen > 0) {
1475 cc_set_assoc_desc(req, DIN_HASH, desc, &idx);
1476 } else {
1477 hw_desc_init(&desc[idx]);
1478 set_din_type(&desc[idx], DMA_DLLI,
1479 sg_dma_address(&req_ctx->ccm_adata_sg),
1480 AES_BLOCK_SIZE + req_ctx->ccm_hdr_size, NS_BIT);
1481 set_flow_mode(&desc[idx], DIN_HASH);
1482 idx++;
1483 }
1484
1485 /* process the cipher */
1486 if (req_ctx->cryptlen)
1487 cc_proc_cipher_desc(req, cipher_flow_mode, desc, &idx);
1488
1489 /* Read temporal MAC */
1490 hw_desc_init(&desc[idx]);
1491 set_cipher_mode(&desc[idx], DRV_CIPHER_CBC_MAC);
1492 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, ctx->authsize,
1493 NS_BIT, 0);
1494 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1495 set_cipher_config0(&desc[idx], HASH_DIGEST_RESULT_LITTLE_ENDIAN);
1496 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1497 set_aes_not_hash_mode(&desc[idx]);
1498 idx++;
1499
1500 /* load AES-CTR state (for last MAC calculation)*/
1501 hw_desc_init(&desc[idx]);
1502 set_cipher_mode(&desc[idx], DRV_CIPHER_CTR);
1503 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1504 set_din_type(&desc[idx], DMA_DLLI, req_ctx->ccm_iv0_dma_addr,
1505 AES_BLOCK_SIZE, NS_BIT);
1506 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1507 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1508 set_flow_mode(&desc[idx], S_DIN_to_AES);
1509 idx++;
1510
1511 hw_desc_init(&desc[idx]);
1512 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1513 set_dout_no_dma(&desc[idx], 0, 0, 1);
1514 idx++;
1515
1516 /* encrypt the "T" value and store MAC in mac_state */
1517 hw_desc_init(&desc[idx]);
1518 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1519 ctx->authsize, NS_BIT);
1520 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1521 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1522 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1523 idx++;
1524
1525 *seq_size = idx;
1526 return 0;
1527 }
1528
1529 static int config_ccm_adata(struct aead_request *req)
1530 {
1531 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1532 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1533 struct device *dev = drvdata_to_dev(ctx->drvdata);
1534 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1535 //unsigned int size_of_a = 0, rem_a_size = 0;
1536 unsigned int lp = req->iv[0];
1537 /* Note: The code assume that req->iv[0] already contains the value
1538 * of L' of RFC3610
1539 */
1540 unsigned int l = lp + 1; /* This is L' of RFC 3610. */
1541 unsigned int m = ctx->authsize; /* This is M' of RFC 3610. */
1542 u8 *b0 = req_ctx->ccm_config + CCM_B0_OFFSET;
1543 u8 *a0 = req_ctx->ccm_config + CCM_A0_OFFSET;
1544 u8 *ctr_count_0 = req_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET;
1545 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1546 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1547 req->cryptlen :
1548 (req->cryptlen - ctx->authsize);
1549 int rc;
1550
1551 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1552 memset(req_ctx->ccm_config, 0, AES_BLOCK_SIZE * 3);
1553
1554 /* taken from crypto/ccm.c */
1555 /* 2 <= L <= 8, so 1 <= L' <= 7. */
1556 if (l < 2 || l > 8) {
1557 dev_err(dev, "illegal iv value %X\n", req->iv[0]);
1558 return -EINVAL;
1559 }
1560 memcpy(b0, req->iv, AES_BLOCK_SIZE);
1561
1562 /* format control info per RFC 3610 and
1563 * NIST Special Publication 800-38C
1564 */
1565 *b0 |= (8 * ((m - 2) / 2));
1566 if (req_ctx->assoclen > 0)
1567 *b0 |= 64; /* Enable bit 6 if Adata exists. */
1568
1569 rc = set_msg_len(b0 + 16 - l, cryptlen, l); /* Write L'. */
1570 if (rc) {
1571 dev_err(dev, "message len overflow detected");
1572 return rc;
1573 }
1574 /* END of "taken from crypto/ccm.c" */
1575
1576 /* l(a) - size of associated data. */
1577 req_ctx->ccm_hdr_size = format_ccm_a0(a0, req_ctx->assoclen);
1578
1579 memset(req->iv + 15 - req->iv[0], 0, req->iv[0] + 1);
1580 req->iv[15] = 1;
1581
1582 memcpy(ctr_count_0, req->iv, AES_BLOCK_SIZE);
1583 ctr_count_0[15] = 0;
1584
1585 return 0;
1586 }
1587
1588 static void cc_proc_rfc4309_ccm(struct aead_request *req)
1589 {
1590 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1591 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1592 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1593
1594 /* L' */
1595 memset(areq_ctx->ctr_iv, 0, AES_BLOCK_SIZE);
1596 /* For RFC 4309, always use 4 bytes for message length
1597 * (at most 2^32-1 bytes).
1598 */
1599 areq_ctx->ctr_iv[0] = 3;
1600
1601 /* In RFC 4309 there is an 11-bytes nonce+IV part,
1602 * that we build here.
1603 */
1604 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_NONCE_OFFSET, ctx->ctr_nonce,
1605 CCM_BLOCK_NONCE_SIZE);
1606 memcpy(areq_ctx->ctr_iv + CCM_BLOCK_IV_OFFSET, req->iv,
1607 CCM_BLOCK_IV_SIZE);
1608 req->iv = areq_ctx->ctr_iv;
1609 areq_ctx->assoclen -= CCM_BLOCK_IV_SIZE;
1610 }
1611
1612 static void cc_set_ghash_desc(struct aead_request *req,
1613 struct cc_hw_desc desc[], unsigned int *seq_size)
1614 {
1615 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1616 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1617 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1618 unsigned int idx = *seq_size;
1619
1620 /* load key to AES*/
1621 hw_desc_init(&desc[idx]);
1622 set_cipher_mode(&desc[idx], DRV_CIPHER_ECB);
1623 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1624 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1625 ctx->enc_keylen, NS_BIT);
1626 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1627 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1628 set_flow_mode(&desc[idx], S_DIN_to_AES);
1629 idx++;
1630
1631 /* process one zero block to generate hkey */
1632 hw_desc_init(&desc[idx]);
1633 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1634 set_dout_dlli(&desc[idx], req_ctx->hkey_dma_addr, AES_BLOCK_SIZE,
1635 NS_BIT, 0);
1636 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1637 idx++;
1638
1639 /* Memory Barrier */
1640 hw_desc_init(&desc[idx]);
1641 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1642 set_dout_no_dma(&desc[idx], 0, 0, 1);
1643 idx++;
1644
1645 /* Load GHASH subkey */
1646 hw_desc_init(&desc[idx]);
1647 set_din_type(&desc[idx], DMA_DLLI, req_ctx->hkey_dma_addr,
1648 AES_BLOCK_SIZE, NS_BIT);
1649 set_dout_no_dma(&desc[idx], 0, 0, 1);
1650 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1651 set_aes_not_hash_mode(&desc[idx]);
1652 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1653 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1654 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1655 idx++;
1656
1657 /* Configure Hash Engine to work with GHASH.
1658 * Since it was not possible to extend HASH submodes to add GHASH,
1659 * The following command is necessary in order to
1660 * select GHASH (according to HW designers)
1661 */
1662 hw_desc_init(&desc[idx]);
1663 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1664 set_dout_no_dma(&desc[idx], 0, 0, 1);
1665 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1666 set_aes_not_hash_mode(&desc[idx]);
1667 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1668 set_cipher_do(&desc[idx], 1); //1=AES_SK RKEK
1669 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1670 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1671 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1672 idx++;
1673
1674 /* Load GHASH initial STATE (which is 0). (for any hash there is an
1675 * initial state)
1676 */
1677 hw_desc_init(&desc[idx]);
1678 set_din_const(&desc[idx], 0x0, AES_BLOCK_SIZE);
1679 set_dout_no_dma(&desc[idx], 0, 0, 1);
1680 set_flow_mode(&desc[idx], S_DIN_to_HASH);
1681 set_aes_not_hash_mode(&desc[idx]);
1682 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1683 set_cipher_config1(&desc[idx], HASH_PADDING_ENABLED);
1684 set_setup_mode(&desc[idx], SETUP_LOAD_STATE0);
1685 idx++;
1686
1687 *seq_size = idx;
1688 }
1689
1690 static void cc_set_gctr_desc(struct aead_request *req, struct cc_hw_desc desc[],
1691 unsigned int *seq_size)
1692 {
1693 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1694 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1695 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1696 unsigned int idx = *seq_size;
1697
1698 /* load key to AES*/
1699 hw_desc_init(&desc[idx]);
1700 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1701 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1702 set_din_type(&desc[idx], DMA_DLLI, ctx->enckey_dma_addr,
1703 ctx->enc_keylen, NS_BIT);
1704 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1705 set_setup_mode(&desc[idx], SETUP_LOAD_KEY0);
1706 set_flow_mode(&desc[idx], S_DIN_to_AES);
1707 idx++;
1708
1709 if (req_ctx->cryptlen && !req_ctx->plaintext_authenticate_only) {
1710 /* load AES/CTR initial CTR value inc by 2*/
1711 hw_desc_init(&desc[idx]);
1712 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1713 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1714 set_din_type(&desc[idx], DMA_DLLI,
1715 req_ctx->gcm_iv_inc2_dma_addr, AES_BLOCK_SIZE,
1716 NS_BIT);
1717 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1718 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1719 set_flow_mode(&desc[idx], S_DIN_to_AES);
1720 idx++;
1721 }
1722
1723 *seq_size = idx;
1724 }
1725
1726 static void cc_proc_gcm_result(struct aead_request *req,
1727 struct cc_hw_desc desc[],
1728 unsigned int *seq_size)
1729 {
1730 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1731 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1732 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1733 dma_addr_t mac_result;
1734 unsigned int idx = *seq_size;
1735
1736 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1737 mac_result = req_ctx->mac_buf_dma_addr;
1738 } else { /* Encrypt */
1739 mac_result = req_ctx->icv_dma_addr;
1740 }
1741
1742 /* process(ghash) gcm_block_len */
1743 hw_desc_init(&desc[idx]);
1744 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_block_len_dma_addr,
1745 AES_BLOCK_SIZE, NS_BIT);
1746 set_flow_mode(&desc[idx], DIN_HASH);
1747 idx++;
1748
1749 /* Store GHASH state after GHASH(Associated Data + Cipher +LenBlock) */
1750 hw_desc_init(&desc[idx]);
1751 set_cipher_mode(&desc[idx], DRV_HASH_HW_GHASH);
1752 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1753 set_dout_dlli(&desc[idx], req_ctx->mac_buf_dma_addr, AES_BLOCK_SIZE,
1754 NS_BIT, 0);
1755 set_setup_mode(&desc[idx], SETUP_WRITE_STATE0);
1756 set_flow_mode(&desc[idx], S_HASH_to_DOUT);
1757 set_aes_not_hash_mode(&desc[idx]);
1758
1759 idx++;
1760
1761 /* load AES/CTR initial CTR value inc by 1*/
1762 hw_desc_init(&desc[idx]);
1763 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1764 set_key_size_aes(&desc[idx], ctx->enc_keylen);
1765 set_din_type(&desc[idx], DMA_DLLI, req_ctx->gcm_iv_inc1_dma_addr,
1766 AES_BLOCK_SIZE, NS_BIT);
1767 set_cipher_config0(&desc[idx], DRV_CRYPTO_DIRECTION_ENCRYPT);
1768 set_setup_mode(&desc[idx], SETUP_LOAD_STATE1);
1769 set_flow_mode(&desc[idx], S_DIN_to_AES);
1770 idx++;
1771
1772 /* Memory Barrier */
1773 hw_desc_init(&desc[idx]);
1774 set_din_no_dma(&desc[idx], 0, 0xfffff0);
1775 set_dout_no_dma(&desc[idx], 0, 0, 1);
1776 idx++;
1777
1778 /* process GCTR on stored GHASH and store MAC in mac_state*/
1779 hw_desc_init(&desc[idx]);
1780 set_cipher_mode(&desc[idx], DRV_CIPHER_GCTR);
1781 set_din_type(&desc[idx], DMA_DLLI, req_ctx->mac_buf_dma_addr,
1782 AES_BLOCK_SIZE, NS_BIT);
1783 set_dout_dlli(&desc[idx], mac_result, ctx->authsize, NS_BIT, 1);
1784 set_queue_last_ind(ctx->drvdata, &desc[idx]);
1785 set_flow_mode(&desc[idx], DIN_AES_DOUT);
1786 idx++;
1787
1788 *seq_size = idx;
1789 }
1790
1791 static int cc_gcm(struct aead_request *req, struct cc_hw_desc desc[],
1792 unsigned int *seq_size)
1793 {
1794 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1795 unsigned int cipher_flow_mode;
1796
1797 if (req_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT) {
1798 cipher_flow_mode = AES_and_HASH;
1799 } else { /* Encrypt */
1800 cipher_flow_mode = AES_to_HASH_and_DOUT;
1801 }
1802
1803 //in RFC4543 no data to encrypt. just copy data from src to dest.
1804 if (req_ctx->plaintext_authenticate_only) {
1805 cc_proc_cipher_desc(req, BYPASS, desc, seq_size);
1806 cc_set_ghash_desc(req, desc, seq_size);
1807 /* process(ghash) assoc data */
1808 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1809 cc_set_gctr_desc(req, desc, seq_size);
1810 cc_proc_gcm_result(req, desc, seq_size);
1811 return 0;
1812 }
1813
1814 // for gcm and rfc4106.
1815 cc_set_ghash_desc(req, desc, seq_size);
1816 /* process(ghash) assoc data */
1817 if (req_ctx->assoclen > 0)
1818 cc_set_assoc_desc(req, DIN_HASH, desc, seq_size);
1819 cc_set_gctr_desc(req, desc, seq_size);
1820 /* process(gctr+ghash) */
1821 if (req_ctx->cryptlen)
1822 cc_proc_cipher_desc(req, cipher_flow_mode, desc, seq_size);
1823 cc_proc_gcm_result(req, desc, seq_size);
1824
1825 return 0;
1826 }
1827
1828 static int config_gcm_context(struct aead_request *req)
1829 {
1830 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1831 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1832 struct aead_req_ctx *req_ctx = aead_request_ctx(req);
1833 struct device *dev = drvdata_to_dev(ctx->drvdata);
1834
1835 unsigned int cryptlen = (req_ctx->gen_ctx.op_type ==
1836 DRV_CRYPTO_DIRECTION_ENCRYPT) ?
1837 req->cryptlen :
1838 (req->cryptlen - ctx->authsize);
1839 __be32 counter = cpu_to_be32(2);
1840
1841 dev_dbg(dev, "%s() cryptlen = %d, req_ctx->assoclen = %d ctx->authsize = %d\n",
1842 __func__, cryptlen, req_ctx->assoclen, ctx->authsize);
1843
1844 memset(req_ctx->hkey, 0, AES_BLOCK_SIZE);
1845
1846 memset(req_ctx->mac_buf, 0, AES_BLOCK_SIZE);
1847
1848 memcpy(req->iv + 12, &counter, 4);
1849 memcpy(req_ctx->gcm_iv_inc2, req->iv, 16);
1850
1851 counter = cpu_to_be32(1);
1852 memcpy(req->iv + 12, &counter, 4);
1853 memcpy(req_ctx->gcm_iv_inc1, req->iv, 16);
1854
1855 if (!req_ctx->plaintext_authenticate_only) {
1856 __be64 temp64;
1857
1858 temp64 = cpu_to_be64(req_ctx->assoclen * 8);
1859 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1860 temp64 = cpu_to_be64(cryptlen * 8);
1861 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1862 } else {
1863 /* rfc4543=> all data(AAD,IV,Plain) are considered additional
1864 * data that is nothing is encrypted.
1865 */
1866 __be64 temp64;
1867
1868 temp64 = cpu_to_be64((req_ctx->assoclen +
1869 GCM_BLOCK_RFC4_IV_SIZE + cryptlen) * 8);
1870 memcpy(&req_ctx->gcm_len_block.len_a, &temp64, sizeof(temp64));
1871 temp64 = 0;
1872 memcpy(&req_ctx->gcm_len_block.len_c, &temp64, 8);
1873 }
1874
1875 return 0;
1876 }
1877
1878 static void cc_proc_rfc4_gcm(struct aead_request *req)
1879 {
1880 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1881 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1882 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1883
1884 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_NONCE_OFFSET,
1885 ctx->ctr_nonce, GCM_BLOCK_RFC4_NONCE_SIZE);
1886 memcpy(areq_ctx->ctr_iv + GCM_BLOCK_RFC4_IV_OFFSET, req->iv,
1887 GCM_BLOCK_RFC4_IV_SIZE);
1888 req->iv = areq_ctx->ctr_iv;
1889 areq_ctx->assoclen -= GCM_BLOCK_RFC4_IV_SIZE;
1890 }
1891
1892 static int cc_proc_aead(struct aead_request *req,
1893 enum drv_crypto_direction direct)
1894 {
1895 int rc = 0;
1896 int seq_len = 0;
1897 struct cc_hw_desc desc[MAX_AEAD_PROCESS_SEQ];
1898 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1899 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
1900 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
1901 struct device *dev = drvdata_to_dev(ctx->drvdata);
1902 struct cc_crypto_req cc_req = {};
1903
1904 dev_dbg(dev, "%s context=%p req=%p iv=%p src=%p src_ofs=%d dst=%p dst_ofs=%d cryptolen=%d\n",
1905 ((direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? "Enc" : "Dec"),
1906 ctx, req, req->iv, sg_virt(req->src), req->src->offset,
1907 sg_virt(req->dst), req->dst->offset, req->cryptlen);
1908
1909 /* STAT_PHASE_0: Init and sanity checks */
1910
1911 /* Check data length according to mode */
1912 if (validate_data_size(ctx, direct, req)) {
1913 dev_err(dev, "Unsupported crypt/assoc len %d/%d.\n",
1914 req->cryptlen, areq_ctx->assoclen);
1915 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_BLOCK_LEN);
1916 return -EINVAL;
1917 }
1918
1919 /* Setup request structure */
1920 cc_req.user_cb = (void *)cc_aead_complete;
1921 cc_req.user_arg = (void *)req;
1922
1923 /* Setup request context */
1924 areq_ctx->gen_ctx.op_type = direct;
1925 areq_ctx->req_authsize = ctx->authsize;
1926 areq_ctx->cipher_mode = ctx->cipher_mode;
1927
1928 /* STAT_PHASE_1: Map buffers */
1929
1930 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1931 /* Build CTR IV - Copy nonce from last 4 bytes in
1932 * CTR key to first 4 bytes in CTR IV
1933 */
1934 memcpy(areq_ctx->ctr_iv, ctx->ctr_nonce,
1935 CTR_RFC3686_NONCE_SIZE);
1936 if (!areq_ctx->backup_giv) /*User none-generated IV*/
1937 memcpy(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE,
1938 req->iv, CTR_RFC3686_IV_SIZE);
1939 /* Initialize counter portion of counter block */
1940 *(__be32 *)(areq_ctx->ctr_iv + CTR_RFC3686_NONCE_SIZE +
1941 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1942
1943 /* Replace with counter iv */
1944 req->iv = areq_ctx->ctr_iv;
1945 areq_ctx->hw_iv_size = CTR_RFC3686_BLOCK_SIZE;
1946 } else if ((ctx->cipher_mode == DRV_CIPHER_CCM) ||
1947 (ctx->cipher_mode == DRV_CIPHER_GCTR)) {
1948 areq_ctx->hw_iv_size = AES_BLOCK_SIZE;
1949 if (areq_ctx->ctr_iv != req->iv) {
1950 memcpy(areq_ctx->ctr_iv, req->iv,
1951 crypto_aead_ivsize(tfm));
1952 req->iv = areq_ctx->ctr_iv;
1953 }
1954 } else {
1955 areq_ctx->hw_iv_size = crypto_aead_ivsize(tfm);
1956 }
1957
1958 if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1959 rc = config_ccm_adata(req);
1960 if (rc) {
1961 dev_dbg(dev, "config_ccm_adata() returned with a failure %d!",
1962 rc);
1963 goto exit;
1964 }
1965 } else {
1966 areq_ctx->ccm_hdr_size = ccm_header_size_null;
1967 }
1968
1969 if (ctx->cipher_mode == DRV_CIPHER_GCTR) {
1970 rc = config_gcm_context(req);
1971 if (rc) {
1972 dev_dbg(dev, "config_gcm_context() returned with a failure %d!",
1973 rc);
1974 goto exit;
1975 }
1976 }
1977
1978 rc = cc_map_aead_request(ctx->drvdata, req);
1979 if (rc) {
1980 dev_err(dev, "map_request() failed\n");
1981 goto exit;
1982 }
1983
1984 /* do we need to generate IV? */
1985 if (areq_ctx->backup_giv) {
1986 /* set the DMA mapped IV address*/
1987 if (ctx->cipher_mode == DRV_CIPHER_CTR) {
1988 cc_req.ivgen_dma_addr[0] =
1989 areq_ctx->gen_ctx.iv_dma_addr +
1990 CTR_RFC3686_NONCE_SIZE;
1991 cc_req.ivgen_dma_addr_len = 1;
1992 } else if (ctx->cipher_mode == DRV_CIPHER_CCM) {
1993 /* In ccm, the IV needs to exist both inside B0 and
1994 * inside the counter.It is also copied to iv_dma_addr
1995 * for other reasons (like returning it to the user).
1996 * So, using 3 (identical) IV outputs.
1997 */
1998 cc_req.ivgen_dma_addr[0] =
1999 areq_ctx->gen_ctx.iv_dma_addr +
2000 CCM_BLOCK_IV_OFFSET;
2001 cc_req.ivgen_dma_addr[1] =
2002 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2003 CCM_B0_OFFSET + CCM_BLOCK_IV_OFFSET;
2004 cc_req.ivgen_dma_addr[2] =
2005 sg_dma_address(&areq_ctx->ccm_adata_sg) +
2006 CCM_CTR_COUNT_0_OFFSET + CCM_BLOCK_IV_OFFSET;
2007 cc_req.ivgen_dma_addr_len = 3;
2008 } else {
2009 cc_req.ivgen_dma_addr[0] =
2010 areq_ctx->gen_ctx.iv_dma_addr;
2011 cc_req.ivgen_dma_addr_len = 1;
2012 }
2013
2014 /* set the IV size (8/16 B long)*/
2015 cc_req.ivgen_size = crypto_aead_ivsize(tfm);
2016 }
2017
2018 /* STAT_PHASE_2: Create sequence */
2019
2020 /* Load MLLI tables to SRAM if necessary */
2021 cc_mlli_to_sram(req, desc, &seq_len);
2022
2023 /*TODO: move seq len by reference */
2024 switch (ctx->auth_mode) {
2025 case DRV_HASH_SHA1:
2026 case DRV_HASH_SHA256:
2027 cc_hmac_authenc(req, desc, &seq_len);
2028 break;
2029 case DRV_HASH_XCBC_MAC:
2030 cc_xcbc_authenc(req, desc, &seq_len);
2031 break;
2032 case DRV_HASH_NULL:
2033 if (ctx->cipher_mode == DRV_CIPHER_CCM)
2034 cc_ccm(req, desc, &seq_len);
2035 if (ctx->cipher_mode == DRV_CIPHER_GCTR)
2036 cc_gcm(req, desc, &seq_len);
2037 break;
2038 default:
2039 dev_err(dev, "Unsupported authenc (%d)\n", ctx->auth_mode);
2040 cc_unmap_aead_request(dev, req);
2041 rc = -ENOTSUPP;
2042 goto exit;
2043 }
2044
2045 /* STAT_PHASE_3: Lock HW and push sequence */
2046
2047 rc = cc_send_request(ctx->drvdata, &cc_req, desc, seq_len, &req->base);
2048
2049 if (rc != -EINPROGRESS && rc != -EBUSY) {
2050 dev_err(dev, "send_request() failed (rc=%d)\n", rc);
2051 cc_unmap_aead_request(dev, req);
2052 }
2053
2054 exit:
2055 return rc;
2056 }
2057
2058 static int cc_aead_encrypt(struct aead_request *req)
2059 {
2060 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2061 int rc;
2062
2063 memset(areq_ctx, 0, sizeof(*areq_ctx));
2064
2065 /* No generated IV required */
2066 areq_ctx->backup_iv = req->iv;
2067 areq_ctx->assoclen = req->assoclen;
2068 areq_ctx->backup_giv = NULL;
2069 areq_ctx->is_gcm4543 = false;
2070
2071 areq_ctx->plaintext_authenticate_only = false;
2072
2073 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2074 if (rc != -EINPROGRESS && rc != -EBUSY)
2075 req->iv = areq_ctx->backup_iv;
2076
2077 return rc;
2078 }
2079
2080 static int cc_rfc4309_ccm_encrypt(struct aead_request *req)
2081 {
2082 /* Very similar to cc_aead_encrypt() above. */
2083
2084 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2085 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2086 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2087 struct device *dev = drvdata_to_dev(ctx->drvdata);
2088 int rc = -EINVAL;
2089
2090 if (!valid_assoclen(req)) {
2091 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2092 goto out;
2093 }
2094
2095 memset(areq_ctx, 0, sizeof(*areq_ctx));
2096
2097 /* No generated IV required */
2098 areq_ctx->backup_iv = req->iv;
2099 areq_ctx->assoclen = req->assoclen;
2100 areq_ctx->backup_giv = NULL;
2101 areq_ctx->is_gcm4543 = true;
2102
2103 cc_proc_rfc4309_ccm(req);
2104
2105 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2106 if (rc != -EINPROGRESS && rc != -EBUSY)
2107 req->iv = areq_ctx->backup_iv;
2108 out:
2109 return rc;
2110 }
2111
2112 static int cc_aead_decrypt(struct aead_request *req)
2113 {
2114 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2115 int rc;
2116
2117 memset(areq_ctx, 0, sizeof(*areq_ctx));
2118
2119 /* No generated IV required */
2120 areq_ctx->backup_iv = req->iv;
2121 areq_ctx->assoclen = req->assoclen;
2122 areq_ctx->backup_giv = NULL;
2123 areq_ctx->is_gcm4543 = false;
2124
2125 areq_ctx->plaintext_authenticate_only = false;
2126
2127 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2128 if (rc != -EINPROGRESS && rc != -EBUSY)
2129 req->iv = areq_ctx->backup_iv;
2130
2131 return rc;
2132 }
2133
2134 static int cc_rfc4309_ccm_decrypt(struct aead_request *req)
2135 {
2136 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2137 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2138 struct device *dev = drvdata_to_dev(ctx->drvdata);
2139 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2140 int rc = -EINVAL;
2141
2142 if (!valid_assoclen(req)) {
2143 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2144 goto out;
2145 }
2146
2147 memset(areq_ctx, 0, sizeof(*areq_ctx));
2148
2149 /* No generated IV required */
2150 areq_ctx->backup_iv = req->iv;
2151 areq_ctx->assoclen = req->assoclen;
2152 areq_ctx->backup_giv = NULL;
2153
2154 areq_ctx->is_gcm4543 = true;
2155 cc_proc_rfc4309_ccm(req);
2156
2157 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2158 if (rc != -EINPROGRESS && rc != -EBUSY)
2159 req->iv = areq_ctx->backup_iv;
2160
2161 out:
2162 return rc;
2163 }
2164
2165 static int cc_rfc4106_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2166 unsigned int keylen)
2167 {
2168 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2169 struct device *dev = drvdata_to_dev(ctx->drvdata);
2170
2171 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2172
2173 if (keylen < 4)
2174 return -EINVAL;
2175
2176 keylen -= 4;
2177 memcpy(ctx->ctr_nonce, key + keylen, 4);
2178
2179 return cc_aead_setkey(tfm, key, keylen);
2180 }
2181
2182 static int cc_rfc4543_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
2183 unsigned int keylen)
2184 {
2185 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2186 struct device *dev = drvdata_to_dev(ctx->drvdata);
2187
2188 dev_dbg(dev, "%s() keylen %d, key %p\n", __func__, keylen, key);
2189
2190 if (keylen < 4)
2191 return -EINVAL;
2192
2193 keylen -= 4;
2194 memcpy(ctx->ctr_nonce, key + keylen, 4);
2195
2196 return cc_aead_setkey(tfm, key, keylen);
2197 }
2198
2199 static int cc_gcm_setauthsize(struct crypto_aead *authenc,
2200 unsigned int authsize)
2201 {
2202 switch (authsize) {
2203 case 4:
2204 case 8:
2205 case 12:
2206 case 13:
2207 case 14:
2208 case 15:
2209 case 16:
2210 break;
2211 default:
2212 return -EINVAL;
2213 }
2214
2215 return cc_aead_setauthsize(authenc, authsize);
2216 }
2217
2218 static int cc_rfc4106_gcm_setauthsize(struct crypto_aead *authenc,
2219 unsigned int authsize)
2220 {
2221 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2222 struct device *dev = drvdata_to_dev(ctx->drvdata);
2223
2224 dev_dbg(dev, "authsize %d\n", authsize);
2225
2226 switch (authsize) {
2227 case 8:
2228 case 12:
2229 case 16:
2230 break;
2231 default:
2232 return -EINVAL;
2233 }
2234
2235 return cc_aead_setauthsize(authenc, authsize);
2236 }
2237
2238 static int cc_rfc4543_gcm_setauthsize(struct crypto_aead *authenc,
2239 unsigned int authsize)
2240 {
2241 struct cc_aead_ctx *ctx = crypto_aead_ctx(authenc);
2242 struct device *dev = drvdata_to_dev(ctx->drvdata);
2243
2244 dev_dbg(dev, "authsize %d\n", authsize);
2245
2246 if (authsize != 16)
2247 return -EINVAL;
2248
2249 return cc_aead_setauthsize(authenc, authsize);
2250 }
2251
2252 static int cc_rfc4106_gcm_encrypt(struct aead_request *req)
2253 {
2254 /* Very similar to cc_aead_encrypt() above. */
2255
2256 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2257 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2258 struct device *dev = drvdata_to_dev(ctx->drvdata);
2259 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2260 int rc = -EINVAL;
2261
2262 if (!valid_assoclen(req)) {
2263 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2264 goto out;
2265 }
2266
2267 memset(areq_ctx, 0, sizeof(*areq_ctx));
2268
2269 /* No generated IV required */
2270 areq_ctx->backup_iv = req->iv;
2271 areq_ctx->assoclen = req->assoclen;
2272 areq_ctx->backup_giv = NULL;
2273
2274 areq_ctx->plaintext_authenticate_only = false;
2275
2276 cc_proc_rfc4_gcm(req);
2277 areq_ctx->is_gcm4543 = true;
2278
2279 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2280 if (rc != -EINPROGRESS && rc != -EBUSY)
2281 req->iv = areq_ctx->backup_iv;
2282 out:
2283 return rc;
2284 }
2285
2286 static int cc_rfc4543_gcm_encrypt(struct aead_request *req)
2287 {
2288 /* Very similar to cc_aead_encrypt() above. */
2289
2290 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2291 int rc;
2292
2293 memset(areq_ctx, 0, sizeof(*areq_ctx));
2294
2295 //plaintext is not encryped with rfc4543
2296 areq_ctx->plaintext_authenticate_only = true;
2297
2298 /* No generated IV required */
2299 areq_ctx->backup_iv = req->iv;
2300 areq_ctx->assoclen = req->assoclen;
2301 areq_ctx->backup_giv = NULL;
2302
2303 cc_proc_rfc4_gcm(req);
2304 areq_ctx->is_gcm4543 = true;
2305
2306 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_ENCRYPT);
2307 if (rc != -EINPROGRESS && rc != -EBUSY)
2308 req->iv = areq_ctx->backup_iv;
2309
2310 return rc;
2311 }
2312
2313 static int cc_rfc4106_gcm_decrypt(struct aead_request *req)
2314 {
2315 /* Very similar to cc_aead_decrypt() above. */
2316
2317 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2318 struct cc_aead_ctx *ctx = crypto_aead_ctx(tfm);
2319 struct device *dev = drvdata_to_dev(ctx->drvdata);
2320 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2321 int rc = -EINVAL;
2322
2323 if (!valid_assoclen(req)) {
2324 dev_err(dev, "invalid Assoclen:%u\n", req->assoclen);
2325 goto out;
2326 }
2327
2328 memset(areq_ctx, 0, sizeof(*areq_ctx));
2329
2330 /* No generated IV required */
2331 areq_ctx->backup_iv = req->iv;
2332 areq_ctx->assoclen = req->assoclen;
2333 areq_ctx->backup_giv = NULL;
2334
2335 areq_ctx->plaintext_authenticate_only = false;
2336
2337 cc_proc_rfc4_gcm(req);
2338 areq_ctx->is_gcm4543 = true;
2339
2340 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2341 if (rc != -EINPROGRESS && rc != -EBUSY)
2342 req->iv = areq_ctx->backup_iv;
2343 out:
2344 return rc;
2345 }
2346
2347 static int cc_rfc4543_gcm_decrypt(struct aead_request *req)
2348 {
2349 /* Very similar to cc_aead_decrypt() above. */
2350
2351 struct aead_req_ctx *areq_ctx = aead_request_ctx(req);
2352 int rc;
2353
2354 memset(areq_ctx, 0, sizeof(*areq_ctx));
2355
2356 //plaintext is not decryped with rfc4543
2357 areq_ctx->plaintext_authenticate_only = true;
2358
2359 /* No generated IV required */
2360 areq_ctx->backup_iv = req->iv;
2361 areq_ctx->assoclen = req->assoclen;
2362 areq_ctx->backup_giv = NULL;
2363
2364 cc_proc_rfc4_gcm(req);
2365 areq_ctx->is_gcm4543 = true;
2366
2367 rc = cc_proc_aead(req, DRV_CRYPTO_DIRECTION_DECRYPT);
2368 if (rc != -EINPROGRESS && rc != -EBUSY)
2369 req->iv = areq_ctx->backup_iv;
2370
2371 return rc;
2372 }
2373
2374 /* aead alg */
2375 static struct cc_alg_template aead_algs[] = {
2376 {
2377 .name = "authenc(hmac(sha1),cbc(aes))",
2378 .driver_name = "authenc-hmac-sha1-cbc-aes-ccree",
2379 .blocksize = AES_BLOCK_SIZE,
2380 .template_aead = {
2381 .setkey = cc_aead_setkey,
2382 .setauthsize = cc_aead_setauthsize,
2383 .encrypt = cc_aead_encrypt,
2384 .decrypt = cc_aead_decrypt,
2385 .init = cc_aead_init,
2386 .exit = cc_aead_exit,
2387 .ivsize = AES_BLOCK_SIZE,
2388 .maxauthsize = SHA1_DIGEST_SIZE,
2389 },
2390 .cipher_mode = DRV_CIPHER_CBC,
2391 .flow_mode = S_DIN_to_AES,
2392 .auth_mode = DRV_HASH_SHA1,
2393 .min_hw_rev = CC_HW_REV_630,
2394 },
2395 {
2396 .name = "authenc(hmac(sha1),cbc(des3_ede))",
2397 .driver_name = "authenc-hmac-sha1-cbc-des3-ccree",
2398 .blocksize = DES3_EDE_BLOCK_SIZE,
2399 .template_aead = {
2400 .setkey = cc_aead_setkey,
2401 .setauthsize = cc_aead_setauthsize,
2402 .encrypt = cc_aead_encrypt,
2403 .decrypt = cc_aead_decrypt,
2404 .init = cc_aead_init,
2405 .exit = cc_aead_exit,
2406 .ivsize = DES3_EDE_BLOCK_SIZE,
2407 .maxauthsize = SHA1_DIGEST_SIZE,
2408 },
2409 .cipher_mode = DRV_CIPHER_CBC,
2410 .flow_mode = S_DIN_to_DES,
2411 .auth_mode = DRV_HASH_SHA1,
2412 .min_hw_rev = CC_HW_REV_630,
2413 },
2414 {
2415 .name = "authenc(hmac(sha256),cbc(aes))",
2416 .driver_name = "authenc-hmac-sha256-cbc-aes-ccree",
2417 .blocksize = AES_BLOCK_SIZE,
2418 .template_aead = {
2419 .setkey = cc_aead_setkey,
2420 .setauthsize = cc_aead_setauthsize,
2421 .encrypt = cc_aead_encrypt,
2422 .decrypt = cc_aead_decrypt,
2423 .init = cc_aead_init,
2424 .exit = cc_aead_exit,
2425 .ivsize = AES_BLOCK_SIZE,
2426 .maxauthsize = SHA256_DIGEST_SIZE,
2427 },
2428 .cipher_mode = DRV_CIPHER_CBC,
2429 .flow_mode = S_DIN_to_AES,
2430 .auth_mode = DRV_HASH_SHA256,
2431 .min_hw_rev = CC_HW_REV_630,
2432 },
2433 {
2434 .name = "authenc(hmac(sha256),cbc(des3_ede))",
2435 .driver_name = "authenc-hmac-sha256-cbc-des3-ccree",
2436 .blocksize = DES3_EDE_BLOCK_SIZE,
2437 .template_aead = {
2438 .setkey = cc_aead_setkey,
2439 .setauthsize = cc_aead_setauthsize,
2440 .encrypt = cc_aead_encrypt,
2441 .decrypt = cc_aead_decrypt,
2442 .init = cc_aead_init,
2443 .exit = cc_aead_exit,
2444 .ivsize = DES3_EDE_BLOCK_SIZE,
2445 .maxauthsize = SHA256_DIGEST_SIZE,
2446 },
2447 .cipher_mode = DRV_CIPHER_CBC,
2448 .flow_mode = S_DIN_to_DES,
2449 .auth_mode = DRV_HASH_SHA256,
2450 .min_hw_rev = CC_HW_REV_630,
2451 },
2452 {
2453 .name = "authenc(xcbc(aes),cbc(aes))",
2454 .driver_name = "authenc-xcbc-aes-cbc-aes-ccree",
2455 .blocksize = AES_BLOCK_SIZE,
2456 .template_aead = {
2457 .setkey = cc_aead_setkey,
2458 .setauthsize = cc_aead_setauthsize,
2459 .encrypt = cc_aead_encrypt,
2460 .decrypt = cc_aead_decrypt,
2461 .init = cc_aead_init,
2462 .exit = cc_aead_exit,
2463 .ivsize = AES_BLOCK_SIZE,
2464 .maxauthsize = AES_BLOCK_SIZE,
2465 },
2466 .cipher_mode = DRV_CIPHER_CBC,
2467 .flow_mode = S_DIN_to_AES,
2468 .auth_mode = DRV_HASH_XCBC_MAC,
2469 .min_hw_rev = CC_HW_REV_630,
2470 },
2471 {
2472 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
2473 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-ccree",
2474 .blocksize = 1,
2475 .template_aead = {
2476 .setkey = cc_aead_setkey,
2477 .setauthsize = cc_aead_setauthsize,
2478 .encrypt = cc_aead_encrypt,
2479 .decrypt = cc_aead_decrypt,
2480 .init = cc_aead_init,
2481 .exit = cc_aead_exit,
2482 .ivsize = CTR_RFC3686_IV_SIZE,
2483 .maxauthsize = SHA1_DIGEST_SIZE,
2484 },
2485 .cipher_mode = DRV_CIPHER_CTR,
2486 .flow_mode = S_DIN_to_AES,
2487 .auth_mode = DRV_HASH_SHA1,
2488 .min_hw_rev = CC_HW_REV_630,
2489 },
2490 {
2491 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
2492 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-ccree",
2493 .blocksize = 1,
2494 .template_aead = {
2495 .setkey = cc_aead_setkey,
2496 .setauthsize = cc_aead_setauthsize,
2497 .encrypt = cc_aead_encrypt,
2498 .decrypt = cc_aead_decrypt,
2499 .init = cc_aead_init,
2500 .exit = cc_aead_exit,
2501 .ivsize = CTR_RFC3686_IV_SIZE,
2502 .maxauthsize = SHA256_DIGEST_SIZE,
2503 },
2504 .cipher_mode = DRV_CIPHER_CTR,
2505 .flow_mode = S_DIN_to_AES,
2506 .auth_mode = DRV_HASH_SHA256,
2507 .min_hw_rev = CC_HW_REV_630,
2508 },
2509 {
2510 .name = "authenc(xcbc(aes),rfc3686(ctr(aes)))",
2511 .driver_name = "authenc-xcbc-aes-rfc3686-ctr-aes-ccree",
2512 .blocksize = 1,
2513 .template_aead = {
2514 .setkey = cc_aead_setkey,
2515 .setauthsize = cc_aead_setauthsize,
2516 .encrypt = cc_aead_encrypt,
2517 .decrypt = cc_aead_decrypt,
2518 .init = cc_aead_init,
2519 .exit = cc_aead_exit,
2520 .ivsize = CTR_RFC3686_IV_SIZE,
2521 .maxauthsize = AES_BLOCK_SIZE,
2522 },
2523 .cipher_mode = DRV_CIPHER_CTR,
2524 .flow_mode = S_DIN_to_AES,
2525 .auth_mode = DRV_HASH_XCBC_MAC,
2526 .min_hw_rev = CC_HW_REV_630,
2527 },
2528 {
2529 .name = "ccm(aes)",
2530 .driver_name = "ccm-aes-ccree",
2531 .blocksize = 1,
2532 .template_aead = {
2533 .setkey = cc_aead_setkey,
2534 .setauthsize = cc_ccm_setauthsize,
2535 .encrypt = cc_aead_encrypt,
2536 .decrypt = cc_aead_decrypt,
2537 .init = cc_aead_init,
2538 .exit = cc_aead_exit,
2539 .ivsize = AES_BLOCK_SIZE,
2540 .maxauthsize = AES_BLOCK_SIZE,
2541 },
2542 .cipher_mode = DRV_CIPHER_CCM,
2543 .flow_mode = S_DIN_to_AES,
2544 .auth_mode = DRV_HASH_NULL,
2545 .min_hw_rev = CC_HW_REV_630,
2546 },
2547 {
2548 .name = "rfc4309(ccm(aes))",
2549 .driver_name = "rfc4309-ccm-aes-ccree",
2550 .blocksize = 1,
2551 .template_aead = {
2552 .setkey = cc_rfc4309_ccm_setkey,
2553 .setauthsize = cc_rfc4309_ccm_setauthsize,
2554 .encrypt = cc_rfc4309_ccm_encrypt,
2555 .decrypt = cc_rfc4309_ccm_decrypt,
2556 .init = cc_aead_init,
2557 .exit = cc_aead_exit,
2558 .ivsize = CCM_BLOCK_IV_SIZE,
2559 .maxauthsize = AES_BLOCK_SIZE,
2560 },
2561 .cipher_mode = DRV_CIPHER_CCM,
2562 .flow_mode = S_DIN_to_AES,
2563 .auth_mode = DRV_HASH_NULL,
2564 .min_hw_rev = CC_HW_REV_630,
2565 },
2566 {
2567 .name = "gcm(aes)",
2568 .driver_name = "gcm-aes-ccree",
2569 .blocksize = 1,
2570 .template_aead = {
2571 .setkey = cc_aead_setkey,
2572 .setauthsize = cc_gcm_setauthsize,
2573 .encrypt = cc_aead_encrypt,
2574 .decrypt = cc_aead_decrypt,
2575 .init = cc_aead_init,
2576 .exit = cc_aead_exit,
2577 .ivsize = 12,
2578 .maxauthsize = AES_BLOCK_SIZE,
2579 },
2580 .cipher_mode = DRV_CIPHER_GCTR,
2581 .flow_mode = S_DIN_to_AES,
2582 .auth_mode = DRV_HASH_NULL,
2583 .min_hw_rev = CC_HW_REV_630,
2584 },
2585 {
2586 .name = "rfc4106(gcm(aes))",
2587 .driver_name = "rfc4106-gcm-aes-ccree",
2588 .blocksize = 1,
2589 .template_aead = {
2590 .setkey = cc_rfc4106_gcm_setkey,
2591 .setauthsize = cc_rfc4106_gcm_setauthsize,
2592 .encrypt = cc_rfc4106_gcm_encrypt,
2593 .decrypt = cc_rfc4106_gcm_decrypt,
2594 .init = cc_aead_init,
2595 .exit = cc_aead_exit,
2596 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2597 .maxauthsize = AES_BLOCK_SIZE,
2598 },
2599 .cipher_mode = DRV_CIPHER_GCTR,
2600 .flow_mode = S_DIN_to_AES,
2601 .auth_mode = DRV_HASH_NULL,
2602 .min_hw_rev = CC_HW_REV_630,
2603 },
2604 {
2605 .name = "rfc4543(gcm(aes))",
2606 .driver_name = "rfc4543-gcm-aes-ccree",
2607 .blocksize = 1,
2608 .template_aead = {
2609 .setkey = cc_rfc4543_gcm_setkey,
2610 .setauthsize = cc_rfc4543_gcm_setauthsize,
2611 .encrypt = cc_rfc4543_gcm_encrypt,
2612 .decrypt = cc_rfc4543_gcm_decrypt,
2613 .init = cc_aead_init,
2614 .exit = cc_aead_exit,
2615 .ivsize = GCM_BLOCK_RFC4_IV_SIZE,
2616 .maxauthsize = AES_BLOCK_SIZE,
2617 },
2618 .cipher_mode = DRV_CIPHER_GCTR,
2619 .flow_mode = S_DIN_to_AES,
2620 .auth_mode = DRV_HASH_NULL,
2621 .min_hw_rev = CC_HW_REV_630,
2622 },
2623 };
2624
2625 static struct cc_crypto_alg *cc_create_aead_alg(struct cc_alg_template *tmpl,
2626 struct device *dev)
2627 {
2628 struct cc_crypto_alg *t_alg;
2629 struct aead_alg *alg;
2630
2631 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
2632 if (!t_alg)
2633 return ERR_PTR(-ENOMEM);
2634
2635 alg = &tmpl->template_aead;
2636
2637 snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name);
2638 snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
2639 tmpl->driver_name);
2640 alg->base.cra_module = THIS_MODULE;
2641 alg->base.cra_priority = CC_CRA_PRIO;
2642
2643 alg->base.cra_ctxsize = sizeof(struct cc_aead_ctx);
2644 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
2645 alg->init = cc_aead_init;
2646 alg->exit = cc_aead_exit;
2647
2648 t_alg->aead_alg = *alg;
2649
2650 t_alg->cipher_mode = tmpl->cipher_mode;
2651 t_alg->flow_mode = tmpl->flow_mode;
2652 t_alg->auth_mode = tmpl->auth_mode;
2653
2654 return t_alg;
2655 }
2656
2657 int cc_aead_free(struct cc_drvdata *drvdata)
2658 {
2659 struct cc_crypto_alg *t_alg, *n;
2660 struct cc_aead_handle *aead_handle =
2661 (struct cc_aead_handle *)drvdata->aead_handle;
2662
2663 if (aead_handle) {
2664 /* Remove registered algs */
2665 list_for_each_entry_safe(t_alg, n, &aead_handle->aead_list,
2666 entry) {
2667 crypto_unregister_aead(&t_alg->aead_alg);
2668 list_del(&t_alg->entry);
2669 kfree(t_alg);
2670 }
2671 kfree(aead_handle);
2672 drvdata->aead_handle = NULL;
2673 }
2674
2675 return 0;
2676 }
2677
2678 int cc_aead_alloc(struct cc_drvdata *drvdata)
2679 {
2680 struct cc_aead_handle *aead_handle;
2681 struct cc_crypto_alg *t_alg;
2682 int rc = -ENOMEM;
2683 int alg;
2684 struct device *dev = drvdata_to_dev(drvdata);
2685
2686 aead_handle = kmalloc(sizeof(*aead_handle), GFP_KERNEL);
2687 if (!aead_handle) {
2688 rc = -ENOMEM;
2689 goto fail0;
2690 }
2691
2692 INIT_LIST_HEAD(&aead_handle->aead_list);
2693 drvdata->aead_handle = aead_handle;
2694
2695 aead_handle->sram_workspace_addr = cc_sram_alloc(drvdata,
2696 MAX_HMAC_DIGEST_SIZE);
2697
2698 if (aead_handle->sram_workspace_addr == NULL_SRAM_ADDR) {
2699 dev_err(dev, "SRAM pool exhausted\n");
2700 rc = -ENOMEM;
2701 goto fail1;
2702 }
2703
2704 /* Linux crypto */
2705 for (alg = 0; alg < ARRAY_SIZE(aead_algs); alg++) {
2706 if (aead_algs[alg].min_hw_rev > drvdata->hw_rev)
2707 continue;
2708
2709 t_alg = cc_create_aead_alg(&aead_algs[alg], dev);
2710 if (IS_ERR(t_alg)) {
2711 rc = PTR_ERR(t_alg);
2712 dev_err(dev, "%s alg allocation failed\n",
2713 aead_algs[alg].driver_name);
2714 goto fail1;
2715 }
2716 t_alg->drvdata = drvdata;
2717 rc = crypto_register_aead(&t_alg->aead_alg);
2718 if (rc) {
2719 dev_err(dev, "%s alg registration failed\n",
2720 t_alg->aead_alg.base.cra_driver_name);
2721 goto fail2;
2722 } else {
2723 list_add_tail(&t_alg->entry, &aead_handle->aead_list);
2724 dev_dbg(dev, "Registered %s\n",
2725 t_alg->aead_alg.base.cra_driver_name);
2726 }
2727 }
2728
2729 return 0;
2730
2731 fail2:
2732 kfree(t_alg);
2733 fail1:
2734 cc_aead_free(drvdata);
2735 fail0:
2736 return rc;
2737 }
Linux with added FPC-III support