?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
sh_eth: fix EESIPR values for SH77{34|63}
[linux/fpc-iii.git] / drivers / crypto / caam / caamhash.c
blobe58639ea53b11e97bb06c91fd117ca8b1f30983a
1 /*
2 * caam - Freescale FSL CAAM support for ahash functions of crypto API
3 *
4 * Copyright 2011 Freescale Semiconductor, Inc.
5 *
6 * Based on caamalg.c crypto API driver.
7 *
8 * relationship of digest job descriptor or first job descriptor after init to
9 * shared descriptors:
10 *
11 * --------------- ---------------
12 * | JobDesc #1 |-------------------->| ShareDesc |
13 * | *(packet 1) | | (hashKey) |
14 * --------------- | (operation) |
15 * ---------------
16 *
17 * relationship of subsequent job descriptors to shared descriptors:
18 *
19 * --------------- ---------------
20 * | JobDesc #2 |-------------------->| ShareDesc |
21 * | *(packet 2) | |------------->| (hashKey) |
22 * --------------- | |-------->| (operation) |
23 * . | | | (load ctx2) |
24 * . | | ---------------
25 * --------------- | |
26 * | JobDesc #3 |------| |
27 * | *(packet 3) | |
28 * --------------- |
29 * . |
30 * . |
31 * --------------- |
32 * | JobDesc #4 |------------
33 * | *(packet 4) |
34 * ---------------
35 *
36 * The SharedDesc never changes for a connection unless rekeyed, but
37 * each packet will likely be in a different place. So all we need
38 * to know to process the packet is where the input is, where the
39 * output goes, and what context we want to process with. Context is
40 * in the SharedDesc, packet references in the JobDesc.
41 *
42 * So, a job desc looks like:
43 *
44 * ---------------------
45 * | Header |
46 * | ShareDesc Pointer |
47 * | SEQ_OUT_PTR |
48 * | (output buffer) |
49 * | (output length) |
50 * | SEQ_IN_PTR |
51 * | (input buffer) |
52 * | (input length) |
53 * ---------------------
54 */
55
56 #include "compat.h"
57
58 #include "regs.h"
59 #include "intern.h"
60 #include "desc_constr.h"
61 #include "jr.h"
62 #include "error.h"
63 #include "sg_sw_sec4.h"
64 #include "key_gen.h"
65
66 #define CAAM_CRA_PRIORITY 3000
67
68 /* max hash key is max split key size */
69 #define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2)
70
71 #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
72 #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE
73
74 /* length of descriptors text */
75 #define DESC_AHASH_BASE (3 * CAAM_CMD_SZ)
76 #define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ)
77 #define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
78 #define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
79 #define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ)
80 #define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ)
81
82 #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \
83 CAAM_MAX_HASH_KEY_SIZE)
84 #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ)
85
86 /* caam context sizes for hashes: running digest + 8 */
87 #define HASH_MSG_LEN 8
88 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE)
89
90 #ifdef DEBUG
91 /* for print_hex_dumps with line references */
92 #define debug(format, arg...) printk(format, arg)
93 #else
94 #define debug(format, arg...)
95 #endif
96
97
98 static struct list_head hash_list;
99
100 /* ahash per-session context */
101 struct caam_hash_ctx {
102 u32 sh_desc_update[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
103 u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
104 u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
105 u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned;
106 dma_addr_t sh_desc_update_dma ____cacheline_aligned;
107 dma_addr_t sh_desc_update_first_dma;
108 dma_addr_t sh_desc_fin_dma;
109 dma_addr_t sh_desc_digest_dma;
110 struct device *jrdev;
111 u8 key[CAAM_MAX_HASH_KEY_SIZE];
112 dma_addr_t key_dma;
113 int ctx_len;
114 struct alginfo adata;
115 };
116
117 /* ahash state */
118 struct caam_hash_state {
119 dma_addr_t buf_dma;
120 dma_addr_t ctx_dma;
121 u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
122 int buflen_0;
123 u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned;
124 int buflen_1;
125 u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned;
126 int (*update)(struct ahash_request *req);
127 int (*final)(struct ahash_request *req);
128 int (*finup)(struct ahash_request *req);
129 int current_buf;
130 };
131
132 struct caam_export_state {
133 u8 buf[CAAM_MAX_HASH_BLOCK_SIZE];
134 u8 caam_ctx[MAX_CTX_LEN];
135 int buflen;
136 int (*update)(struct ahash_request *req);
137 int (*final)(struct ahash_request *req);
138 int (*finup)(struct ahash_request *req);
139 };
140
141 /* Common job descriptor seq in/out ptr routines */
142
143 /* Map state->caam_ctx, and append seq_out_ptr command that points to it */
144 static inline int map_seq_out_ptr_ctx(u32 *desc, struct device *jrdev,
145 struct caam_hash_state *state,
146 int ctx_len)
147 {
148 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx,
149 ctx_len, DMA_FROM_DEVICE);
150 if (dma_mapping_error(jrdev, state->ctx_dma)) {
151 dev_err(jrdev, "unable to map ctx\n");
152 return -ENOMEM;
153 }
154
155 append_seq_out_ptr(desc, state->ctx_dma, ctx_len, 0);
156
157 return 0;
158 }
159
160 /* Map req->result, and append seq_out_ptr command that points to it */
161 static inline dma_addr_t map_seq_out_ptr_result(u32 *desc, struct device *jrdev,
162 u8 *result, int digestsize)
163 {
164 dma_addr_t dst_dma;
165
166 dst_dma = dma_map_single(jrdev, result, digestsize, DMA_FROM_DEVICE);
167 append_seq_out_ptr(desc, dst_dma, digestsize, 0);
168
169 return dst_dma;
170 }
171
172 /* Map current buffer in state and put it in link table */
173 static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev,
174 struct sec4_sg_entry *sec4_sg,
175 u8 *buf, int buflen)
176 {
177 dma_addr_t buf_dma;
178
179 buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
180 dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0);
181
182 return buf_dma;
183 }
184
185 /*
186 * Only put buffer in link table if it contains data, which is possible,
187 * since a buffer has previously been used, and needs to be unmapped,
188 */
189 static inline dma_addr_t
190 try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg,
191 u8 *buf, dma_addr_t buf_dma, int buflen,
192 int last_buflen)
193 {
194 if (buf_dma && !dma_mapping_error(jrdev, buf_dma))
195 dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE);
196 if (buflen)
197 buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen);
198 else
199 buf_dma = 0;
200
201 return buf_dma;
202 }
203
204 /* Map state->caam_ctx, and add it to link table */
205 static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev,
206 struct caam_hash_state *state, int ctx_len,
207 struct sec4_sg_entry *sec4_sg, u32 flag)
208 {
209 state->ctx_dma = dma_map_single(jrdev, state->caam_ctx, ctx_len, flag);
210 if (dma_mapping_error(jrdev, state->ctx_dma)) {
211 dev_err(jrdev, "unable to map ctx\n");
212 return -ENOMEM;
213 }
214
215 dma_to_sec4_sg_one(sec4_sg, state->ctx_dma, ctx_len, 0);
216
217 return 0;
218 }
219
220 /*
221 * For ahash update, final and finup (import_ctx = true)
222 * import context, read and write to seqout
223 * For ahash firsts and digest (import_ctx = false)
224 * read and write to seqout
225 */
226 static inline void ahash_gen_sh_desc(u32 *desc, u32 state, int digestsize,
227 struct caam_hash_ctx *ctx, bool import_ctx)
228 {
229 u32 op = ctx->adata.algtype;
230 u32 *skip_key_load;
231
232 init_sh_desc(desc, HDR_SHARE_SERIAL);
233
234 /* Append key if it has been set; ahash update excluded */
235 if ((state != OP_ALG_AS_UPDATE) && (ctx->adata.keylen)) {
236 /* Skip key loading if already shared */
237 skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
238 JUMP_COND_SHRD);
239
240 append_key_as_imm(desc, ctx->key, ctx->adata.keylen_pad,
241 ctx->adata.keylen, CLASS_2 |
242 KEY_DEST_MDHA_SPLIT | KEY_ENC);
243
244 set_jump_tgt_here(desc, skip_key_load);
245
246 op |= OP_ALG_AAI_HMAC_PRECOMP;
247 }
248
249 /* If needed, import context from software */
250 if (import_ctx)
251 append_seq_load(desc, ctx->ctx_len, LDST_CLASS_2_CCB |
252 LDST_SRCDST_BYTE_CONTEXT);
253
254 /* Class 2 operation */
255 append_operation(desc, op | state | OP_ALG_ENCRYPT);
256
257 /*
258 * Load from buf and/or src and write to req->result or state->context
259 * Calculate remaining bytes to read
260 */
261 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
262 /* Read remaining bytes */
263 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 |
264 FIFOLD_TYPE_MSG | KEY_VLF);
265 /* Store class2 context bytes */
266 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
267 LDST_SRCDST_BYTE_CONTEXT);
268 }
269
270 static int ahash_set_sh_desc(struct crypto_ahash *ahash)
271 {
272 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
273 int digestsize = crypto_ahash_digestsize(ahash);
274 struct device *jrdev = ctx->jrdev;
275 u32 *desc;
276
277 /* ahash_update shared descriptor */
278 desc = ctx->sh_desc_update;
279 ahash_gen_sh_desc(desc, OP_ALG_AS_UPDATE, ctx->ctx_len, ctx, true);
280 ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
281 DMA_TO_DEVICE);
282 if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) {
283 dev_err(jrdev, "unable to map shared descriptor\n");
284 return -ENOMEM;
285 }
286 #ifdef DEBUG
287 print_hex_dump(KERN_ERR,
288 "ahash update shdesc@"__stringify(__LINE__)": ",
289 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
290 #endif
291
292 /* ahash_update_first shared descriptor */
293 desc = ctx->sh_desc_update_first;
294 ahash_gen_sh_desc(desc, OP_ALG_AS_INIT, ctx->ctx_len, ctx, false);
295 ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc,
296 desc_bytes(desc),
297 DMA_TO_DEVICE);
298 if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) {
299 dev_err(jrdev, "unable to map shared descriptor\n");
300 return -ENOMEM;
301 }
302 #ifdef DEBUG
303 print_hex_dump(KERN_ERR,
304 "ahash update first shdesc@"__stringify(__LINE__)": ",
305 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
306 #endif
307
308 /* ahash_final shared descriptor */
309 desc = ctx->sh_desc_fin;
310 ahash_gen_sh_desc(desc, OP_ALG_AS_FINALIZE, digestsize, ctx, true);
311 ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
312 DMA_TO_DEVICE);
313 if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) {
314 dev_err(jrdev, "unable to map shared descriptor\n");
315 return -ENOMEM;
316 }
317 #ifdef DEBUG
318 print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ",
319 DUMP_PREFIX_ADDRESS, 16, 4, desc,
320 desc_bytes(desc), 1);
321 #endif
322
323 /* ahash_digest shared descriptor */
324 desc = ctx->sh_desc_digest;
325 ahash_gen_sh_desc(desc, OP_ALG_AS_INITFINAL, digestsize, ctx, false);
326 ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc,
327 desc_bytes(desc),
328 DMA_TO_DEVICE);
329 if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) {
330 dev_err(jrdev, "unable to map shared descriptor\n");
331 return -ENOMEM;
332 }
333 #ifdef DEBUG
334 print_hex_dump(KERN_ERR,
335 "ahash digest shdesc@"__stringify(__LINE__)": ",
336 DUMP_PREFIX_ADDRESS, 16, 4, desc,
337 desc_bytes(desc), 1);
338 #endif
339
340 return 0;
341 }
342
343 /* Digest hash size if it is too large */
344 static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in,
345 u32 *keylen, u8 *key_out, u32 digestsize)
346 {
347 struct device *jrdev = ctx->jrdev;
348 u32 *desc;
349 struct split_key_result result;
350 dma_addr_t src_dma, dst_dma;
351 int ret;
352
353 desc = kmalloc(CAAM_CMD_SZ * 8 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA);
354 if (!desc) {
355 dev_err(jrdev, "unable to allocate key input memory\n");
356 return -ENOMEM;
357 }
358
359 init_job_desc(desc, 0);
360
361 src_dma = dma_map_single(jrdev, (void *)key_in, *keylen,
362 DMA_TO_DEVICE);
363 if (dma_mapping_error(jrdev, src_dma)) {
364 dev_err(jrdev, "unable to map key input memory\n");
365 kfree(desc);
366 return -ENOMEM;
367 }
368 dst_dma = dma_map_single(jrdev, (void *)key_out, digestsize,
369 DMA_FROM_DEVICE);
370 if (dma_mapping_error(jrdev, dst_dma)) {
371 dev_err(jrdev, "unable to map key output memory\n");
372 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
373 kfree(desc);
374 return -ENOMEM;
375 }
376
377 /* Job descriptor to perform unkeyed hash on key_in */
378 append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT |
379 OP_ALG_AS_INITFINAL);
380 append_seq_in_ptr(desc, src_dma, *keylen, 0);
381 append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 |
382 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG);
383 append_seq_out_ptr(desc, dst_dma, digestsize, 0);
384 append_seq_store(desc, digestsize, LDST_CLASS_2_CCB |
385 LDST_SRCDST_BYTE_CONTEXT);
386
387 #ifdef DEBUG
388 print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ",
389 DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1);
390 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
391 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
392 #endif
393
394 result.err = 0;
395 init_completion(&result.completion);
396
397 ret = caam_jr_enqueue(jrdev, desc, split_key_done, &result);
398 if (!ret) {
399 /* in progress */
400 wait_for_completion_interruptible(&result.completion);
401 ret = result.err;
402 #ifdef DEBUG
403 print_hex_dump(KERN_ERR,
404 "digested key@"__stringify(__LINE__)": ",
405 DUMP_PREFIX_ADDRESS, 16, 4, key_in,
406 digestsize, 1);
407 #endif
408 }
409 dma_unmap_single(jrdev, src_dma, *keylen, DMA_TO_DEVICE);
410 dma_unmap_single(jrdev, dst_dma, digestsize, DMA_FROM_DEVICE);
411
412 *keylen = digestsize;
413
414 kfree(desc);
415
416 return ret;
417 }
418
419 static int ahash_setkey(struct crypto_ahash *ahash,
420 const u8 *key, unsigned int keylen)
421 {
422 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
423 struct device *jrdev = ctx->jrdev;
424 int blocksize = crypto_tfm_alg_blocksize(&ahash->base);
425 int digestsize = crypto_ahash_digestsize(ahash);
426 int ret;
427 u8 *hashed_key = NULL;
428
429 #ifdef DEBUG
430 printk(KERN_ERR "keylen %d\n", keylen);
431 #endif
432
433 if (keylen > blocksize) {
434 hashed_key = kmalloc_array(digestsize,
435 sizeof(*hashed_key),
436 GFP_KERNEL | GFP_DMA);
437 if (!hashed_key)
438 return -ENOMEM;
439 ret = hash_digest_key(ctx, key, &keylen, hashed_key,
440 digestsize);
441 if (ret)
442 goto bad_free_key;
443 key = hashed_key;
444 }
445
446 ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, keylen,
447 CAAM_MAX_HASH_KEY_SIZE);
448 if (ret)
449 goto bad_free_key;
450
451 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->adata.keylen_pad,
452 DMA_TO_DEVICE);
453 if (dma_mapping_error(jrdev, ctx->key_dma)) {
454 dev_err(jrdev, "unable to map key i/o memory\n");
455 ret = -ENOMEM;
456 goto error_free_key;
457 }
458 #ifdef DEBUG
459 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
460 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
461 ctx->adata.keylen_pad, 1);
462 #endif
463
464 ret = ahash_set_sh_desc(ahash);
465 if (ret) {
466 dma_unmap_single(jrdev, ctx->key_dma, ctx->adata.keylen_pad,
467 DMA_TO_DEVICE);
468 }
469
470 error_free_key:
471 kfree(hashed_key);
472 return ret;
473 bad_free_key:
474 kfree(hashed_key);
475 crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN);
476 return -EINVAL;
477 }
478
479 /*
480 * ahash_edesc - s/w-extended ahash descriptor
481 * @dst_dma: physical mapped address of req->result
482 * @sec4_sg_dma: physical mapped address of h/w link table
483 * @src_nents: number of segments in input scatterlist
484 * @sec4_sg_bytes: length of dma mapped sec4_sg space
485 * @hw_desc: the h/w job descriptor followed by any referenced link tables
486 * @sec4_sg: h/w link table
487 */
488 struct ahash_edesc {
489 dma_addr_t dst_dma;
490 dma_addr_t sec4_sg_dma;
491 int src_nents;
492 int sec4_sg_bytes;
493 u32 hw_desc[DESC_JOB_IO_LEN / sizeof(u32)] ____cacheline_aligned;
494 struct sec4_sg_entry sec4_sg[0];
495 };
496
497 static inline void ahash_unmap(struct device *dev,
498 struct ahash_edesc *edesc,
499 struct ahash_request *req, int dst_len)
500 {
501 if (edesc->src_nents)
502 dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE);
503 if (edesc->dst_dma)
504 dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE);
505
506 if (edesc->sec4_sg_bytes)
507 dma_unmap_single(dev, edesc->sec4_sg_dma,
508 edesc->sec4_sg_bytes, DMA_TO_DEVICE);
509 }
510
511 static inline void ahash_unmap_ctx(struct device *dev,
512 struct ahash_edesc *edesc,
513 struct ahash_request *req, int dst_len, u32 flag)
514 {
515 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
516 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
517 struct caam_hash_state *state = ahash_request_ctx(req);
518
519 if (state->ctx_dma)
520 dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag);
521 ahash_unmap(dev, edesc, req, dst_len);
522 }
523
524 static void ahash_done(struct device *jrdev, u32 *desc, u32 err,
525 void *context)
526 {
527 struct ahash_request *req = context;
528 struct ahash_edesc *edesc;
529 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
530 int digestsize = crypto_ahash_digestsize(ahash);
531 #ifdef DEBUG
532 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
533 struct caam_hash_state *state = ahash_request_ctx(req);
534
535 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
536 #endif
537
538 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
539 if (err)
540 caam_jr_strstatus(jrdev, err);
541
542 ahash_unmap(jrdev, edesc, req, digestsize);
543 kfree(edesc);
544
545 #ifdef DEBUG
546 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
547 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
548 ctx->ctx_len, 1);
549 if (req->result)
550 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
551 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
552 digestsize, 1);
553 #endif
554
555 req->base.complete(&req->base, err);
556 }
557
558 static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err,
559 void *context)
560 {
561 struct ahash_request *req = context;
562 struct ahash_edesc *edesc;
563 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
564 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
565 #ifdef DEBUG
566 struct caam_hash_state *state = ahash_request_ctx(req);
567 int digestsize = crypto_ahash_digestsize(ahash);
568
569 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
570 #endif
571
572 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
573 if (err)
574 caam_jr_strstatus(jrdev, err);
575
576 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
577 kfree(edesc);
578
579 #ifdef DEBUG
580 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
581 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
582 ctx->ctx_len, 1);
583 if (req->result)
584 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
585 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
586 digestsize, 1);
587 #endif
588
589 req->base.complete(&req->base, err);
590 }
591
592 static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err,
593 void *context)
594 {
595 struct ahash_request *req = context;
596 struct ahash_edesc *edesc;
597 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
598 int digestsize = crypto_ahash_digestsize(ahash);
599 #ifdef DEBUG
600 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
601 struct caam_hash_state *state = ahash_request_ctx(req);
602
603 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
604 #endif
605
606 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
607 if (err)
608 caam_jr_strstatus(jrdev, err);
609
610 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_TO_DEVICE);
611 kfree(edesc);
612
613 #ifdef DEBUG
614 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
615 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
616 ctx->ctx_len, 1);
617 if (req->result)
618 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
619 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
620 digestsize, 1);
621 #endif
622
623 req->base.complete(&req->base, err);
624 }
625
626 static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err,
627 void *context)
628 {
629 struct ahash_request *req = context;
630 struct ahash_edesc *edesc;
631 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
632 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
633 #ifdef DEBUG
634 struct caam_hash_state *state = ahash_request_ctx(req);
635 int digestsize = crypto_ahash_digestsize(ahash);
636
637 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
638 #endif
639
640 edesc = container_of(desc, struct ahash_edesc, hw_desc[0]);
641 if (err)
642 caam_jr_strstatus(jrdev, err);
643
644 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE);
645 kfree(edesc);
646
647 #ifdef DEBUG
648 print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ",
649 DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx,
650 ctx->ctx_len, 1);
651 if (req->result)
652 print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ",
653 DUMP_PREFIX_ADDRESS, 16, 4, req->result,
654 digestsize, 1);
655 #endif
656
657 req->base.complete(&req->base, err);
658 }
659
660 /*
661 * Allocate an enhanced descriptor, which contains the hardware descriptor
662 * and space for hardware scatter table containing sg_num entries.
663 */
664 static struct ahash_edesc *ahash_edesc_alloc(struct caam_hash_ctx *ctx,
665 int sg_num, u32 *sh_desc,
666 dma_addr_t sh_desc_dma,
667 gfp_t flags)
668 {
669 struct ahash_edesc *edesc;
670 unsigned int sg_size = sg_num * sizeof(struct sec4_sg_entry);
671
672 edesc = kzalloc(sizeof(*edesc) + sg_size, GFP_DMA | flags);
673 if (!edesc) {
674 dev_err(ctx->jrdev, "could not allocate extended descriptor\n");
675 return NULL;
676 }
677
678 init_job_desc_shared(edesc->hw_desc, sh_desc_dma, desc_len(sh_desc),
679 HDR_SHARE_DEFER | HDR_REVERSE);
680
681 return edesc;
682 }
683
684 static int ahash_edesc_add_src(struct caam_hash_ctx *ctx,
685 struct ahash_edesc *edesc,
686 struct ahash_request *req, int nents,
687 unsigned int first_sg,
688 unsigned int first_bytes, size_t to_hash)
689 {
690 dma_addr_t src_dma;
691 u32 options;
692
693 if (nents > 1 || first_sg) {
694 struct sec4_sg_entry *sg = edesc->sec4_sg;
695 unsigned int sgsize = sizeof(*sg) * (first_sg + nents);
696
697 sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0);
698
699 src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE);
700 if (dma_mapping_error(ctx->jrdev, src_dma)) {
701 dev_err(ctx->jrdev, "unable to map S/G table\n");
702 return -ENOMEM;
703 }
704
705 edesc->sec4_sg_bytes = sgsize;
706 edesc->sec4_sg_dma = src_dma;
707 options = LDST_SGF;
708 } else {
709 src_dma = sg_dma_address(req->src);
710 options = 0;
711 }
712
713 append_seq_in_ptr(edesc->hw_desc, src_dma, first_bytes + to_hash,
714 options);
715
716 return 0;
717 }
718
719 /* submit update job descriptor */
720 static int ahash_update_ctx(struct ahash_request *req)
721 {
722 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
723 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
724 struct caam_hash_state *state = ahash_request_ctx(req);
725 struct device *jrdev = ctx->jrdev;
726 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
727 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
728 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
729 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0;
730 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1;
731 int *next_buflen = state->current_buf ? &state->buflen_0 :
732 &state->buflen_1, last_buflen;
733 int in_len = *buflen + req->nbytes, to_hash;
734 u32 *desc;
735 int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index;
736 struct ahash_edesc *edesc;
737 int ret = 0;
738
739 last_buflen = *next_buflen;
740 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
741 to_hash = in_len - *next_buflen;
742
743 if (to_hash) {
744 src_nents = sg_nents_for_len(req->src,
745 req->nbytes - (*next_buflen));
746 if (src_nents < 0) {
747 dev_err(jrdev, "Invalid number of src SG.\n");
748 return src_nents;
749 }
750
751 if (src_nents) {
752 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
753 DMA_TO_DEVICE);
754 if (!mapped_nents) {
755 dev_err(jrdev, "unable to DMA map source\n");
756 return -ENOMEM;
757 }
758 } else {
759 mapped_nents = 0;
760 }
761
762 sec4_sg_src_index = 1 + (*buflen ? 1 : 0);
763 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
764 sizeof(struct sec4_sg_entry);
765
766 /*
767 * allocate space for base edesc and hw desc commands,
768 * link tables
769 */
770 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
771 ctx->sh_desc_update,
772 ctx->sh_desc_update_dma, flags);
773 if (!edesc) {
774 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
775 return -ENOMEM;
776 }
777
778 edesc->src_nents = src_nents;
779 edesc->sec4_sg_bytes = sec4_sg_bytes;
780
781 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
782 edesc->sec4_sg, DMA_BIDIRECTIONAL);
783 if (ret)
784 goto unmap_ctx;
785
786 state->buf_dma = try_buf_map_to_sec4_sg(jrdev,
787 edesc->sec4_sg + 1,
788 buf, state->buf_dma,
789 *buflen, last_buflen);
790
791 if (mapped_nents) {
792 sg_to_sec4_sg_last(req->src, mapped_nents,
793 edesc->sec4_sg + sec4_sg_src_index,
794 0);
795 if (*next_buflen)
796 scatterwalk_map_and_copy(next_buf, req->src,
797 to_hash - *buflen,
798 *next_buflen, 0);
799 } else {
800 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
801 cpu_to_caam32(SEC4_SG_LEN_FIN);
802 }
803
804 state->current_buf = !state->current_buf;
805
806 desc = edesc->hw_desc;
807
808 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
809 sec4_sg_bytes,
810 DMA_TO_DEVICE);
811 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
812 dev_err(jrdev, "unable to map S/G table\n");
813 ret = -ENOMEM;
814 goto unmap_ctx;
815 }
816
817 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len +
818 to_hash, LDST_SGF);
819
820 append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0);
821
822 #ifdef DEBUG
823 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
824 DUMP_PREFIX_ADDRESS, 16, 4, desc,
825 desc_bytes(desc), 1);
826 #endif
827
828 ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req);
829 if (ret)
830 goto unmap_ctx;
831
832 ret = -EINPROGRESS;
833 } else if (*next_buflen) {
834 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
835 req->nbytes, 0);
836 *buflen = *next_buflen;
837 *next_buflen = last_buflen;
838 }
839 #ifdef DEBUG
840 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
841 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
842 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
843 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
844 *next_buflen, 1);
845 #endif
846
847 return ret;
848 unmap_ctx:
849 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL);
850 kfree(edesc);
851 return ret;
852 }
853
854 static int ahash_final_ctx(struct ahash_request *req)
855 {
856 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
857 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
858 struct caam_hash_state *state = ahash_request_ctx(req);
859 struct device *jrdev = ctx->jrdev;
860 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
861 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
862 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
863 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
864 int last_buflen = state->current_buf ? state->buflen_0 :
865 state->buflen_1;
866 u32 *desc;
867 int sec4_sg_bytes, sec4_sg_src_index;
868 int digestsize = crypto_ahash_digestsize(ahash);
869 struct ahash_edesc *edesc;
870 int ret;
871
872 sec4_sg_src_index = 1 + (buflen ? 1 : 0);
873 sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry);
874
875 /* allocate space for base edesc and hw desc commands, link tables */
876 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index,
877 ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
878 flags);
879 if (!edesc)
880 return -ENOMEM;
881
882 desc = edesc->hw_desc;
883
884 edesc->sec4_sg_bytes = sec4_sg_bytes;
885 edesc->src_nents = 0;
886
887 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
888 edesc->sec4_sg, DMA_TO_DEVICE);
889 if (ret)
890 goto unmap_ctx;
891
892 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
893 buf, state->buf_dma, buflen,
894 last_buflen);
895 (edesc->sec4_sg + sec4_sg_src_index - 1)->len |=
896 cpu_to_caam32(SEC4_SG_LEN_FIN);
897
898 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
899 sec4_sg_bytes, DMA_TO_DEVICE);
900 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
901 dev_err(jrdev, "unable to map S/G table\n");
902 ret = -ENOMEM;
903 goto unmap_ctx;
904 }
905
906 append_seq_in_ptr(desc, edesc->sec4_sg_dma, ctx->ctx_len + buflen,
907 LDST_SGF);
908
909 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
910 digestsize);
911 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
912 dev_err(jrdev, "unable to map dst\n");
913 ret = -ENOMEM;
914 goto unmap_ctx;
915 }
916
917 #ifdef DEBUG
918 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
919 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
920 #endif
921
922 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
923 if (ret)
924 goto unmap_ctx;
925
926 return -EINPROGRESS;
927 unmap_ctx:
928 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
929 kfree(edesc);
930 return ret;
931 }
932
933 static int ahash_finup_ctx(struct ahash_request *req)
934 {
935 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
936 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
937 struct caam_hash_state *state = ahash_request_ctx(req);
938 struct device *jrdev = ctx->jrdev;
939 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
940 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
941 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
942 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
943 int last_buflen = state->current_buf ? state->buflen_0 :
944 state->buflen_1;
945 u32 *desc;
946 int sec4_sg_src_index;
947 int src_nents, mapped_nents;
948 int digestsize = crypto_ahash_digestsize(ahash);
949 struct ahash_edesc *edesc;
950 int ret;
951
952 src_nents = sg_nents_for_len(req->src, req->nbytes);
953 if (src_nents < 0) {
954 dev_err(jrdev, "Invalid number of src SG.\n");
955 return src_nents;
956 }
957
958 if (src_nents) {
959 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
960 DMA_TO_DEVICE);
961 if (!mapped_nents) {
962 dev_err(jrdev, "unable to DMA map source\n");
963 return -ENOMEM;
964 }
965 } else {
966 mapped_nents = 0;
967 }
968
969 sec4_sg_src_index = 1 + (buflen ? 1 : 0);
970
971 /* allocate space for base edesc and hw desc commands, link tables */
972 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
973 ctx->sh_desc_fin, ctx->sh_desc_fin_dma,
974 flags);
975 if (!edesc) {
976 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
977 return -ENOMEM;
978 }
979
980 desc = edesc->hw_desc;
981
982 edesc->src_nents = src_nents;
983
984 ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len,
985 edesc->sec4_sg, DMA_TO_DEVICE);
986 if (ret)
987 goto unmap_ctx;
988
989 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1,
990 buf, state->buf_dma, buflen,
991 last_buflen);
992
993 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents,
994 sec4_sg_src_index, ctx->ctx_len + buflen,
995 req->nbytes);
996 if (ret)
997 goto unmap_ctx;
998
999 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1000 digestsize);
1001 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1002 dev_err(jrdev, "unable to map dst\n");
1003 ret = -ENOMEM;
1004 goto unmap_ctx;
1005 }
1006
1007 #ifdef DEBUG
1008 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1009 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1010 #endif
1011
1012 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req);
1013 if (ret)
1014 goto unmap_ctx;
1015
1016 return -EINPROGRESS;
1017 unmap_ctx:
1018 ahash_unmap_ctx(jrdev, edesc, req, digestsize, DMA_FROM_DEVICE);
1019 kfree(edesc);
1020 return ret;
1021 }
1022
1023 static int ahash_digest(struct ahash_request *req)
1024 {
1025 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1026 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1027 struct device *jrdev = ctx->jrdev;
1028 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1029 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1030 u32 *desc;
1031 int digestsize = crypto_ahash_digestsize(ahash);
1032 int src_nents, mapped_nents;
1033 struct ahash_edesc *edesc;
1034 int ret;
1035
1036 src_nents = sg_nents_for_len(req->src, req->nbytes);
1037 if (src_nents < 0) {
1038 dev_err(jrdev, "Invalid number of src SG.\n");
1039 return src_nents;
1040 }
1041
1042 if (src_nents) {
1043 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1044 DMA_TO_DEVICE);
1045 if (!mapped_nents) {
1046 dev_err(jrdev, "unable to map source for DMA\n");
1047 return -ENOMEM;
1048 }
1049 } else {
1050 mapped_nents = 0;
1051 }
1052
1053 /* allocate space for base edesc and hw desc commands, link tables */
1054 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ? mapped_nents : 0,
1055 ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1056 flags);
1057 if (!edesc) {
1058 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1059 return -ENOMEM;
1060 }
1061
1062 edesc->src_nents = src_nents;
1063
1064 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1065 req->nbytes);
1066 if (ret) {
1067 ahash_unmap(jrdev, edesc, req, digestsize);
1068 kfree(edesc);
1069 return ret;
1070 }
1071
1072 desc = edesc->hw_desc;
1073
1074 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1075 digestsize);
1076 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1077 dev_err(jrdev, "unable to map dst\n");
1078 ahash_unmap(jrdev, edesc, req, digestsize);
1079 kfree(edesc);
1080 return -ENOMEM;
1081 }
1082
1083 #ifdef DEBUG
1084 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1085 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1086 #endif
1087
1088 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1089 if (!ret) {
1090 ret = -EINPROGRESS;
1091 } else {
1092 ahash_unmap(jrdev, edesc, req, digestsize);
1093 kfree(edesc);
1094 }
1095
1096 return ret;
1097 }
1098
1099 /* submit ahash final if it the first job descriptor */
1100 static int ahash_final_no_ctx(struct ahash_request *req)
1101 {
1102 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1103 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1104 struct caam_hash_state *state = ahash_request_ctx(req);
1105 struct device *jrdev = ctx->jrdev;
1106 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1107 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1108 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1109 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1110 u32 *desc;
1111 int digestsize = crypto_ahash_digestsize(ahash);
1112 struct ahash_edesc *edesc;
1113 int ret;
1114
1115 /* allocate space for base edesc and hw desc commands, link tables */
1116 edesc = ahash_edesc_alloc(ctx, 0, ctx->sh_desc_digest,
1117 ctx->sh_desc_digest_dma, flags);
1118 if (!edesc)
1119 return -ENOMEM;
1120
1121 desc = edesc->hw_desc;
1122
1123 state->buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE);
1124 if (dma_mapping_error(jrdev, state->buf_dma)) {
1125 dev_err(jrdev, "unable to map src\n");
1126 goto unmap;
1127 }
1128
1129 append_seq_in_ptr(desc, state->buf_dma, buflen, 0);
1130
1131 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1132 digestsize);
1133 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1134 dev_err(jrdev, "unable to map dst\n");
1135 goto unmap;
1136 }
1137 edesc->src_nents = 0;
1138
1139 #ifdef DEBUG
1140 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1141 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1142 #endif
1143
1144 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1145 if (!ret) {
1146 ret = -EINPROGRESS;
1147 } else {
1148 ahash_unmap(jrdev, edesc, req, digestsize);
1149 kfree(edesc);
1150 }
1151
1152 return ret;
1153 unmap:
1154 ahash_unmap(jrdev, edesc, req, digestsize);
1155 kfree(edesc);
1156 return -ENOMEM;
1157
1158 }
1159
1160 /* submit ahash update if it the first job descriptor after update */
1161 static int ahash_update_no_ctx(struct ahash_request *req)
1162 {
1163 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1164 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1165 struct caam_hash_state *state = ahash_request_ctx(req);
1166 struct device *jrdev = ctx->jrdev;
1167 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1168 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1169 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1170 int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0;
1171 u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1;
1172 int *next_buflen = state->current_buf ? &state->buflen_0 :
1173 &state->buflen_1;
1174 int in_len = *buflen + req->nbytes, to_hash;
1175 int sec4_sg_bytes, src_nents, mapped_nents;
1176 struct ahash_edesc *edesc;
1177 u32 *desc;
1178 int ret = 0;
1179
1180 *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1);
1181 to_hash = in_len - *next_buflen;
1182
1183 if (to_hash) {
1184 src_nents = sg_nents_for_len(req->src,
1185 req->nbytes - *next_buflen);
1186 if (src_nents < 0) {
1187 dev_err(jrdev, "Invalid number of src SG.\n");
1188 return src_nents;
1189 }
1190
1191 if (src_nents) {
1192 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1193 DMA_TO_DEVICE);
1194 if (!mapped_nents) {
1195 dev_err(jrdev, "unable to DMA map source\n");
1196 return -ENOMEM;
1197 }
1198 } else {
1199 mapped_nents = 0;
1200 }
1201
1202 sec4_sg_bytes = (1 + mapped_nents) *
1203 sizeof(struct sec4_sg_entry);
1204
1205 /*
1206 * allocate space for base edesc and hw desc commands,
1207 * link tables
1208 */
1209 edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents,
1210 ctx->sh_desc_update_first,
1211 ctx->sh_desc_update_first_dma,
1212 flags);
1213 if (!edesc) {
1214 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1215 return -ENOMEM;
1216 }
1217
1218 edesc->src_nents = src_nents;
1219 edesc->sec4_sg_bytes = sec4_sg_bytes;
1220 edesc->dst_dma = 0;
1221
1222 state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg,
1223 buf, *buflen);
1224 sg_to_sec4_sg_last(req->src, mapped_nents,
1225 edesc->sec4_sg + 1, 0);
1226
1227 if (*next_buflen) {
1228 scatterwalk_map_and_copy(next_buf, req->src,
1229 to_hash - *buflen,
1230 *next_buflen, 0);
1231 }
1232
1233 state->current_buf = !state->current_buf;
1234
1235 desc = edesc->hw_desc;
1236
1237 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
1238 sec4_sg_bytes,
1239 DMA_TO_DEVICE);
1240 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
1241 dev_err(jrdev, "unable to map S/G table\n");
1242 ret = -ENOMEM;
1243 goto unmap_ctx;
1244 }
1245
1246 append_seq_in_ptr(desc, edesc->sec4_sg_dma, to_hash, LDST_SGF);
1247
1248 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1249 if (ret)
1250 goto unmap_ctx;
1251
1252 #ifdef DEBUG
1253 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1254 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1255 desc_bytes(desc), 1);
1256 #endif
1257
1258 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1259 if (ret)
1260 goto unmap_ctx;
1261
1262 ret = -EINPROGRESS;
1263 state->update = ahash_update_ctx;
1264 state->finup = ahash_finup_ctx;
1265 state->final = ahash_final_ctx;
1266 } else if (*next_buflen) {
1267 scatterwalk_map_and_copy(buf + *buflen, req->src, 0,
1268 req->nbytes, 0);
1269 *buflen = *next_buflen;
1270 *next_buflen = 0;
1271 }
1272 #ifdef DEBUG
1273 print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ",
1274 DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1);
1275 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1276 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1277 *next_buflen, 1);
1278 #endif
1279
1280 return ret;
1281 unmap_ctx:
1282 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1283 kfree(edesc);
1284 return ret;
1285 }
1286
1287 /* submit ahash finup if it the first job descriptor after update */
1288 static int ahash_finup_no_ctx(struct ahash_request *req)
1289 {
1290 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1291 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1292 struct caam_hash_state *state = ahash_request_ctx(req);
1293 struct device *jrdev = ctx->jrdev;
1294 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1295 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1296 u8 *buf = state->current_buf ? state->buf_1 : state->buf_0;
1297 int buflen = state->current_buf ? state->buflen_1 : state->buflen_0;
1298 int last_buflen = state->current_buf ? state->buflen_0 :
1299 state->buflen_1;
1300 u32 *desc;
1301 int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents;
1302 int digestsize = crypto_ahash_digestsize(ahash);
1303 struct ahash_edesc *edesc;
1304 int ret;
1305
1306 src_nents = sg_nents_for_len(req->src, req->nbytes);
1307 if (src_nents < 0) {
1308 dev_err(jrdev, "Invalid number of src SG.\n");
1309 return src_nents;
1310 }
1311
1312 if (src_nents) {
1313 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1314 DMA_TO_DEVICE);
1315 if (!mapped_nents) {
1316 dev_err(jrdev, "unable to DMA map source\n");
1317 return -ENOMEM;
1318 }
1319 } else {
1320 mapped_nents = 0;
1321 }
1322
1323 sec4_sg_src_index = 2;
1324 sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) *
1325 sizeof(struct sec4_sg_entry);
1326
1327 /* allocate space for base edesc and hw desc commands, link tables */
1328 edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents,
1329 ctx->sh_desc_digest, ctx->sh_desc_digest_dma,
1330 flags);
1331 if (!edesc) {
1332 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1333 return -ENOMEM;
1334 }
1335
1336 desc = edesc->hw_desc;
1337
1338 edesc->src_nents = src_nents;
1339 edesc->sec4_sg_bytes = sec4_sg_bytes;
1340
1341 state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf,
1342 state->buf_dma, buflen,
1343 last_buflen);
1344
1345 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen,
1346 req->nbytes);
1347 if (ret) {
1348 dev_err(jrdev, "unable to map S/G table\n");
1349 goto unmap;
1350 }
1351
1352 edesc->dst_dma = map_seq_out_ptr_result(desc, jrdev, req->result,
1353 digestsize);
1354 if (dma_mapping_error(jrdev, edesc->dst_dma)) {
1355 dev_err(jrdev, "unable to map dst\n");
1356 goto unmap;
1357 }
1358
1359 #ifdef DEBUG
1360 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1361 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1362 #endif
1363
1364 ret = caam_jr_enqueue(jrdev, desc, ahash_done, req);
1365 if (!ret) {
1366 ret = -EINPROGRESS;
1367 } else {
1368 ahash_unmap(jrdev, edesc, req, digestsize);
1369 kfree(edesc);
1370 }
1371
1372 return ret;
1373 unmap:
1374 ahash_unmap(jrdev, edesc, req, digestsize);
1375 kfree(edesc);
1376 return -ENOMEM;
1377
1378 }
1379
1380 /* submit first update job descriptor after init */
1381 static int ahash_update_first(struct ahash_request *req)
1382 {
1383 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1384 struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash);
1385 struct caam_hash_state *state = ahash_request_ctx(req);
1386 struct device *jrdev = ctx->jrdev;
1387 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
1388 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
1389 u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0;
1390 int *next_buflen = state->current_buf ?
1391 &state->buflen_1 : &state->buflen_0;
1392 int to_hash;
1393 u32 *desc;
1394 int src_nents, mapped_nents;
1395 struct ahash_edesc *edesc;
1396 int ret = 0;
1397
1398 *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) -
1399 1);
1400 to_hash = req->nbytes - *next_buflen;
1401
1402 if (to_hash) {
1403 src_nents = sg_nents_for_len(req->src,
1404 req->nbytes - *next_buflen);
1405 if (src_nents < 0) {
1406 dev_err(jrdev, "Invalid number of src SG.\n");
1407 return src_nents;
1408 }
1409
1410 if (src_nents) {
1411 mapped_nents = dma_map_sg(jrdev, req->src, src_nents,
1412 DMA_TO_DEVICE);
1413 if (!mapped_nents) {
1414 dev_err(jrdev, "unable to map source for DMA\n");
1415 return -ENOMEM;
1416 }
1417 } else {
1418 mapped_nents = 0;
1419 }
1420
1421 /*
1422 * allocate space for base edesc and hw desc commands,
1423 * link tables
1424 */
1425 edesc = ahash_edesc_alloc(ctx, mapped_nents > 1 ?
1426 mapped_nents : 0,
1427 ctx->sh_desc_update_first,
1428 ctx->sh_desc_update_first_dma,
1429 flags);
1430 if (!edesc) {
1431 dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE);
1432 return -ENOMEM;
1433 }
1434
1435 edesc->src_nents = src_nents;
1436 edesc->dst_dma = 0;
1437
1438 ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0,
1439 to_hash);
1440 if (ret)
1441 goto unmap_ctx;
1442
1443 if (*next_buflen)
1444 scatterwalk_map_and_copy(next_buf, req->src, to_hash,
1445 *next_buflen, 0);
1446
1447 desc = edesc->hw_desc;
1448
1449 ret = map_seq_out_ptr_ctx(desc, jrdev, state, ctx->ctx_len);
1450 if (ret)
1451 goto unmap_ctx;
1452
1453 #ifdef DEBUG
1454 print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ",
1455 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1456 desc_bytes(desc), 1);
1457 #endif
1458
1459 ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req);
1460 if (ret)
1461 goto unmap_ctx;
1462
1463 ret = -EINPROGRESS;
1464 state->update = ahash_update_ctx;
1465 state->finup = ahash_finup_ctx;
1466 state->final = ahash_final_ctx;
1467 } else if (*next_buflen) {
1468 state->update = ahash_update_no_ctx;
1469 state->finup = ahash_finup_no_ctx;
1470 state->final = ahash_final_no_ctx;
1471 scatterwalk_map_and_copy(next_buf, req->src, 0,
1472 req->nbytes, 0);
1473 }
1474 #ifdef DEBUG
1475 print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ",
1476 DUMP_PREFIX_ADDRESS, 16, 4, next_buf,
1477 *next_buflen, 1);
1478 #endif
1479
1480 return ret;
1481 unmap_ctx:
1482 ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE);
1483 kfree(edesc);
1484 return ret;
1485 }
1486
1487 static int ahash_finup_first(struct ahash_request *req)
1488 {
1489 return ahash_digest(req);
1490 }
1491
1492 static int ahash_init(struct ahash_request *req)
1493 {
1494 struct caam_hash_state *state = ahash_request_ctx(req);
1495
1496 state->update = ahash_update_first;
1497 state->finup = ahash_finup_first;
1498 state->final = ahash_final_no_ctx;
1499
1500 state->current_buf = 0;
1501 state->buf_dma = 0;
1502 state->buflen_0 = 0;
1503 state->buflen_1 = 0;
1504
1505 return 0;
1506 }
1507
1508 static int ahash_update(struct ahash_request *req)
1509 {
1510 struct caam_hash_state *state = ahash_request_ctx(req);
1511
1512 return state->update(req);
1513 }
1514
1515 static int ahash_finup(struct ahash_request *req)
1516 {
1517 struct caam_hash_state *state = ahash_request_ctx(req);
1518
1519 return state->finup(req);
1520 }
1521
1522 static int ahash_final(struct ahash_request *req)
1523 {
1524 struct caam_hash_state *state = ahash_request_ctx(req);
1525
1526 return state->final(req);
1527 }
1528
1529 static int ahash_export(struct ahash_request *req, void *out)
1530 {
1531 struct caam_hash_state *state = ahash_request_ctx(req);
1532 struct caam_export_state *export = out;
1533 int len;
1534 u8 *buf;
1535
1536 if (state->current_buf) {
1537 buf = state->buf_1;
1538 len = state->buflen_1;
1539 } else {
1540 buf = state->buf_0;
1541 len = state->buflen_0;
1542 }
1543
1544 memcpy(export->buf, buf, len);
1545 memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx));
1546 export->buflen = len;
1547 export->update = state->update;
1548 export->final = state->final;
1549 export->finup = state->finup;
1550
1551 return 0;
1552 }
1553
1554 static int ahash_import(struct ahash_request *req, const void *in)
1555 {
1556 struct caam_hash_state *state = ahash_request_ctx(req);
1557 const struct caam_export_state *export = in;
1558
1559 memset(state, 0, sizeof(*state));
1560 memcpy(state->buf_0, export->buf, export->buflen);
1561 memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx));
1562 state->buflen_0 = export->buflen;
1563 state->update = export->update;
1564 state->final = export->final;
1565 state->finup = export->finup;
1566
1567 return 0;
1568 }
1569
1570 struct caam_hash_template {
1571 char name[CRYPTO_MAX_ALG_NAME];
1572 char driver_name[CRYPTO_MAX_ALG_NAME];
1573 char hmac_name[CRYPTO_MAX_ALG_NAME];
1574 char hmac_driver_name[CRYPTO_MAX_ALG_NAME];
1575 unsigned int blocksize;
1576 struct ahash_alg template_ahash;
1577 u32 alg_type;
1578 };
1579
1580 /* ahash descriptors */
1581 static struct caam_hash_template driver_hash[] = {
1582 {
1583 .name = "sha1",
1584 .driver_name = "sha1-caam",
1585 .hmac_name = "hmac(sha1)",
1586 .hmac_driver_name = "hmac-sha1-caam",
1587 .blocksize = SHA1_BLOCK_SIZE,
1588 .template_ahash = {
1589 .init = ahash_init,
1590 .update = ahash_update,
1591 .final = ahash_final,
1592 .finup = ahash_finup,
1593 .digest = ahash_digest,
1594 .export = ahash_export,
1595 .import = ahash_import,
1596 .setkey = ahash_setkey,
1597 .halg = {
1598 .digestsize = SHA1_DIGEST_SIZE,
1599 .statesize = sizeof(struct caam_export_state),
1600 },
1601 },
1602 .alg_type = OP_ALG_ALGSEL_SHA1,
1603 }, {
1604 .name = "sha224",
1605 .driver_name = "sha224-caam",
1606 .hmac_name = "hmac(sha224)",
1607 .hmac_driver_name = "hmac-sha224-caam",
1608 .blocksize = SHA224_BLOCK_SIZE,
1609 .template_ahash = {
1610 .init = ahash_init,
1611 .update = ahash_update,
1612 .final = ahash_final,
1613 .finup = ahash_finup,
1614 .digest = ahash_digest,
1615 .export = ahash_export,
1616 .import = ahash_import,
1617 .setkey = ahash_setkey,
1618 .halg = {
1619 .digestsize = SHA224_DIGEST_SIZE,
1620 .statesize = sizeof(struct caam_export_state),
1621 },
1622 },
1623 .alg_type = OP_ALG_ALGSEL_SHA224,
1624 }, {
1625 .name = "sha256",
1626 .driver_name = "sha256-caam",
1627 .hmac_name = "hmac(sha256)",
1628 .hmac_driver_name = "hmac-sha256-caam",
1629 .blocksize = SHA256_BLOCK_SIZE,
1630 .template_ahash = {
1631 .init = ahash_init,
1632 .update = ahash_update,
1633 .final = ahash_final,
1634 .finup = ahash_finup,
1635 .digest = ahash_digest,
1636 .export = ahash_export,
1637 .import = ahash_import,
1638 .setkey = ahash_setkey,
1639 .halg = {
1640 .digestsize = SHA256_DIGEST_SIZE,
1641 .statesize = sizeof(struct caam_export_state),
1642 },
1643 },
1644 .alg_type = OP_ALG_ALGSEL_SHA256,
1645 }, {
1646 .name = "sha384",
1647 .driver_name = "sha384-caam",
1648 .hmac_name = "hmac(sha384)",
1649 .hmac_driver_name = "hmac-sha384-caam",
1650 .blocksize = SHA384_BLOCK_SIZE,
1651 .template_ahash = {
1652 .init = ahash_init,
1653 .update = ahash_update,
1654 .final = ahash_final,
1655 .finup = ahash_finup,
1656 .digest = ahash_digest,
1657 .export = ahash_export,
1658 .import = ahash_import,
1659 .setkey = ahash_setkey,
1660 .halg = {
1661 .digestsize = SHA384_DIGEST_SIZE,
1662 .statesize = sizeof(struct caam_export_state),
1663 },
1664 },
1665 .alg_type = OP_ALG_ALGSEL_SHA384,
1666 }, {
1667 .name = "sha512",
1668 .driver_name = "sha512-caam",
1669 .hmac_name = "hmac(sha512)",
1670 .hmac_driver_name = "hmac-sha512-caam",
1671 .blocksize = SHA512_BLOCK_SIZE,
1672 .template_ahash = {
1673 .init = ahash_init,
1674 .update = ahash_update,
1675 .final = ahash_final,
1676 .finup = ahash_finup,
1677 .digest = ahash_digest,
1678 .export = ahash_export,
1679 .import = ahash_import,
1680 .setkey = ahash_setkey,
1681 .halg = {
1682 .digestsize = SHA512_DIGEST_SIZE,
1683 .statesize = sizeof(struct caam_export_state),
1684 },
1685 },
1686 .alg_type = OP_ALG_ALGSEL_SHA512,
1687 }, {
1688 .name = "md5",
1689 .driver_name = "md5-caam",
1690 .hmac_name = "hmac(md5)",
1691 .hmac_driver_name = "hmac-md5-caam",
1692 .blocksize = MD5_BLOCK_WORDS * 4,
1693 .template_ahash = {
1694 .init = ahash_init,
1695 .update = ahash_update,
1696 .final = ahash_final,
1697 .finup = ahash_finup,
1698 .digest = ahash_digest,
1699 .export = ahash_export,
1700 .import = ahash_import,
1701 .setkey = ahash_setkey,
1702 .halg = {
1703 .digestsize = MD5_DIGEST_SIZE,
1704 .statesize = sizeof(struct caam_export_state),
1705 },
1706 },
1707 .alg_type = OP_ALG_ALGSEL_MD5,
1708 },
1709 };
1710
1711 struct caam_hash_alg {
1712 struct list_head entry;
1713 int alg_type;
1714 struct ahash_alg ahash_alg;
1715 };
1716
1717 static int caam_hash_cra_init(struct crypto_tfm *tfm)
1718 {
1719 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
1720 struct crypto_alg *base = tfm->__crt_alg;
1721 struct hash_alg_common *halg =
1722 container_of(base, struct hash_alg_common, base);
1723 struct ahash_alg *alg =
1724 container_of(halg, struct ahash_alg, halg);
1725 struct caam_hash_alg *caam_hash =
1726 container_of(alg, struct caam_hash_alg, ahash_alg);
1727 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1728 /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */
1729 static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE,
1730 HASH_MSG_LEN + SHA1_DIGEST_SIZE,
1731 HASH_MSG_LEN + 32,
1732 HASH_MSG_LEN + SHA256_DIGEST_SIZE,
1733 HASH_MSG_LEN + 64,
1734 HASH_MSG_LEN + SHA512_DIGEST_SIZE };
1735
1736 /*
1737 * Get a Job ring from Job Ring driver to ensure in-order
1738 * crypto request processing per tfm
1739 */
1740 ctx->jrdev = caam_jr_alloc();
1741 if (IS_ERR(ctx->jrdev)) {
1742 pr_err("Job Ring Device allocation for transform failed\n");
1743 return PTR_ERR(ctx->jrdev);
1744 }
1745 /* copy descriptor header template value */
1746 ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type;
1747
1748 ctx->ctx_len = runninglen[(ctx->adata.algtype &
1749 OP_ALG_ALGSEL_SUBMASK) >>
1750 OP_ALG_ALGSEL_SHIFT];
1751
1752 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1753 sizeof(struct caam_hash_state));
1754 return ahash_set_sh_desc(ahash);
1755 }
1756
1757 static void caam_hash_cra_exit(struct crypto_tfm *tfm)
1758 {
1759 struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm);
1760
1761 if (ctx->sh_desc_update_dma &&
1762 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma))
1763 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma,
1764 desc_bytes(ctx->sh_desc_update),
1765 DMA_TO_DEVICE);
1766 if (ctx->sh_desc_update_first_dma &&
1767 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma))
1768 dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma,
1769 desc_bytes(ctx->sh_desc_update_first),
1770 DMA_TO_DEVICE);
1771 if (ctx->sh_desc_fin_dma &&
1772 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma))
1773 dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma,
1774 desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE);
1775 if (ctx->sh_desc_digest_dma &&
1776 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma))
1777 dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma,
1778 desc_bytes(ctx->sh_desc_digest),
1779 DMA_TO_DEVICE);
1780
1781 caam_jr_free(ctx->jrdev);
1782 }
1783
1784 static void __exit caam_algapi_hash_exit(void)
1785 {
1786 struct caam_hash_alg *t_alg, *n;
1787
1788 if (!hash_list.next)
1789 return;
1790
1791 list_for_each_entry_safe(t_alg, n, &hash_list, entry) {
1792 crypto_unregister_ahash(&t_alg->ahash_alg);
1793 list_del(&t_alg->entry);
1794 kfree(t_alg);
1795 }
1796 }
1797
1798 static struct caam_hash_alg *
1799 caam_hash_alloc(struct caam_hash_template *template,
1800 bool keyed)
1801 {
1802 struct caam_hash_alg *t_alg;
1803 struct ahash_alg *halg;
1804 struct crypto_alg *alg;
1805
1806 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
1807 if (!t_alg) {
1808 pr_err("failed to allocate t_alg\n");
1809 return ERR_PTR(-ENOMEM);
1810 }
1811
1812 t_alg->ahash_alg = template->template_ahash;
1813 halg = &t_alg->ahash_alg;
1814 alg = &halg->halg.base;
1815
1816 if (keyed) {
1817 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1818 template->hmac_name);
1819 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1820 template->hmac_driver_name);
1821 } else {
1822 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s",
1823 template->name);
1824 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
1825 template->driver_name);
1826 t_alg->ahash_alg.setkey = NULL;
1827 }
1828 alg->cra_module = THIS_MODULE;
1829 alg->cra_init = caam_hash_cra_init;
1830 alg->cra_exit = caam_hash_cra_exit;
1831 alg->cra_ctxsize = sizeof(struct caam_hash_ctx);
1832 alg->cra_priority = CAAM_CRA_PRIORITY;
1833 alg->cra_blocksize = template->blocksize;
1834 alg->cra_alignmask = 0;
1835 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH;
1836 alg->cra_type = &crypto_ahash_type;
1837
1838 t_alg->alg_type = template->alg_type;
1839
1840 return t_alg;
1841 }
1842
1843 static int __init caam_algapi_hash_init(void)
1844 {
1845 struct device_node *dev_node;
1846 struct platform_device *pdev;
1847 struct device *ctrldev;
1848 int i = 0, err = 0;
1849 struct caam_drv_private *priv;
1850 unsigned int md_limit = SHA512_DIGEST_SIZE;
1851 u32 cha_inst, cha_vid;
1852
1853 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
1854 if (!dev_node) {
1855 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
1856 if (!dev_node)
1857 return -ENODEV;
1858 }
1859
1860 pdev = of_find_device_by_node(dev_node);
1861 if (!pdev) {
1862 of_node_put(dev_node);
1863 return -ENODEV;
1864 }
1865
1866 ctrldev = &pdev->dev;
1867 priv = dev_get_drvdata(ctrldev);
1868 of_node_put(dev_node);
1869
1870 /*
1871 * If priv is NULL, it's probably because the caam driver wasn't
1872 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
1873 */
1874 if (!priv)
1875 return -ENODEV;
1876
1877 /*
1878 * Register crypto algorithms the device supports. First, identify
1879 * presence and attributes of MD block.
1880 */
1881 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
1882 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
1883
1884 /*
1885 * Skip registration of any hashing algorithms if MD block
1886 * is not present.
1887 */
1888 if (!((cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT))
1889 return -ENODEV;
1890
1891 /* Limit digest size based on LP256 */
1892 if ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)
1893 md_limit = SHA256_DIGEST_SIZE;
1894
1895 INIT_LIST_HEAD(&hash_list);
1896
1897 /* register crypto algorithms the device supports */
1898 for (i = 0; i < ARRAY_SIZE(driver_hash); i++) {
1899 struct caam_hash_alg *t_alg;
1900 struct caam_hash_template *alg = driver_hash + i;
1901
1902 /* If MD size is not supported by device, skip registration */
1903 if (alg->template_ahash.halg.digestsize > md_limit)
1904 continue;
1905
1906 /* register hmac version */
1907 t_alg = caam_hash_alloc(alg, true);
1908 if (IS_ERR(t_alg)) {
1909 err = PTR_ERR(t_alg);
1910 pr_warn("%s alg allocation failed\n", alg->driver_name);
1911 continue;
1912 }
1913
1914 err = crypto_register_ahash(&t_alg->ahash_alg);
1915 if (err) {
1916 pr_warn("%s alg registration failed: %d\n",
1917 t_alg->ahash_alg.halg.base.cra_driver_name,
1918 err);
1919 kfree(t_alg);
1920 } else
1921 list_add_tail(&t_alg->entry, &hash_list);
1922
1923 /* register unkeyed version */
1924 t_alg = caam_hash_alloc(alg, false);
1925 if (IS_ERR(t_alg)) {
1926 err = PTR_ERR(t_alg);
1927 pr_warn("%s alg allocation failed\n", alg->driver_name);
1928 continue;
1929 }
1930
1931 err = crypto_register_ahash(&t_alg->ahash_alg);
1932 if (err) {
1933 pr_warn("%s alg registration failed: %d\n",
1934 t_alg->ahash_alg.halg.base.cra_driver_name,
1935 err);
1936 kfree(t_alg);
1937 } else
1938 list_add_tail(&t_alg->entry, &hash_list);
1939 }
1940
1941 return err;
1942 }
1943
1944 module_init(caam_algapi_hash_init);
1945 module_exit(caam_algapi_hash_exit);
1946
1947 MODULE_LICENSE("GPL");
1948 MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API");
1949 MODULE_AUTHOR("Freescale Semiconductor - NMG");
Linux with added FPC-III support