?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / crypto / inside-secure / safexcel_hash.c
blob43962bc709c6c064089cd4d9a8191a2879a9df4f
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2017 Marvell
4 *
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
6 */
7
8 #include <crypto/aes.h>
9 #include <crypto/hmac.h>
10 #include <crypto/md5.h>
11 #include <crypto/sha.h>
12 #include <crypto/sha3.h>
13 #include <crypto/skcipher.h>
14 #include <crypto/sm3.h>
15 #include <linux/device.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/dmapool.h>
18
19 #include "safexcel.h"
20
21 struct safexcel_ahash_ctx {
22 struct safexcel_context base;
23 struct safexcel_crypto_priv *priv;
24
25 u32 alg;
26 u8 key_sz;
27 bool cbcmac;
28 bool do_fallback;
29 bool fb_init_done;
30 bool fb_do_setkey;
31
32 __le32 ipad[SHA3_512_BLOCK_SIZE / sizeof(__le32)];
33 __le32 opad[SHA3_512_BLOCK_SIZE / sizeof(__le32)];
34
35 struct crypto_cipher *kaes;
36 struct crypto_ahash *fback;
37 struct crypto_shash *shpre;
38 struct shash_desc *shdesc;
39 };
40
41 struct safexcel_ahash_req {
42 bool last_req;
43 bool finish;
44 bool hmac;
45 bool needs_inv;
46 bool hmac_zlen;
47 bool len_is_le;
48 bool not_first;
49 bool xcbcmac;
50
51 int nents;
52 dma_addr_t result_dma;
53
54 u32 digest;
55
56 u8 state_sz; /* expected state size, only set once */
57 u8 block_sz; /* block size, only set once */
58 u8 digest_sz; /* output digest size, only set once */
59 __le32 state[SHA3_512_BLOCK_SIZE /
60 sizeof(__le32)] __aligned(sizeof(__le32));
61
62 u64 len;
63 u64 processed;
64
65 u8 cache[HASH_CACHE_SIZE] __aligned(sizeof(u32));
66 dma_addr_t cache_dma;
67 unsigned int cache_sz;
68
69 u8 cache_next[HASH_CACHE_SIZE] __aligned(sizeof(u32));
70 };
71
72 static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req)
73 {
74 return req->len - req->processed;
75 }
76
77 static void safexcel_hash_token(struct safexcel_command_desc *cdesc,
78 u32 input_length, u32 result_length,
79 bool cbcmac)
80 {
81 struct safexcel_token *token =
82 (struct safexcel_token *)cdesc->control_data.token;
83
84 token[0].opcode = EIP197_TOKEN_OPCODE_DIRECTION;
85 token[0].packet_length = input_length;
86 token[0].instructions = EIP197_TOKEN_INS_TYPE_HASH;
87
88 input_length &= 15;
89 if (unlikely(cbcmac && input_length)) {
90 token[0].stat = 0;
91 token[1].opcode = EIP197_TOKEN_OPCODE_INSERT;
92 token[1].packet_length = 16 - input_length;
93 token[1].stat = EIP197_TOKEN_STAT_LAST_HASH;
94 token[1].instructions = EIP197_TOKEN_INS_TYPE_HASH;
95 } else {
96 token[0].stat = EIP197_TOKEN_STAT_LAST_HASH;
97 eip197_noop_token(&token[1]);
98 }
99
100 token[2].opcode = EIP197_TOKEN_OPCODE_INSERT;
101 token[2].stat = EIP197_TOKEN_STAT_LAST_HASH |
102 EIP197_TOKEN_STAT_LAST_PACKET;
103 token[2].packet_length = result_length;
104 token[2].instructions = EIP197_TOKEN_INS_TYPE_OUTPUT |
105 EIP197_TOKEN_INS_INSERT_HASH_DIGEST;
106
107 eip197_noop_token(&token[3]);
108 }
109
110 static void safexcel_context_control(struct safexcel_ahash_ctx *ctx,
111 struct safexcel_ahash_req *req,
112 struct safexcel_command_desc *cdesc)
113 {
114 struct safexcel_crypto_priv *priv = ctx->priv;
115 u64 count = 0;
116
117 cdesc->control_data.control0 = ctx->alg;
118 cdesc->control_data.control1 = 0;
119
120 /*
121 * Copy the input digest if needed, and setup the context
122 * fields. Do this now as we need it to setup the first command
123 * descriptor.
124 */
125 if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM)) {
126 if (req->xcbcmac)
127 memcpy(ctx->base.ctxr->data, ctx->ipad, ctx->key_sz);
128 else
129 memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
130
131 if (!req->finish && req->xcbcmac)
132 cdesc->control_data.control0 |=
133 CONTEXT_CONTROL_DIGEST_XCM |
134 CONTEXT_CONTROL_TYPE_HASH_OUT |
135 CONTEXT_CONTROL_NO_FINISH_HASH |
136 CONTEXT_CONTROL_SIZE(req->state_sz /
137 sizeof(u32));
138 else
139 cdesc->control_data.control0 |=
140 CONTEXT_CONTROL_DIGEST_XCM |
141 CONTEXT_CONTROL_TYPE_HASH_OUT |
142 CONTEXT_CONTROL_SIZE(req->state_sz /
143 sizeof(u32));
144 return;
145 } else if (!req->processed) {
146 /* First - and possibly only - block of basic hash only */
147 if (req->finish)
148 cdesc->control_data.control0 |= req->digest |
149 CONTEXT_CONTROL_TYPE_HASH_OUT |
150 CONTEXT_CONTROL_RESTART_HASH |
151 /* ensure its not 0! */
152 CONTEXT_CONTROL_SIZE(1);
153 else
154 cdesc->control_data.control0 |= req->digest |
155 CONTEXT_CONTROL_TYPE_HASH_OUT |
156 CONTEXT_CONTROL_RESTART_HASH |
157 CONTEXT_CONTROL_NO_FINISH_HASH |
158 /* ensure its not 0! */
159 CONTEXT_CONTROL_SIZE(1);
160 return;
161 }
162
163 /* Hash continuation or HMAC, setup (inner) digest from state */
164 memcpy(ctx->base.ctxr->data, req->state, req->state_sz);
165
166 if (req->finish) {
167 /* Compute digest count for hash/HMAC finish operations */
168 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
169 req->hmac_zlen || (req->processed != req->block_sz)) {
170 count = req->processed / EIP197_COUNTER_BLOCK_SIZE;
171
172 /* This is a hardware limitation, as the
173 * counter must fit into an u32. This represents
174 * a fairly big amount of input data, so we
175 * shouldn't see this.
176 */
177 if (unlikely(count & 0xffffffff00000000ULL)) {
178 dev_warn(priv->dev,
179 "Input data is too big\n");
180 return;
181 }
182 }
183
184 if ((req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) ||
185 /* Special case: zero length HMAC */
186 req->hmac_zlen ||
187 /* PE HW < 4.4 cannot do HMAC continue, fake using hash */
188 (req->processed != req->block_sz)) {
189 /* Basic hash continue operation, need digest + cnt */
190 cdesc->control_data.control0 |=
191 CONTEXT_CONTROL_SIZE((req->state_sz >> 2) + 1) |
192 CONTEXT_CONTROL_TYPE_HASH_OUT |
193 CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
194 /* For zero-len HMAC, don't finalize, already padded! */
195 if (req->hmac_zlen)
196 cdesc->control_data.control0 |=
197 CONTEXT_CONTROL_NO_FINISH_HASH;
198 cdesc->control_data.control1 |=
199 CONTEXT_CONTROL_DIGEST_CNT;
200 ctx->base.ctxr->data[req->state_sz >> 2] =
201 cpu_to_le32(count);
202 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
203
204 /* Clear zero-length HMAC flag for next operation! */
205 req->hmac_zlen = false;
206 } else { /* HMAC */
207 /* Need outer digest for HMAC finalization */
208 memcpy(ctx->base.ctxr->data + (req->state_sz >> 2),
209 ctx->opad, req->state_sz);
210
211 /* Single pass HMAC - no digest count */
212 cdesc->control_data.control0 |=
213 CONTEXT_CONTROL_SIZE(req->state_sz >> 1) |
214 CONTEXT_CONTROL_TYPE_HASH_OUT |
215 CONTEXT_CONTROL_DIGEST_HMAC;
216 }
217 } else { /* Hash continuation, do not finish yet */
218 cdesc->control_data.control0 |=
219 CONTEXT_CONTROL_SIZE(req->state_sz >> 2) |
220 CONTEXT_CONTROL_DIGEST_PRECOMPUTED |
221 CONTEXT_CONTROL_TYPE_HASH_OUT |
222 CONTEXT_CONTROL_NO_FINISH_HASH;
223 }
224 }
225
226 static int safexcel_ahash_enqueue(struct ahash_request *areq);
227
228 static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv,
229 int ring,
230 struct crypto_async_request *async,
231 bool *should_complete, int *ret)
232 {
233 struct safexcel_result_desc *rdesc;
234 struct ahash_request *areq = ahash_request_cast(async);
235 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
236 struct safexcel_ahash_req *sreq = ahash_request_ctx(areq);
237 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
238 u64 cache_len;
239
240 *ret = 0;
241
242 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
243 if (IS_ERR(rdesc)) {
244 dev_err(priv->dev,
245 "hash: result: could not retrieve the result descriptor\n");
246 *ret = PTR_ERR(rdesc);
247 } else {
248 *ret = safexcel_rdesc_check_errors(priv, rdesc);
249 }
250
251 safexcel_complete(priv, ring);
252
253 if (sreq->nents) {
254 dma_unmap_sg(priv->dev, areq->src, sreq->nents, DMA_TO_DEVICE);
255 sreq->nents = 0;
256 }
257
258 if (sreq->result_dma) {
259 dma_unmap_single(priv->dev, sreq->result_dma, sreq->digest_sz,
260 DMA_FROM_DEVICE);
261 sreq->result_dma = 0;
262 }
263
264 if (sreq->cache_dma) {
265 dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz,
266 DMA_TO_DEVICE);
267 sreq->cache_dma = 0;
268 sreq->cache_sz = 0;
269 }
270
271 if (sreq->finish) {
272 if (sreq->hmac &&
273 (sreq->digest != CONTEXT_CONTROL_DIGEST_HMAC)) {
274 /* Faking HMAC using hash - need to do outer hash */
275 memcpy(sreq->cache, sreq->state,
276 crypto_ahash_digestsize(ahash));
277
278 memcpy(sreq->state, ctx->opad, sreq->digest_sz);
279
280 sreq->len = sreq->block_sz +
281 crypto_ahash_digestsize(ahash);
282 sreq->processed = sreq->block_sz;
283 sreq->hmac = 0;
284
285 if (priv->flags & EIP197_TRC_CACHE)
286 ctx->base.needs_inv = true;
287 areq->nbytes = 0;
288 safexcel_ahash_enqueue(areq);
289
290 *should_complete = false; /* Not done yet */
291 return 1;
292 }
293
294 if (unlikely(sreq->digest == CONTEXT_CONTROL_DIGEST_XCM &&
295 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_CRC32)) {
296 /* Undo final XOR with 0xffffffff ...*/
297 *(__le32 *)areq->result = ~sreq->state[0];
298 } else {
299 memcpy(areq->result, sreq->state,
300 crypto_ahash_digestsize(ahash));
301 }
302 }
303
304 cache_len = safexcel_queued_len(sreq);
305 if (cache_len)
306 memcpy(sreq->cache, sreq->cache_next, cache_len);
307
308 *should_complete = true;
309
310 return 1;
311 }
312
313 static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring,
314 int *commands, int *results)
315 {
316 struct ahash_request *areq = ahash_request_cast(async);
317 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
318 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
319 struct safexcel_crypto_priv *priv = ctx->priv;
320 struct safexcel_command_desc *cdesc, *first_cdesc = NULL;
321 struct safexcel_result_desc *rdesc;
322 struct scatterlist *sg;
323 struct safexcel_token *dmmy;
324 int i, extra = 0, n_cdesc = 0, ret = 0, cache_len, skip = 0;
325 u64 queued, len;
326
327 queued = safexcel_queued_len(req);
328 if (queued <= HASH_CACHE_SIZE)
329 cache_len = queued;
330 else
331 cache_len = queued - areq->nbytes;
332
333 if (!req->finish && !req->last_req) {
334 /* If this is not the last request and the queued data does not
335 * fit into full cache blocks, cache it for the next send call.
336 */
337 extra = queued & (HASH_CACHE_SIZE - 1);
338
339 /* If this is not the last request and the queued data
340 * is a multiple of a block, cache the last one for now.
341 */
342 if (!extra)
343 extra = HASH_CACHE_SIZE;
344
345 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
346 req->cache_next, extra,
347 areq->nbytes - extra);
348
349 queued -= extra;
350
351 if (!queued) {
352 *commands = 0;
353 *results = 0;
354 return 0;
355 }
356
357 extra = 0;
358 }
359
360 if (unlikely(req->xcbcmac && req->processed > AES_BLOCK_SIZE)) {
361 if (unlikely(cache_len < AES_BLOCK_SIZE)) {
362 /*
363 * Cache contains less than 1 full block, complete.
364 */
365 extra = AES_BLOCK_SIZE - cache_len;
366 if (queued > cache_len) {
367 /* More data follows: borrow bytes */
368 u64 tmp = queued - cache_len;
369
370 skip = min_t(u64, tmp, extra);
371 sg_pcopy_to_buffer(areq->src,
372 sg_nents(areq->src),
373 req->cache + cache_len,
374 skip, 0);
375 }
376 extra -= skip;
377 memset(req->cache + cache_len + skip, 0, extra);
378 if (!ctx->cbcmac && extra) {
379 // 10- padding for XCBCMAC & CMAC
380 req->cache[cache_len + skip] = 0x80;
381 // HW will use K2 iso K3 - compensate!
382 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
383 ((__be32 *)req->cache)[i] ^=
384 cpu_to_be32(le32_to_cpu(
385 ctx->ipad[i] ^ ctx->ipad[i + 4]));
386 }
387 cache_len = AES_BLOCK_SIZE;
388 queued = queued + extra;
389 }
390
391 /* XCBC continue: XOR previous result into 1st word */
392 crypto_xor(req->cache, (const u8 *)req->state, AES_BLOCK_SIZE);
393 }
394
395 len = queued;
396 /* Add a command descriptor for the cached data, if any */
397 if (cache_len) {
398 req->cache_dma = dma_map_single(priv->dev, req->cache,
399 cache_len, DMA_TO_DEVICE);
400 if (dma_mapping_error(priv->dev, req->cache_dma))
401 return -EINVAL;
402
403 req->cache_sz = cache_len;
404 first_cdesc = safexcel_add_cdesc(priv, ring, 1,
405 (cache_len == len),
406 req->cache_dma, cache_len,
407 len, ctx->base.ctxr_dma,
408 &dmmy);
409 if (IS_ERR(first_cdesc)) {
410 ret = PTR_ERR(first_cdesc);
411 goto unmap_cache;
412 }
413 n_cdesc++;
414
415 queued -= cache_len;
416 if (!queued)
417 goto send_command;
418 }
419
420 /* Now handle the current ahash request buffer(s) */
421 req->nents = dma_map_sg(priv->dev, areq->src,
422 sg_nents_for_len(areq->src,
423 areq->nbytes),
424 DMA_TO_DEVICE);
425 if (!req->nents) {
426 ret = -ENOMEM;
427 goto cdesc_rollback;
428 }
429
430 for_each_sg(areq->src, sg, req->nents, i) {
431 int sglen = sg_dma_len(sg);
432
433 if (unlikely(sglen <= skip)) {
434 skip -= sglen;
435 continue;
436 }
437
438 /* Do not overflow the request */
439 if ((queued + skip) <= sglen)
440 sglen = queued;
441 else
442 sglen -= skip;
443
444 cdesc = safexcel_add_cdesc(priv, ring, !n_cdesc,
445 !(queued - sglen),
446 sg_dma_address(sg) + skip, sglen,
447 len, ctx->base.ctxr_dma, &dmmy);
448 if (IS_ERR(cdesc)) {
449 ret = PTR_ERR(cdesc);
450 goto unmap_sg;
451 }
452
453 if (!n_cdesc)
454 first_cdesc = cdesc;
455 n_cdesc++;
456
457 queued -= sglen;
458 if (!queued)
459 break;
460 skip = 0;
461 }
462
463 send_command:
464 /* Setup the context options */
465 safexcel_context_control(ctx, req, first_cdesc);
466
467 /* Add the token */
468 safexcel_hash_token(first_cdesc, len, req->digest_sz, ctx->cbcmac);
469
470 req->result_dma = dma_map_single(priv->dev, req->state, req->digest_sz,
471 DMA_FROM_DEVICE);
472 if (dma_mapping_error(priv->dev, req->result_dma)) {
473 ret = -EINVAL;
474 goto unmap_sg;
475 }
476
477 /* Add a result descriptor */
478 rdesc = safexcel_add_rdesc(priv, ring, 1, 1, req->result_dma,
479 req->digest_sz);
480 if (IS_ERR(rdesc)) {
481 ret = PTR_ERR(rdesc);
482 goto unmap_result;
483 }
484
485 safexcel_rdr_req_set(priv, ring, rdesc, &areq->base);
486
487 req->processed += len - extra;
488
489 *commands = n_cdesc;
490 *results = 1;
491 return 0;
492
493 unmap_result:
494 dma_unmap_single(priv->dev, req->result_dma, req->digest_sz,
495 DMA_FROM_DEVICE);
496 unmap_sg:
497 if (req->nents) {
498 dma_unmap_sg(priv->dev, areq->src, req->nents, DMA_TO_DEVICE);
499 req->nents = 0;
500 }
501 cdesc_rollback:
502 for (i = 0; i < n_cdesc; i++)
503 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
504 unmap_cache:
505 if (req->cache_dma) {
506 dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz,
507 DMA_TO_DEVICE);
508 req->cache_dma = 0;
509 req->cache_sz = 0;
510 }
511
512 return ret;
513 }
514
515 static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv,
516 int ring,
517 struct crypto_async_request *async,
518 bool *should_complete, int *ret)
519 {
520 struct safexcel_result_desc *rdesc;
521 struct ahash_request *areq = ahash_request_cast(async);
522 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
523 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(ahash);
524 int enq_ret;
525
526 *ret = 0;
527
528 rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr);
529 if (IS_ERR(rdesc)) {
530 dev_err(priv->dev,
531 "hash: invalidate: could not retrieve the result descriptor\n");
532 *ret = PTR_ERR(rdesc);
533 } else {
534 *ret = safexcel_rdesc_check_errors(priv, rdesc);
535 }
536
537 safexcel_complete(priv, ring);
538
539 if (ctx->base.exit_inv) {
540 dma_pool_free(priv->context_pool, ctx->base.ctxr,
541 ctx->base.ctxr_dma);
542
543 *should_complete = true;
544 return 1;
545 }
546
547 ring = safexcel_select_ring(priv);
548 ctx->base.ring = ring;
549
550 spin_lock_bh(&priv->ring[ring].queue_lock);
551 enq_ret = crypto_enqueue_request(&priv->ring[ring].queue, async);
552 spin_unlock_bh(&priv->ring[ring].queue_lock);
553
554 if (enq_ret != -EINPROGRESS)
555 *ret = enq_ret;
556
557 queue_work(priv->ring[ring].workqueue,
558 &priv->ring[ring].work_data.work);
559
560 *should_complete = false;
561
562 return 1;
563 }
564
565 static int safexcel_handle_result(struct safexcel_crypto_priv *priv, int ring,
566 struct crypto_async_request *async,
567 bool *should_complete, int *ret)
568 {
569 struct ahash_request *areq = ahash_request_cast(async);
570 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
571 int err;
572
573 BUG_ON(!(priv->flags & EIP197_TRC_CACHE) && req->needs_inv);
574
575 if (req->needs_inv) {
576 req->needs_inv = false;
577 err = safexcel_handle_inv_result(priv, ring, async,
578 should_complete, ret);
579 } else {
580 err = safexcel_handle_req_result(priv, ring, async,
581 should_complete, ret);
582 }
583
584 return err;
585 }
586
587 static int safexcel_ahash_send_inv(struct crypto_async_request *async,
588 int ring, int *commands, int *results)
589 {
590 struct ahash_request *areq = ahash_request_cast(async);
591 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
592 int ret;
593
594 ret = safexcel_invalidate_cache(async, ctx->priv,
595 ctx->base.ctxr_dma, ring);
596 if (unlikely(ret))
597 return ret;
598
599 *commands = 1;
600 *results = 1;
601
602 return 0;
603 }
604
605 static int safexcel_ahash_send(struct crypto_async_request *async,
606 int ring, int *commands, int *results)
607 {
608 struct ahash_request *areq = ahash_request_cast(async);
609 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
610 int ret;
611
612 if (req->needs_inv)
613 ret = safexcel_ahash_send_inv(async, ring, commands, results);
614 else
615 ret = safexcel_ahash_send_req(async, ring, commands, results);
616
617 return ret;
618 }
619
620 static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm)
621 {
622 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
623 struct safexcel_crypto_priv *priv = ctx->priv;
624 EIP197_REQUEST_ON_STACK(req, ahash, EIP197_AHASH_REQ_SIZE);
625 struct safexcel_ahash_req *rctx = ahash_request_ctx(req);
626 struct safexcel_inv_result result = {};
627 int ring = ctx->base.ring;
628
629 memset(req, 0, EIP197_AHASH_REQ_SIZE);
630
631 /* create invalidation request */
632 init_completion(&result.completion);
633 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
634 safexcel_inv_complete, &result);
635
636 ahash_request_set_tfm(req, __crypto_ahash_cast(tfm));
637 ctx = crypto_tfm_ctx(req->base.tfm);
638 ctx->base.exit_inv = true;
639 rctx->needs_inv = true;
640
641 spin_lock_bh(&priv->ring[ring].queue_lock);
642 crypto_enqueue_request(&priv->ring[ring].queue, &req->base);
643 spin_unlock_bh(&priv->ring[ring].queue_lock);
644
645 queue_work(priv->ring[ring].workqueue,
646 &priv->ring[ring].work_data.work);
647
648 wait_for_completion(&result.completion);
649
650 if (result.error) {
651 dev_warn(priv->dev, "hash: completion error (%d)\n",
652 result.error);
653 return result.error;
654 }
655
656 return 0;
657 }
658
659 /* safexcel_ahash_cache: cache data until at least one request can be sent to
660 * the engine, aka. when there is at least 1 block size in the pipe.
661 */
662 static int safexcel_ahash_cache(struct ahash_request *areq)
663 {
664 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
665 u64 cache_len;
666
667 /* cache_len: everything accepted by the driver but not sent yet,
668 * tot sz handled by update() - last req sz - tot sz handled by send()
669 */
670 cache_len = safexcel_queued_len(req);
671
672 /*
673 * In case there isn't enough bytes to proceed (less than a
674 * block size), cache the data until we have enough.
675 */
676 if (cache_len + areq->nbytes <= HASH_CACHE_SIZE) {
677 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
678 req->cache + cache_len,
679 areq->nbytes, 0);
680 return 0;
681 }
682
683 /* We couldn't cache all the data */
684 return -E2BIG;
685 }
686
687 static int safexcel_ahash_enqueue(struct ahash_request *areq)
688 {
689 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
690 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
691 struct safexcel_crypto_priv *priv = ctx->priv;
692 int ret, ring;
693
694 req->needs_inv = false;
695
696 if (ctx->base.ctxr) {
697 if (priv->flags & EIP197_TRC_CACHE && !ctx->base.needs_inv &&
698 /* invalidate for *any* non-XCBC continuation */
699 ((req->not_first && !req->xcbcmac) ||
700 /* invalidate if (i)digest changed */
701 memcmp(ctx->base.ctxr->data, req->state, req->state_sz) ||
702 /* invalidate for HMAC finish with odigest changed */
703 (req->finish && req->hmac &&
704 memcmp(ctx->base.ctxr->data + (req->state_sz>>2),
705 ctx->opad, req->state_sz))))
706 /*
707 * We're still setting needs_inv here, even though it is
708 * cleared right away, because the needs_inv flag can be
709 * set in other functions and we want to keep the same
710 * logic.
711 */
712 ctx->base.needs_inv = true;
713
714 if (ctx->base.needs_inv) {
715 ctx->base.needs_inv = false;
716 req->needs_inv = true;
717 }
718 } else {
719 ctx->base.ring = safexcel_select_ring(priv);
720 ctx->base.ctxr = dma_pool_zalloc(priv->context_pool,
721 EIP197_GFP_FLAGS(areq->base),
722 &ctx->base.ctxr_dma);
723 if (!ctx->base.ctxr)
724 return -ENOMEM;
725 }
726 req->not_first = true;
727
728 ring = ctx->base.ring;
729
730 spin_lock_bh(&priv->ring[ring].queue_lock);
731 ret = crypto_enqueue_request(&priv->ring[ring].queue, &areq->base);
732 spin_unlock_bh(&priv->ring[ring].queue_lock);
733
734 queue_work(priv->ring[ring].workqueue,
735 &priv->ring[ring].work_data.work);
736
737 return ret;
738 }
739
740 static int safexcel_ahash_update(struct ahash_request *areq)
741 {
742 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
743 int ret;
744
745 /* If the request is 0 length, do nothing */
746 if (!areq->nbytes)
747 return 0;
748
749 /* Add request to the cache if it fits */
750 ret = safexcel_ahash_cache(areq);
751
752 /* Update total request length */
753 req->len += areq->nbytes;
754
755 /* If not all data could fit into the cache, go process the excess.
756 * Also go process immediately for an HMAC IV precompute, which
757 * will never be finished at all, but needs to be processed anyway.
758 */
759 if ((ret && !req->finish) || req->last_req)
760 return safexcel_ahash_enqueue(areq);
761
762 return 0;
763 }
764
765 static int safexcel_ahash_final(struct ahash_request *areq)
766 {
767 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
768 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
769
770 req->finish = true;
771
772 if (unlikely(!req->len && !areq->nbytes)) {
773 /*
774 * If we have an overall 0 length *hash* request:
775 * The HW cannot do 0 length hash, so we provide the correct
776 * result directly here.
777 */
778 if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5)
779 memcpy(areq->result, md5_zero_message_hash,
780 MD5_DIGEST_SIZE);
781 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA1)
782 memcpy(areq->result, sha1_zero_message_hash,
783 SHA1_DIGEST_SIZE);
784 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA224)
785 memcpy(areq->result, sha224_zero_message_hash,
786 SHA224_DIGEST_SIZE);
787 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA256)
788 memcpy(areq->result, sha256_zero_message_hash,
789 SHA256_DIGEST_SIZE);
790 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA384)
791 memcpy(areq->result, sha384_zero_message_hash,
792 SHA384_DIGEST_SIZE);
793 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SHA512)
794 memcpy(areq->result, sha512_zero_message_hash,
795 SHA512_DIGEST_SIZE);
796 else if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_SM3) {
797 memcpy(areq->result,
798 EIP197_SM3_ZEROM_HASH, SM3_DIGEST_SIZE);
799 }
800
801 return 0;
802 } else if (unlikely(req->digest == CONTEXT_CONTROL_DIGEST_XCM &&
803 ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5 &&
804 req->len == sizeof(u32) && !areq->nbytes)) {
805 /* Zero length CRC32 */
806 memcpy(areq->result, ctx->ipad, sizeof(u32));
807 return 0;
808 } else if (unlikely(ctx->cbcmac && req->len == AES_BLOCK_SIZE &&
809 !areq->nbytes)) {
810 /* Zero length CBC MAC */
811 memset(areq->result, 0, AES_BLOCK_SIZE);
812 return 0;
813 } else if (unlikely(req->xcbcmac && req->len == AES_BLOCK_SIZE &&
814 !areq->nbytes)) {
815 /* Zero length (X)CBC/CMAC */
816 int i;
817
818 for (i = 0; i < AES_BLOCK_SIZE / sizeof(u32); i++)
819 ((__be32 *)areq->result)[i] =
820 cpu_to_be32(le32_to_cpu(ctx->ipad[i + 4]));//K3
821 areq->result[0] ^= 0x80; // 10- padding
822 crypto_cipher_encrypt_one(ctx->kaes, areq->result, areq->result);
823 return 0;
824 } else if (unlikely(req->hmac &&
825 (req->len == req->block_sz) &&
826 !areq->nbytes)) {
827 /*
828 * If we have an overall 0 length *HMAC* request:
829 * For HMAC, we need to finalize the inner digest
830 * and then perform the outer hash.
831 */
832
833 /* generate pad block in the cache */
834 /* start with a hash block of all zeroes */
835 memset(req->cache, 0, req->block_sz);
836 /* set the first byte to 0x80 to 'append a 1 bit' */
837 req->cache[0] = 0x80;
838 /* add the length in bits in the last 2 bytes */
839 if (req->len_is_le) {
840 /* Little endian length word (e.g. MD5) */
841 req->cache[req->block_sz-8] = (req->block_sz << 3) &
842 255;
843 req->cache[req->block_sz-7] = (req->block_sz >> 5);
844 } else {
845 /* Big endian length word (e.g. any SHA) */
846 req->cache[req->block_sz-2] = (req->block_sz >> 5);
847 req->cache[req->block_sz-1] = (req->block_sz << 3) &
848 255;
849 }
850
851 req->len += req->block_sz; /* plus 1 hash block */
852
853 /* Set special zero-length HMAC flag */
854 req->hmac_zlen = true;
855
856 /* Finalize HMAC */
857 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
858 } else if (req->hmac) {
859 /* Finalize HMAC */
860 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
861 }
862
863 return safexcel_ahash_enqueue(areq);
864 }
865
866 static int safexcel_ahash_finup(struct ahash_request *areq)
867 {
868 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
869
870 req->finish = true;
871
872 safexcel_ahash_update(areq);
873 return safexcel_ahash_final(areq);
874 }
875
876 static int safexcel_ahash_export(struct ahash_request *areq, void *out)
877 {
878 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
879 struct safexcel_ahash_export_state *export = out;
880
881 export->len = req->len;
882 export->processed = req->processed;
883
884 export->digest = req->digest;
885
886 memcpy(export->state, req->state, req->state_sz);
887 memcpy(export->cache, req->cache, HASH_CACHE_SIZE);
888
889 return 0;
890 }
891
892 static int safexcel_ahash_import(struct ahash_request *areq, const void *in)
893 {
894 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
895 const struct safexcel_ahash_export_state *export = in;
896 int ret;
897
898 ret = crypto_ahash_init(areq);
899 if (ret)
900 return ret;
901
902 req->len = export->len;
903 req->processed = export->processed;
904
905 req->digest = export->digest;
906
907 memcpy(req->cache, export->cache, HASH_CACHE_SIZE);
908 memcpy(req->state, export->state, req->state_sz);
909
910 return 0;
911 }
912
913 static int safexcel_ahash_cra_init(struct crypto_tfm *tfm)
914 {
915 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
916 struct safexcel_alg_template *tmpl =
917 container_of(__crypto_ahash_alg(tfm->__crt_alg),
918 struct safexcel_alg_template, alg.ahash);
919
920 ctx->priv = tmpl->priv;
921 ctx->base.send = safexcel_ahash_send;
922 ctx->base.handle_result = safexcel_handle_result;
923 ctx->fb_do_setkey = false;
924
925 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
926 sizeof(struct safexcel_ahash_req));
927 return 0;
928 }
929
930 static int safexcel_sha1_init(struct ahash_request *areq)
931 {
932 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
933 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
934
935 memset(req, 0, sizeof(*req));
936
937 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
938 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
939 req->state_sz = SHA1_DIGEST_SIZE;
940 req->digest_sz = SHA1_DIGEST_SIZE;
941 req->block_sz = SHA1_BLOCK_SIZE;
942
943 return 0;
944 }
945
946 static int safexcel_sha1_digest(struct ahash_request *areq)
947 {
948 int ret = safexcel_sha1_init(areq);
949
950 if (ret)
951 return ret;
952
953 return safexcel_ahash_finup(areq);
954 }
955
956 static void safexcel_ahash_cra_exit(struct crypto_tfm *tfm)
957 {
958 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
959 struct safexcel_crypto_priv *priv = ctx->priv;
960 int ret;
961
962 /* context not allocated, skip invalidation */
963 if (!ctx->base.ctxr)
964 return;
965
966 if (priv->flags & EIP197_TRC_CACHE) {
967 ret = safexcel_ahash_exit_inv(tfm);
968 if (ret)
969 dev_warn(priv->dev, "hash: invalidation error %d\n", ret);
970 } else {
971 dma_pool_free(priv->context_pool, ctx->base.ctxr,
972 ctx->base.ctxr_dma);
973 }
974 }
975
976 struct safexcel_alg_template safexcel_alg_sha1 = {
977 .type = SAFEXCEL_ALG_TYPE_AHASH,
978 .algo_mask = SAFEXCEL_ALG_SHA1,
979 .alg.ahash = {
980 .init = safexcel_sha1_init,
981 .update = safexcel_ahash_update,
982 .final = safexcel_ahash_final,
983 .finup = safexcel_ahash_finup,
984 .digest = safexcel_sha1_digest,
985 .export = safexcel_ahash_export,
986 .import = safexcel_ahash_import,
987 .halg = {
988 .digestsize = SHA1_DIGEST_SIZE,
989 .statesize = sizeof(struct safexcel_ahash_export_state),
990 .base = {
991 .cra_name = "sha1",
992 .cra_driver_name = "safexcel-sha1",
993 .cra_priority = SAFEXCEL_CRA_PRIORITY,
994 .cra_flags = CRYPTO_ALG_ASYNC |
995 CRYPTO_ALG_KERN_DRIVER_ONLY,
996 .cra_blocksize = SHA1_BLOCK_SIZE,
997 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
998 .cra_init = safexcel_ahash_cra_init,
999 .cra_exit = safexcel_ahash_cra_exit,
1000 .cra_module = THIS_MODULE,
1001 },
1002 },
1003 },
1004 };
1005
1006 static int safexcel_hmac_sha1_init(struct ahash_request *areq)
1007 {
1008 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1009 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1010
1011 memset(req, 0, sizeof(*req));
1012
1013 /* Start from ipad precompute */
1014 memcpy(req->state, ctx->ipad, SHA1_DIGEST_SIZE);
1015 /* Already processed the key^ipad part now! */
1016 req->len = SHA1_BLOCK_SIZE;
1017 req->processed = SHA1_BLOCK_SIZE;
1018
1019 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA1;
1020 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1021 req->state_sz = SHA1_DIGEST_SIZE;
1022 req->digest_sz = SHA1_DIGEST_SIZE;
1023 req->block_sz = SHA1_BLOCK_SIZE;
1024 req->hmac = true;
1025
1026 return 0;
1027 }
1028
1029 static int safexcel_hmac_sha1_digest(struct ahash_request *areq)
1030 {
1031 int ret = safexcel_hmac_sha1_init(areq);
1032
1033 if (ret)
1034 return ret;
1035
1036 return safexcel_ahash_finup(areq);
1037 }
1038
1039 struct safexcel_ahash_result {
1040 struct completion completion;
1041 int error;
1042 };
1043
1044 static void safexcel_ahash_complete(struct crypto_async_request *req, int error)
1045 {
1046 struct safexcel_ahash_result *result = req->data;
1047
1048 if (error == -EINPROGRESS)
1049 return;
1050
1051 result->error = error;
1052 complete(&result->completion);
1053 }
1054
1055 static int safexcel_hmac_init_pad(struct ahash_request *areq,
1056 unsigned int blocksize, const u8 *key,
1057 unsigned int keylen, u8 *ipad, u8 *opad)
1058 {
1059 struct safexcel_ahash_result result;
1060 struct scatterlist sg;
1061 int ret, i;
1062 u8 *keydup;
1063
1064 if (keylen <= blocksize) {
1065 memcpy(ipad, key, keylen);
1066 } else {
1067 keydup = kmemdup(key, keylen, GFP_KERNEL);
1068 if (!keydup)
1069 return -ENOMEM;
1070
1071 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1072 safexcel_ahash_complete, &result);
1073 sg_init_one(&sg, keydup, keylen);
1074 ahash_request_set_crypt(areq, &sg, ipad, keylen);
1075 init_completion(&result.completion);
1076
1077 ret = crypto_ahash_digest(areq);
1078 if (ret == -EINPROGRESS || ret == -EBUSY) {
1079 wait_for_completion_interruptible(&result.completion);
1080 ret = result.error;
1081 }
1082
1083 /* Avoid leaking */
1084 memzero_explicit(keydup, keylen);
1085 kfree(keydup);
1086
1087 if (ret)
1088 return ret;
1089
1090 keylen = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1091 }
1092
1093 memset(ipad + keylen, 0, blocksize - keylen);
1094 memcpy(opad, ipad, blocksize);
1095
1096 for (i = 0; i < blocksize; i++) {
1097 ipad[i] ^= HMAC_IPAD_VALUE;
1098 opad[i] ^= HMAC_OPAD_VALUE;
1099 }
1100
1101 return 0;
1102 }
1103
1104 static int safexcel_hmac_init_iv(struct ahash_request *areq,
1105 unsigned int blocksize, u8 *pad, void *state)
1106 {
1107 struct safexcel_ahash_result result;
1108 struct safexcel_ahash_req *req;
1109 struct scatterlist sg;
1110 int ret;
1111
1112 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_BACKLOG,
1113 safexcel_ahash_complete, &result);
1114 sg_init_one(&sg, pad, blocksize);
1115 ahash_request_set_crypt(areq, &sg, pad, blocksize);
1116 init_completion(&result.completion);
1117
1118 ret = crypto_ahash_init(areq);
1119 if (ret)
1120 return ret;
1121
1122 req = ahash_request_ctx(areq);
1123 req->hmac = true;
1124 req->last_req = true;
1125
1126 ret = crypto_ahash_update(areq);
1127 if (ret && ret != -EINPROGRESS && ret != -EBUSY)
1128 return ret;
1129
1130 wait_for_completion_interruptible(&result.completion);
1131 if (result.error)
1132 return result.error;
1133
1134 return crypto_ahash_export(areq, state);
1135 }
1136
1137 int safexcel_hmac_setkey(const char *alg, const u8 *key, unsigned int keylen,
1138 void *istate, void *ostate)
1139 {
1140 struct ahash_request *areq;
1141 struct crypto_ahash *tfm;
1142 unsigned int blocksize;
1143 u8 *ipad, *opad;
1144 int ret;
1145
1146 tfm = crypto_alloc_ahash(alg, 0, 0);
1147 if (IS_ERR(tfm))
1148 return PTR_ERR(tfm);
1149
1150 areq = ahash_request_alloc(tfm, GFP_KERNEL);
1151 if (!areq) {
1152 ret = -ENOMEM;
1153 goto free_ahash;
1154 }
1155
1156 crypto_ahash_clear_flags(tfm, ~0);
1157 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1158
1159 ipad = kcalloc(2, blocksize, GFP_KERNEL);
1160 if (!ipad) {
1161 ret = -ENOMEM;
1162 goto free_request;
1163 }
1164
1165 opad = ipad + blocksize;
1166
1167 ret = safexcel_hmac_init_pad(areq, blocksize, key, keylen, ipad, opad);
1168 if (ret)
1169 goto free_ipad;
1170
1171 ret = safexcel_hmac_init_iv(areq, blocksize, ipad, istate);
1172 if (ret)
1173 goto free_ipad;
1174
1175 ret = safexcel_hmac_init_iv(areq, blocksize, opad, ostate);
1176
1177 free_ipad:
1178 kfree(ipad);
1179 free_request:
1180 ahash_request_free(areq);
1181 free_ahash:
1182 crypto_free_ahash(tfm);
1183
1184 return ret;
1185 }
1186
1187 static int safexcel_hmac_alg_setkey(struct crypto_ahash *tfm, const u8 *key,
1188 unsigned int keylen, const char *alg,
1189 unsigned int state_sz)
1190 {
1191 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1192 struct safexcel_crypto_priv *priv = ctx->priv;
1193 struct safexcel_ahash_export_state istate, ostate;
1194 int ret;
1195
1196 ret = safexcel_hmac_setkey(alg, key, keylen, &istate, &ostate);
1197 if (ret)
1198 return ret;
1199
1200 if (priv->flags & EIP197_TRC_CACHE && ctx->base.ctxr &&
1201 (memcmp(ctx->ipad, istate.state, state_sz) ||
1202 memcmp(ctx->opad, ostate.state, state_sz)))
1203 ctx->base.needs_inv = true;
1204
1205 memcpy(ctx->ipad, &istate.state, state_sz);
1206 memcpy(ctx->opad, &ostate.state, state_sz);
1207
1208 return 0;
1209 }
1210
1211 static int safexcel_hmac_sha1_setkey(struct crypto_ahash *tfm, const u8 *key,
1212 unsigned int keylen)
1213 {
1214 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha1",
1215 SHA1_DIGEST_SIZE);
1216 }
1217
1218 struct safexcel_alg_template safexcel_alg_hmac_sha1 = {
1219 .type = SAFEXCEL_ALG_TYPE_AHASH,
1220 .algo_mask = SAFEXCEL_ALG_SHA1,
1221 .alg.ahash = {
1222 .init = safexcel_hmac_sha1_init,
1223 .update = safexcel_ahash_update,
1224 .final = safexcel_ahash_final,
1225 .finup = safexcel_ahash_finup,
1226 .digest = safexcel_hmac_sha1_digest,
1227 .setkey = safexcel_hmac_sha1_setkey,
1228 .export = safexcel_ahash_export,
1229 .import = safexcel_ahash_import,
1230 .halg = {
1231 .digestsize = SHA1_DIGEST_SIZE,
1232 .statesize = sizeof(struct safexcel_ahash_export_state),
1233 .base = {
1234 .cra_name = "hmac(sha1)",
1235 .cra_driver_name = "safexcel-hmac-sha1",
1236 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1237 .cra_flags = CRYPTO_ALG_ASYNC |
1238 CRYPTO_ALG_KERN_DRIVER_ONLY,
1239 .cra_blocksize = SHA1_BLOCK_SIZE,
1240 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1241 .cra_init = safexcel_ahash_cra_init,
1242 .cra_exit = safexcel_ahash_cra_exit,
1243 .cra_module = THIS_MODULE,
1244 },
1245 },
1246 },
1247 };
1248
1249 static int safexcel_sha256_init(struct ahash_request *areq)
1250 {
1251 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1252 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1253
1254 memset(req, 0, sizeof(*req));
1255
1256 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1257 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1258 req->state_sz = SHA256_DIGEST_SIZE;
1259 req->digest_sz = SHA256_DIGEST_SIZE;
1260 req->block_sz = SHA256_BLOCK_SIZE;
1261
1262 return 0;
1263 }
1264
1265 static int safexcel_sha256_digest(struct ahash_request *areq)
1266 {
1267 int ret = safexcel_sha256_init(areq);
1268
1269 if (ret)
1270 return ret;
1271
1272 return safexcel_ahash_finup(areq);
1273 }
1274
1275 struct safexcel_alg_template safexcel_alg_sha256 = {
1276 .type = SAFEXCEL_ALG_TYPE_AHASH,
1277 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1278 .alg.ahash = {
1279 .init = safexcel_sha256_init,
1280 .update = safexcel_ahash_update,
1281 .final = safexcel_ahash_final,
1282 .finup = safexcel_ahash_finup,
1283 .digest = safexcel_sha256_digest,
1284 .export = safexcel_ahash_export,
1285 .import = safexcel_ahash_import,
1286 .halg = {
1287 .digestsize = SHA256_DIGEST_SIZE,
1288 .statesize = sizeof(struct safexcel_ahash_export_state),
1289 .base = {
1290 .cra_name = "sha256",
1291 .cra_driver_name = "safexcel-sha256",
1292 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1293 .cra_flags = CRYPTO_ALG_ASYNC |
1294 CRYPTO_ALG_KERN_DRIVER_ONLY,
1295 .cra_blocksize = SHA256_BLOCK_SIZE,
1296 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1297 .cra_init = safexcel_ahash_cra_init,
1298 .cra_exit = safexcel_ahash_cra_exit,
1299 .cra_module = THIS_MODULE,
1300 },
1301 },
1302 },
1303 };
1304
1305 static int safexcel_sha224_init(struct ahash_request *areq)
1306 {
1307 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1308 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1309
1310 memset(req, 0, sizeof(*req));
1311
1312 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1313 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1314 req->state_sz = SHA256_DIGEST_SIZE;
1315 req->digest_sz = SHA256_DIGEST_SIZE;
1316 req->block_sz = SHA256_BLOCK_SIZE;
1317
1318 return 0;
1319 }
1320
1321 static int safexcel_sha224_digest(struct ahash_request *areq)
1322 {
1323 int ret = safexcel_sha224_init(areq);
1324
1325 if (ret)
1326 return ret;
1327
1328 return safexcel_ahash_finup(areq);
1329 }
1330
1331 struct safexcel_alg_template safexcel_alg_sha224 = {
1332 .type = SAFEXCEL_ALG_TYPE_AHASH,
1333 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1334 .alg.ahash = {
1335 .init = safexcel_sha224_init,
1336 .update = safexcel_ahash_update,
1337 .final = safexcel_ahash_final,
1338 .finup = safexcel_ahash_finup,
1339 .digest = safexcel_sha224_digest,
1340 .export = safexcel_ahash_export,
1341 .import = safexcel_ahash_import,
1342 .halg = {
1343 .digestsize = SHA224_DIGEST_SIZE,
1344 .statesize = sizeof(struct safexcel_ahash_export_state),
1345 .base = {
1346 .cra_name = "sha224",
1347 .cra_driver_name = "safexcel-sha224",
1348 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1349 .cra_flags = CRYPTO_ALG_ASYNC |
1350 CRYPTO_ALG_KERN_DRIVER_ONLY,
1351 .cra_blocksize = SHA224_BLOCK_SIZE,
1352 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1353 .cra_init = safexcel_ahash_cra_init,
1354 .cra_exit = safexcel_ahash_cra_exit,
1355 .cra_module = THIS_MODULE,
1356 },
1357 },
1358 },
1359 };
1360
1361 static int safexcel_hmac_sha224_setkey(struct crypto_ahash *tfm, const u8 *key,
1362 unsigned int keylen)
1363 {
1364 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha224",
1365 SHA256_DIGEST_SIZE);
1366 }
1367
1368 static int safexcel_hmac_sha224_init(struct ahash_request *areq)
1369 {
1370 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1371 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1372
1373 memset(req, 0, sizeof(*req));
1374
1375 /* Start from ipad precompute */
1376 memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);
1377 /* Already processed the key^ipad part now! */
1378 req->len = SHA256_BLOCK_SIZE;
1379 req->processed = SHA256_BLOCK_SIZE;
1380
1381 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA224;
1382 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1383 req->state_sz = SHA256_DIGEST_SIZE;
1384 req->digest_sz = SHA256_DIGEST_SIZE;
1385 req->block_sz = SHA256_BLOCK_SIZE;
1386 req->hmac = true;
1387
1388 return 0;
1389 }
1390
1391 static int safexcel_hmac_sha224_digest(struct ahash_request *areq)
1392 {
1393 int ret = safexcel_hmac_sha224_init(areq);
1394
1395 if (ret)
1396 return ret;
1397
1398 return safexcel_ahash_finup(areq);
1399 }
1400
1401 struct safexcel_alg_template safexcel_alg_hmac_sha224 = {
1402 .type = SAFEXCEL_ALG_TYPE_AHASH,
1403 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1404 .alg.ahash = {
1405 .init = safexcel_hmac_sha224_init,
1406 .update = safexcel_ahash_update,
1407 .final = safexcel_ahash_final,
1408 .finup = safexcel_ahash_finup,
1409 .digest = safexcel_hmac_sha224_digest,
1410 .setkey = safexcel_hmac_sha224_setkey,
1411 .export = safexcel_ahash_export,
1412 .import = safexcel_ahash_import,
1413 .halg = {
1414 .digestsize = SHA224_DIGEST_SIZE,
1415 .statesize = sizeof(struct safexcel_ahash_export_state),
1416 .base = {
1417 .cra_name = "hmac(sha224)",
1418 .cra_driver_name = "safexcel-hmac-sha224",
1419 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1420 .cra_flags = CRYPTO_ALG_ASYNC |
1421 CRYPTO_ALG_KERN_DRIVER_ONLY,
1422 .cra_blocksize = SHA224_BLOCK_SIZE,
1423 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1424 .cra_init = safexcel_ahash_cra_init,
1425 .cra_exit = safexcel_ahash_cra_exit,
1426 .cra_module = THIS_MODULE,
1427 },
1428 },
1429 },
1430 };
1431
1432 static int safexcel_hmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key,
1433 unsigned int keylen)
1434 {
1435 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha256",
1436 SHA256_DIGEST_SIZE);
1437 }
1438
1439 static int safexcel_hmac_sha256_init(struct ahash_request *areq)
1440 {
1441 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1442 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1443
1444 memset(req, 0, sizeof(*req));
1445
1446 /* Start from ipad precompute */
1447 memcpy(req->state, ctx->ipad, SHA256_DIGEST_SIZE);
1448 /* Already processed the key^ipad part now! */
1449 req->len = SHA256_BLOCK_SIZE;
1450 req->processed = SHA256_BLOCK_SIZE;
1451
1452 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA256;
1453 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1454 req->state_sz = SHA256_DIGEST_SIZE;
1455 req->digest_sz = SHA256_DIGEST_SIZE;
1456 req->block_sz = SHA256_BLOCK_SIZE;
1457 req->hmac = true;
1458
1459 return 0;
1460 }
1461
1462 static int safexcel_hmac_sha256_digest(struct ahash_request *areq)
1463 {
1464 int ret = safexcel_hmac_sha256_init(areq);
1465
1466 if (ret)
1467 return ret;
1468
1469 return safexcel_ahash_finup(areq);
1470 }
1471
1472 struct safexcel_alg_template safexcel_alg_hmac_sha256 = {
1473 .type = SAFEXCEL_ALG_TYPE_AHASH,
1474 .algo_mask = SAFEXCEL_ALG_SHA2_256,
1475 .alg.ahash = {
1476 .init = safexcel_hmac_sha256_init,
1477 .update = safexcel_ahash_update,
1478 .final = safexcel_ahash_final,
1479 .finup = safexcel_ahash_finup,
1480 .digest = safexcel_hmac_sha256_digest,
1481 .setkey = safexcel_hmac_sha256_setkey,
1482 .export = safexcel_ahash_export,
1483 .import = safexcel_ahash_import,
1484 .halg = {
1485 .digestsize = SHA256_DIGEST_SIZE,
1486 .statesize = sizeof(struct safexcel_ahash_export_state),
1487 .base = {
1488 .cra_name = "hmac(sha256)",
1489 .cra_driver_name = "safexcel-hmac-sha256",
1490 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1491 .cra_flags = CRYPTO_ALG_ASYNC |
1492 CRYPTO_ALG_KERN_DRIVER_ONLY,
1493 .cra_blocksize = SHA256_BLOCK_SIZE,
1494 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1495 .cra_init = safexcel_ahash_cra_init,
1496 .cra_exit = safexcel_ahash_cra_exit,
1497 .cra_module = THIS_MODULE,
1498 },
1499 },
1500 },
1501 };
1502
1503 static int safexcel_sha512_init(struct ahash_request *areq)
1504 {
1505 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1506 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1507
1508 memset(req, 0, sizeof(*req));
1509
1510 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1511 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1512 req->state_sz = SHA512_DIGEST_SIZE;
1513 req->digest_sz = SHA512_DIGEST_SIZE;
1514 req->block_sz = SHA512_BLOCK_SIZE;
1515
1516 return 0;
1517 }
1518
1519 static int safexcel_sha512_digest(struct ahash_request *areq)
1520 {
1521 int ret = safexcel_sha512_init(areq);
1522
1523 if (ret)
1524 return ret;
1525
1526 return safexcel_ahash_finup(areq);
1527 }
1528
1529 struct safexcel_alg_template safexcel_alg_sha512 = {
1530 .type = SAFEXCEL_ALG_TYPE_AHASH,
1531 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1532 .alg.ahash = {
1533 .init = safexcel_sha512_init,
1534 .update = safexcel_ahash_update,
1535 .final = safexcel_ahash_final,
1536 .finup = safexcel_ahash_finup,
1537 .digest = safexcel_sha512_digest,
1538 .export = safexcel_ahash_export,
1539 .import = safexcel_ahash_import,
1540 .halg = {
1541 .digestsize = SHA512_DIGEST_SIZE,
1542 .statesize = sizeof(struct safexcel_ahash_export_state),
1543 .base = {
1544 .cra_name = "sha512",
1545 .cra_driver_name = "safexcel-sha512",
1546 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1547 .cra_flags = CRYPTO_ALG_ASYNC |
1548 CRYPTO_ALG_KERN_DRIVER_ONLY,
1549 .cra_blocksize = SHA512_BLOCK_SIZE,
1550 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1551 .cra_init = safexcel_ahash_cra_init,
1552 .cra_exit = safexcel_ahash_cra_exit,
1553 .cra_module = THIS_MODULE,
1554 },
1555 },
1556 },
1557 };
1558
1559 static int safexcel_sha384_init(struct ahash_request *areq)
1560 {
1561 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1562 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1563
1564 memset(req, 0, sizeof(*req));
1565
1566 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1567 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1568 req->state_sz = SHA512_DIGEST_SIZE;
1569 req->digest_sz = SHA512_DIGEST_SIZE;
1570 req->block_sz = SHA512_BLOCK_SIZE;
1571
1572 return 0;
1573 }
1574
1575 static int safexcel_sha384_digest(struct ahash_request *areq)
1576 {
1577 int ret = safexcel_sha384_init(areq);
1578
1579 if (ret)
1580 return ret;
1581
1582 return safexcel_ahash_finup(areq);
1583 }
1584
1585 struct safexcel_alg_template safexcel_alg_sha384 = {
1586 .type = SAFEXCEL_ALG_TYPE_AHASH,
1587 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1588 .alg.ahash = {
1589 .init = safexcel_sha384_init,
1590 .update = safexcel_ahash_update,
1591 .final = safexcel_ahash_final,
1592 .finup = safexcel_ahash_finup,
1593 .digest = safexcel_sha384_digest,
1594 .export = safexcel_ahash_export,
1595 .import = safexcel_ahash_import,
1596 .halg = {
1597 .digestsize = SHA384_DIGEST_SIZE,
1598 .statesize = sizeof(struct safexcel_ahash_export_state),
1599 .base = {
1600 .cra_name = "sha384",
1601 .cra_driver_name = "safexcel-sha384",
1602 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1603 .cra_flags = CRYPTO_ALG_ASYNC |
1604 CRYPTO_ALG_KERN_DRIVER_ONLY,
1605 .cra_blocksize = SHA384_BLOCK_SIZE,
1606 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1607 .cra_init = safexcel_ahash_cra_init,
1608 .cra_exit = safexcel_ahash_cra_exit,
1609 .cra_module = THIS_MODULE,
1610 },
1611 },
1612 },
1613 };
1614
1615 static int safexcel_hmac_sha512_setkey(struct crypto_ahash *tfm, const u8 *key,
1616 unsigned int keylen)
1617 {
1618 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha512",
1619 SHA512_DIGEST_SIZE);
1620 }
1621
1622 static int safexcel_hmac_sha512_init(struct ahash_request *areq)
1623 {
1624 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1625 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1626
1627 memset(req, 0, sizeof(*req));
1628
1629 /* Start from ipad precompute */
1630 memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);
1631 /* Already processed the key^ipad part now! */
1632 req->len = SHA512_BLOCK_SIZE;
1633 req->processed = SHA512_BLOCK_SIZE;
1634
1635 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA512;
1636 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1637 req->state_sz = SHA512_DIGEST_SIZE;
1638 req->digest_sz = SHA512_DIGEST_SIZE;
1639 req->block_sz = SHA512_BLOCK_SIZE;
1640 req->hmac = true;
1641
1642 return 0;
1643 }
1644
1645 static int safexcel_hmac_sha512_digest(struct ahash_request *areq)
1646 {
1647 int ret = safexcel_hmac_sha512_init(areq);
1648
1649 if (ret)
1650 return ret;
1651
1652 return safexcel_ahash_finup(areq);
1653 }
1654
1655 struct safexcel_alg_template safexcel_alg_hmac_sha512 = {
1656 .type = SAFEXCEL_ALG_TYPE_AHASH,
1657 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1658 .alg.ahash = {
1659 .init = safexcel_hmac_sha512_init,
1660 .update = safexcel_ahash_update,
1661 .final = safexcel_ahash_final,
1662 .finup = safexcel_ahash_finup,
1663 .digest = safexcel_hmac_sha512_digest,
1664 .setkey = safexcel_hmac_sha512_setkey,
1665 .export = safexcel_ahash_export,
1666 .import = safexcel_ahash_import,
1667 .halg = {
1668 .digestsize = SHA512_DIGEST_SIZE,
1669 .statesize = sizeof(struct safexcel_ahash_export_state),
1670 .base = {
1671 .cra_name = "hmac(sha512)",
1672 .cra_driver_name = "safexcel-hmac-sha512",
1673 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1674 .cra_flags = CRYPTO_ALG_ASYNC |
1675 CRYPTO_ALG_KERN_DRIVER_ONLY,
1676 .cra_blocksize = SHA512_BLOCK_SIZE,
1677 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1678 .cra_init = safexcel_ahash_cra_init,
1679 .cra_exit = safexcel_ahash_cra_exit,
1680 .cra_module = THIS_MODULE,
1681 },
1682 },
1683 },
1684 };
1685
1686 static int safexcel_hmac_sha384_setkey(struct crypto_ahash *tfm, const u8 *key,
1687 unsigned int keylen)
1688 {
1689 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sha384",
1690 SHA512_DIGEST_SIZE);
1691 }
1692
1693 static int safexcel_hmac_sha384_init(struct ahash_request *areq)
1694 {
1695 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1696 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1697
1698 memset(req, 0, sizeof(*req));
1699
1700 /* Start from ipad precompute */
1701 memcpy(req->state, ctx->ipad, SHA512_DIGEST_SIZE);
1702 /* Already processed the key^ipad part now! */
1703 req->len = SHA512_BLOCK_SIZE;
1704 req->processed = SHA512_BLOCK_SIZE;
1705
1706 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA384;
1707 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1708 req->state_sz = SHA512_DIGEST_SIZE;
1709 req->digest_sz = SHA512_DIGEST_SIZE;
1710 req->block_sz = SHA512_BLOCK_SIZE;
1711 req->hmac = true;
1712
1713 return 0;
1714 }
1715
1716 static int safexcel_hmac_sha384_digest(struct ahash_request *areq)
1717 {
1718 int ret = safexcel_hmac_sha384_init(areq);
1719
1720 if (ret)
1721 return ret;
1722
1723 return safexcel_ahash_finup(areq);
1724 }
1725
1726 struct safexcel_alg_template safexcel_alg_hmac_sha384 = {
1727 .type = SAFEXCEL_ALG_TYPE_AHASH,
1728 .algo_mask = SAFEXCEL_ALG_SHA2_512,
1729 .alg.ahash = {
1730 .init = safexcel_hmac_sha384_init,
1731 .update = safexcel_ahash_update,
1732 .final = safexcel_ahash_final,
1733 .finup = safexcel_ahash_finup,
1734 .digest = safexcel_hmac_sha384_digest,
1735 .setkey = safexcel_hmac_sha384_setkey,
1736 .export = safexcel_ahash_export,
1737 .import = safexcel_ahash_import,
1738 .halg = {
1739 .digestsize = SHA384_DIGEST_SIZE,
1740 .statesize = sizeof(struct safexcel_ahash_export_state),
1741 .base = {
1742 .cra_name = "hmac(sha384)",
1743 .cra_driver_name = "safexcel-hmac-sha384",
1744 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1745 .cra_flags = CRYPTO_ALG_ASYNC |
1746 CRYPTO_ALG_KERN_DRIVER_ONLY,
1747 .cra_blocksize = SHA384_BLOCK_SIZE,
1748 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1749 .cra_init = safexcel_ahash_cra_init,
1750 .cra_exit = safexcel_ahash_cra_exit,
1751 .cra_module = THIS_MODULE,
1752 },
1753 },
1754 },
1755 };
1756
1757 static int safexcel_md5_init(struct ahash_request *areq)
1758 {
1759 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1760 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1761
1762 memset(req, 0, sizeof(*req));
1763
1764 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1765 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1766 req->state_sz = MD5_DIGEST_SIZE;
1767 req->digest_sz = MD5_DIGEST_SIZE;
1768 req->block_sz = MD5_HMAC_BLOCK_SIZE;
1769
1770 return 0;
1771 }
1772
1773 static int safexcel_md5_digest(struct ahash_request *areq)
1774 {
1775 int ret = safexcel_md5_init(areq);
1776
1777 if (ret)
1778 return ret;
1779
1780 return safexcel_ahash_finup(areq);
1781 }
1782
1783 struct safexcel_alg_template safexcel_alg_md5 = {
1784 .type = SAFEXCEL_ALG_TYPE_AHASH,
1785 .algo_mask = SAFEXCEL_ALG_MD5,
1786 .alg.ahash = {
1787 .init = safexcel_md5_init,
1788 .update = safexcel_ahash_update,
1789 .final = safexcel_ahash_final,
1790 .finup = safexcel_ahash_finup,
1791 .digest = safexcel_md5_digest,
1792 .export = safexcel_ahash_export,
1793 .import = safexcel_ahash_import,
1794 .halg = {
1795 .digestsize = MD5_DIGEST_SIZE,
1796 .statesize = sizeof(struct safexcel_ahash_export_state),
1797 .base = {
1798 .cra_name = "md5",
1799 .cra_driver_name = "safexcel-md5",
1800 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1801 .cra_flags = CRYPTO_ALG_ASYNC |
1802 CRYPTO_ALG_KERN_DRIVER_ONLY,
1803 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1804 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1805 .cra_init = safexcel_ahash_cra_init,
1806 .cra_exit = safexcel_ahash_cra_exit,
1807 .cra_module = THIS_MODULE,
1808 },
1809 },
1810 },
1811 };
1812
1813 static int safexcel_hmac_md5_init(struct ahash_request *areq)
1814 {
1815 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1816 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1817
1818 memset(req, 0, sizeof(*req));
1819
1820 /* Start from ipad precompute */
1821 memcpy(req->state, ctx->ipad, MD5_DIGEST_SIZE);
1822 /* Already processed the key^ipad part now! */
1823 req->len = MD5_HMAC_BLOCK_SIZE;
1824 req->processed = MD5_HMAC_BLOCK_SIZE;
1825
1826 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_MD5;
1827 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
1828 req->state_sz = MD5_DIGEST_SIZE;
1829 req->digest_sz = MD5_DIGEST_SIZE;
1830 req->block_sz = MD5_HMAC_BLOCK_SIZE;
1831 req->len_is_le = true; /* MD5 is little endian! ... */
1832 req->hmac = true;
1833
1834 return 0;
1835 }
1836
1837 static int safexcel_hmac_md5_setkey(struct crypto_ahash *tfm, const u8 *key,
1838 unsigned int keylen)
1839 {
1840 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-md5",
1841 MD5_DIGEST_SIZE);
1842 }
1843
1844 static int safexcel_hmac_md5_digest(struct ahash_request *areq)
1845 {
1846 int ret = safexcel_hmac_md5_init(areq);
1847
1848 if (ret)
1849 return ret;
1850
1851 return safexcel_ahash_finup(areq);
1852 }
1853
1854 struct safexcel_alg_template safexcel_alg_hmac_md5 = {
1855 .type = SAFEXCEL_ALG_TYPE_AHASH,
1856 .algo_mask = SAFEXCEL_ALG_MD5,
1857 .alg.ahash = {
1858 .init = safexcel_hmac_md5_init,
1859 .update = safexcel_ahash_update,
1860 .final = safexcel_ahash_final,
1861 .finup = safexcel_ahash_finup,
1862 .digest = safexcel_hmac_md5_digest,
1863 .setkey = safexcel_hmac_md5_setkey,
1864 .export = safexcel_ahash_export,
1865 .import = safexcel_ahash_import,
1866 .halg = {
1867 .digestsize = MD5_DIGEST_SIZE,
1868 .statesize = sizeof(struct safexcel_ahash_export_state),
1869 .base = {
1870 .cra_name = "hmac(md5)",
1871 .cra_driver_name = "safexcel-hmac-md5",
1872 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1873 .cra_flags = CRYPTO_ALG_ASYNC |
1874 CRYPTO_ALG_KERN_DRIVER_ONLY,
1875 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
1876 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1877 .cra_init = safexcel_ahash_cra_init,
1878 .cra_exit = safexcel_ahash_cra_exit,
1879 .cra_module = THIS_MODULE,
1880 },
1881 },
1882 },
1883 };
1884
1885 static int safexcel_crc32_cra_init(struct crypto_tfm *tfm)
1886 {
1887 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
1888 int ret = safexcel_ahash_cra_init(tfm);
1889
1890 /* Default 'key' is all zeroes */
1891 memset(ctx->ipad, 0, sizeof(u32));
1892 return ret;
1893 }
1894
1895 static int safexcel_crc32_init(struct ahash_request *areq)
1896 {
1897 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1898 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1899
1900 memset(req, 0, sizeof(*req));
1901
1902 /* Start from loaded key */
1903 req->state[0] = (__force __le32)le32_to_cpu(~ctx->ipad[0]);
1904 /* Set processed to non-zero to enable invalidation detection */
1905 req->len = sizeof(u32);
1906 req->processed = sizeof(u32);
1907
1908 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_CRC32;
1909 req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1910 req->state_sz = sizeof(u32);
1911 req->digest_sz = sizeof(u32);
1912 req->block_sz = sizeof(u32);
1913
1914 return 0;
1915 }
1916
1917 static int safexcel_crc32_setkey(struct crypto_ahash *tfm, const u8 *key,
1918 unsigned int keylen)
1919 {
1920 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1921
1922 if (keylen != sizeof(u32))
1923 return -EINVAL;
1924
1925 memcpy(ctx->ipad, key, sizeof(u32));
1926 return 0;
1927 }
1928
1929 static int safexcel_crc32_digest(struct ahash_request *areq)
1930 {
1931 return safexcel_crc32_init(areq) ?: safexcel_ahash_finup(areq);
1932 }
1933
1934 struct safexcel_alg_template safexcel_alg_crc32 = {
1935 .type = SAFEXCEL_ALG_TYPE_AHASH,
1936 .algo_mask = 0,
1937 .alg.ahash = {
1938 .init = safexcel_crc32_init,
1939 .update = safexcel_ahash_update,
1940 .final = safexcel_ahash_final,
1941 .finup = safexcel_ahash_finup,
1942 .digest = safexcel_crc32_digest,
1943 .setkey = safexcel_crc32_setkey,
1944 .export = safexcel_ahash_export,
1945 .import = safexcel_ahash_import,
1946 .halg = {
1947 .digestsize = sizeof(u32),
1948 .statesize = sizeof(struct safexcel_ahash_export_state),
1949 .base = {
1950 .cra_name = "crc32",
1951 .cra_driver_name = "safexcel-crc32",
1952 .cra_priority = SAFEXCEL_CRA_PRIORITY,
1953 .cra_flags = CRYPTO_ALG_OPTIONAL_KEY |
1954 CRYPTO_ALG_ASYNC |
1955 CRYPTO_ALG_KERN_DRIVER_ONLY,
1956 .cra_blocksize = 1,
1957 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
1958 .cra_init = safexcel_crc32_cra_init,
1959 .cra_exit = safexcel_ahash_cra_exit,
1960 .cra_module = THIS_MODULE,
1961 },
1962 },
1963 },
1964 };
1965
1966 static int safexcel_cbcmac_init(struct ahash_request *areq)
1967 {
1968 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
1969 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
1970
1971 memset(req, 0, sizeof(*req));
1972
1973 /* Start from loaded keys */
1974 memcpy(req->state, ctx->ipad, ctx->key_sz);
1975 /* Set processed to non-zero to enable invalidation detection */
1976 req->len = AES_BLOCK_SIZE;
1977 req->processed = AES_BLOCK_SIZE;
1978
1979 req->digest = CONTEXT_CONTROL_DIGEST_XCM;
1980 req->state_sz = ctx->key_sz;
1981 req->digest_sz = AES_BLOCK_SIZE;
1982 req->block_sz = AES_BLOCK_SIZE;
1983 req->xcbcmac = true;
1984
1985 return 0;
1986 }
1987
1988 static int safexcel_cbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
1989 unsigned int len)
1990 {
1991 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1992 struct crypto_aes_ctx aes;
1993 int ret, i;
1994
1995 ret = aes_expandkey(&aes, key, len);
1996 if (ret)
1997 return ret;
1998
1999 memset(ctx->ipad, 0, 2 * AES_BLOCK_SIZE);
2000 for (i = 0; i < len / sizeof(u32); i++)
2001 ctx->ipad[i + 8] = (__force __le32)cpu_to_be32(aes.key_enc[i]);
2002
2003 if (len == AES_KEYSIZE_192) {
2004 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2005 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2006 } else if (len == AES_KEYSIZE_256) {
2007 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2008 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2009 } else {
2010 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2011 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2012 }
2013 ctx->cbcmac = true;
2014
2015 memzero_explicit(&aes, sizeof(aes));
2016 return 0;
2017 }
2018
2019 static int safexcel_cbcmac_digest(struct ahash_request *areq)
2020 {
2021 return safexcel_cbcmac_init(areq) ?: safexcel_ahash_finup(areq);
2022 }
2023
2024 struct safexcel_alg_template safexcel_alg_cbcmac = {
2025 .type = SAFEXCEL_ALG_TYPE_AHASH,
2026 .algo_mask = 0,
2027 .alg.ahash = {
2028 .init = safexcel_cbcmac_init,
2029 .update = safexcel_ahash_update,
2030 .final = safexcel_ahash_final,
2031 .finup = safexcel_ahash_finup,
2032 .digest = safexcel_cbcmac_digest,
2033 .setkey = safexcel_cbcmac_setkey,
2034 .export = safexcel_ahash_export,
2035 .import = safexcel_ahash_import,
2036 .halg = {
2037 .digestsize = AES_BLOCK_SIZE,
2038 .statesize = sizeof(struct safexcel_ahash_export_state),
2039 .base = {
2040 .cra_name = "cbcmac(aes)",
2041 .cra_driver_name = "safexcel-cbcmac-aes",
2042 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2043 .cra_flags = CRYPTO_ALG_ASYNC |
2044 CRYPTO_ALG_KERN_DRIVER_ONLY,
2045 .cra_blocksize = 1,
2046 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2047 .cra_init = safexcel_ahash_cra_init,
2048 .cra_exit = safexcel_ahash_cra_exit,
2049 .cra_module = THIS_MODULE,
2050 },
2051 },
2052 },
2053 };
2054
2055 static int safexcel_xcbcmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2056 unsigned int len)
2057 {
2058 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2059 struct crypto_aes_ctx aes;
2060 u32 key_tmp[3 * AES_BLOCK_SIZE / sizeof(u32)];
2061 int ret, i;
2062
2063 ret = aes_expandkey(&aes, key, len);
2064 if (ret)
2065 return ret;
2066
2067 /* precompute the XCBC key material */
2068 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2069 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2070 CRYPTO_TFM_REQ_MASK);
2071 ret = crypto_cipher_setkey(ctx->kaes, key, len);
2072 if (ret)
2073 return ret;
2074
2075 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2076 "\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1\x1");
2077 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp,
2078 "\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2\x2");
2079 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)key_tmp + AES_BLOCK_SIZE,
2080 "\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3\x3");
2081 for (i = 0; i < 3 * AES_BLOCK_SIZE / sizeof(u32); i++)
2082 ctx->ipad[i] =
2083 cpu_to_le32((__force u32)cpu_to_be32(key_tmp[i]));
2084
2085 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2086 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2087 CRYPTO_TFM_REQ_MASK);
2088 ret = crypto_cipher_setkey(ctx->kaes,
2089 (u8 *)key_tmp + 2 * AES_BLOCK_SIZE,
2090 AES_MIN_KEY_SIZE);
2091 if (ret)
2092 return ret;
2093
2094 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2095 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2096 ctx->cbcmac = false;
2097
2098 memzero_explicit(&aes, sizeof(aes));
2099 return 0;
2100 }
2101
2102 static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm)
2103 {
2104 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2105
2106 safexcel_ahash_cra_init(tfm);
2107 ctx->kaes = crypto_alloc_cipher("aes", 0, 0);
2108 return PTR_ERR_OR_ZERO(ctx->kaes);
2109 }
2110
2111 static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm)
2112 {
2113 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2114
2115 crypto_free_cipher(ctx->kaes);
2116 safexcel_ahash_cra_exit(tfm);
2117 }
2118
2119 struct safexcel_alg_template safexcel_alg_xcbcmac = {
2120 .type = SAFEXCEL_ALG_TYPE_AHASH,
2121 .algo_mask = 0,
2122 .alg.ahash = {
2123 .init = safexcel_cbcmac_init,
2124 .update = safexcel_ahash_update,
2125 .final = safexcel_ahash_final,
2126 .finup = safexcel_ahash_finup,
2127 .digest = safexcel_cbcmac_digest,
2128 .setkey = safexcel_xcbcmac_setkey,
2129 .export = safexcel_ahash_export,
2130 .import = safexcel_ahash_import,
2131 .halg = {
2132 .digestsize = AES_BLOCK_SIZE,
2133 .statesize = sizeof(struct safexcel_ahash_export_state),
2134 .base = {
2135 .cra_name = "xcbc(aes)",
2136 .cra_driver_name = "safexcel-xcbc-aes",
2137 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2138 .cra_flags = CRYPTO_ALG_ASYNC |
2139 CRYPTO_ALG_KERN_DRIVER_ONLY,
2140 .cra_blocksize = AES_BLOCK_SIZE,
2141 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2142 .cra_init = safexcel_xcbcmac_cra_init,
2143 .cra_exit = safexcel_xcbcmac_cra_exit,
2144 .cra_module = THIS_MODULE,
2145 },
2146 },
2147 },
2148 };
2149
2150 static int safexcel_cmac_setkey(struct crypto_ahash *tfm, const u8 *key,
2151 unsigned int len)
2152 {
2153 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2154 struct crypto_aes_ctx aes;
2155 __be64 consts[4];
2156 u64 _const[2];
2157 u8 msb_mask, gfmask;
2158 int ret, i;
2159
2160 ret = aes_expandkey(&aes, key, len);
2161 if (ret)
2162 return ret;
2163
2164 for (i = 0; i < len / sizeof(u32); i++)
2165 ctx->ipad[i + 8] =
2166 cpu_to_le32((__force u32)cpu_to_be32(aes.key_enc[i]));
2167
2168 /* precompute the CMAC key material */
2169 crypto_cipher_clear_flags(ctx->kaes, CRYPTO_TFM_REQ_MASK);
2170 crypto_cipher_set_flags(ctx->kaes, crypto_ahash_get_flags(tfm) &
2171 CRYPTO_TFM_REQ_MASK);
2172 ret = crypto_cipher_setkey(ctx->kaes, key, len);
2173 if (ret)
2174 return ret;
2175
2176 /* code below borrowed from crypto/cmac.c */
2177 /* encrypt the zero block */
2178 memset(consts, 0, AES_BLOCK_SIZE);
2179 crypto_cipher_encrypt_one(ctx->kaes, (u8 *)consts, (u8 *)consts);
2180
2181 gfmask = 0x87;
2182 _const[0] = be64_to_cpu(consts[1]);
2183 _const[1] = be64_to_cpu(consts[0]);
2184
2185 /* gf(2^128) multiply zero-ciphertext with u and u^2 */
2186 for (i = 0; i < 4; i += 2) {
2187 msb_mask = ((s64)_const[1] >> 63) & gfmask;
2188 _const[1] = (_const[1] << 1) | (_const[0] >> 63);
2189 _const[0] = (_const[0] << 1) ^ msb_mask;
2190
2191 consts[i + 0] = cpu_to_be64(_const[1]);
2192 consts[i + 1] = cpu_to_be64(_const[0]);
2193 }
2194 /* end of code borrowed from crypto/cmac.c */
2195
2196 for (i = 0; i < 2 * AES_BLOCK_SIZE / sizeof(u32); i++)
2197 ctx->ipad[i] = (__force __le32)cpu_to_be32(((u32 *)consts)[i]);
2198
2199 if (len == AES_KEYSIZE_192) {
2200 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC192;
2201 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2202 } else if (len == AES_KEYSIZE_256) {
2203 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC256;
2204 ctx->key_sz = AES_MAX_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2205 } else {
2206 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_XCBC128;
2207 ctx->key_sz = AES_MIN_KEY_SIZE + 2 * AES_BLOCK_SIZE;
2208 }
2209 ctx->cbcmac = false;
2210
2211 memzero_explicit(&aes, sizeof(aes));
2212 return 0;
2213 }
2214
2215 struct safexcel_alg_template safexcel_alg_cmac = {
2216 .type = SAFEXCEL_ALG_TYPE_AHASH,
2217 .algo_mask = 0,
2218 .alg.ahash = {
2219 .init = safexcel_cbcmac_init,
2220 .update = safexcel_ahash_update,
2221 .final = safexcel_ahash_final,
2222 .finup = safexcel_ahash_finup,
2223 .digest = safexcel_cbcmac_digest,
2224 .setkey = safexcel_cmac_setkey,
2225 .export = safexcel_ahash_export,
2226 .import = safexcel_ahash_import,
2227 .halg = {
2228 .digestsize = AES_BLOCK_SIZE,
2229 .statesize = sizeof(struct safexcel_ahash_export_state),
2230 .base = {
2231 .cra_name = "cmac(aes)",
2232 .cra_driver_name = "safexcel-cmac-aes",
2233 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2234 .cra_flags = CRYPTO_ALG_ASYNC |
2235 CRYPTO_ALG_KERN_DRIVER_ONLY,
2236 .cra_blocksize = AES_BLOCK_SIZE,
2237 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2238 .cra_init = safexcel_xcbcmac_cra_init,
2239 .cra_exit = safexcel_xcbcmac_cra_exit,
2240 .cra_module = THIS_MODULE,
2241 },
2242 },
2243 },
2244 };
2245
2246 static int safexcel_sm3_init(struct ahash_request *areq)
2247 {
2248 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2249 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2250
2251 memset(req, 0, sizeof(*req));
2252
2253 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2254 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2255 req->state_sz = SM3_DIGEST_SIZE;
2256 req->digest_sz = SM3_DIGEST_SIZE;
2257 req->block_sz = SM3_BLOCK_SIZE;
2258
2259 return 0;
2260 }
2261
2262 static int safexcel_sm3_digest(struct ahash_request *areq)
2263 {
2264 int ret = safexcel_sm3_init(areq);
2265
2266 if (ret)
2267 return ret;
2268
2269 return safexcel_ahash_finup(areq);
2270 }
2271
2272 struct safexcel_alg_template safexcel_alg_sm3 = {
2273 .type = SAFEXCEL_ALG_TYPE_AHASH,
2274 .algo_mask = SAFEXCEL_ALG_SM3,
2275 .alg.ahash = {
2276 .init = safexcel_sm3_init,
2277 .update = safexcel_ahash_update,
2278 .final = safexcel_ahash_final,
2279 .finup = safexcel_ahash_finup,
2280 .digest = safexcel_sm3_digest,
2281 .export = safexcel_ahash_export,
2282 .import = safexcel_ahash_import,
2283 .halg = {
2284 .digestsize = SM3_DIGEST_SIZE,
2285 .statesize = sizeof(struct safexcel_ahash_export_state),
2286 .base = {
2287 .cra_name = "sm3",
2288 .cra_driver_name = "safexcel-sm3",
2289 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2290 .cra_flags = CRYPTO_ALG_ASYNC |
2291 CRYPTO_ALG_KERN_DRIVER_ONLY,
2292 .cra_blocksize = SM3_BLOCK_SIZE,
2293 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2294 .cra_init = safexcel_ahash_cra_init,
2295 .cra_exit = safexcel_ahash_cra_exit,
2296 .cra_module = THIS_MODULE,
2297 },
2298 },
2299 },
2300 };
2301
2302 static int safexcel_hmac_sm3_setkey(struct crypto_ahash *tfm, const u8 *key,
2303 unsigned int keylen)
2304 {
2305 return safexcel_hmac_alg_setkey(tfm, key, keylen, "safexcel-sm3",
2306 SM3_DIGEST_SIZE);
2307 }
2308
2309 static int safexcel_hmac_sm3_init(struct ahash_request *areq)
2310 {
2311 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(areq));
2312 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2313
2314 memset(req, 0, sizeof(*req));
2315
2316 /* Start from ipad precompute */
2317 memcpy(req->state, ctx->ipad, SM3_DIGEST_SIZE);
2318 /* Already processed the key^ipad part now! */
2319 req->len = SM3_BLOCK_SIZE;
2320 req->processed = SM3_BLOCK_SIZE;
2321
2322 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SM3;
2323 req->digest = CONTEXT_CONTROL_DIGEST_PRECOMPUTED;
2324 req->state_sz = SM3_DIGEST_SIZE;
2325 req->digest_sz = SM3_DIGEST_SIZE;
2326 req->block_sz = SM3_BLOCK_SIZE;
2327 req->hmac = true;
2328
2329 return 0;
2330 }
2331
2332 static int safexcel_hmac_sm3_digest(struct ahash_request *areq)
2333 {
2334 int ret = safexcel_hmac_sm3_init(areq);
2335
2336 if (ret)
2337 return ret;
2338
2339 return safexcel_ahash_finup(areq);
2340 }
2341
2342 struct safexcel_alg_template safexcel_alg_hmac_sm3 = {
2343 .type = SAFEXCEL_ALG_TYPE_AHASH,
2344 .algo_mask = SAFEXCEL_ALG_SM3,
2345 .alg.ahash = {
2346 .init = safexcel_hmac_sm3_init,
2347 .update = safexcel_ahash_update,
2348 .final = safexcel_ahash_final,
2349 .finup = safexcel_ahash_finup,
2350 .digest = safexcel_hmac_sm3_digest,
2351 .setkey = safexcel_hmac_sm3_setkey,
2352 .export = safexcel_ahash_export,
2353 .import = safexcel_ahash_import,
2354 .halg = {
2355 .digestsize = SM3_DIGEST_SIZE,
2356 .statesize = sizeof(struct safexcel_ahash_export_state),
2357 .base = {
2358 .cra_name = "hmac(sm3)",
2359 .cra_driver_name = "safexcel-hmac-sm3",
2360 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2361 .cra_flags = CRYPTO_ALG_ASYNC |
2362 CRYPTO_ALG_KERN_DRIVER_ONLY,
2363 .cra_blocksize = SM3_BLOCK_SIZE,
2364 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2365 .cra_init = safexcel_ahash_cra_init,
2366 .cra_exit = safexcel_ahash_cra_exit,
2367 .cra_module = THIS_MODULE,
2368 },
2369 },
2370 },
2371 };
2372
2373 static int safexcel_sha3_224_init(struct ahash_request *areq)
2374 {
2375 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2376 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2377 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2378
2379 memset(req, 0, sizeof(*req));
2380
2381 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2382 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2383 req->state_sz = SHA3_224_DIGEST_SIZE;
2384 req->digest_sz = SHA3_224_DIGEST_SIZE;
2385 req->block_sz = SHA3_224_BLOCK_SIZE;
2386 ctx->do_fallback = false;
2387 ctx->fb_init_done = false;
2388 return 0;
2389 }
2390
2391 static int safexcel_sha3_fbcheck(struct ahash_request *req)
2392 {
2393 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2394 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2395 struct ahash_request *subreq = ahash_request_ctx(req);
2396 int ret = 0;
2397
2398 if (ctx->do_fallback) {
2399 ahash_request_set_tfm(subreq, ctx->fback);
2400 ahash_request_set_callback(subreq, req->base.flags,
2401 req->base.complete, req->base.data);
2402 ahash_request_set_crypt(subreq, req->src, req->result,
2403 req->nbytes);
2404 if (!ctx->fb_init_done) {
2405 if (ctx->fb_do_setkey) {
2406 /* Set fallback cipher HMAC key */
2407 u8 key[SHA3_224_BLOCK_SIZE];
2408
2409 memcpy(key, ctx->ipad,
2410 crypto_ahash_blocksize(ctx->fback) / 2);
2411 memcpy(key +
2412 crypto_ahash_blocksize(ctx->fback) / 2,
2413 ctx->opad,
2414 crypto_ahash_blocksize(ctx->fback) / 2);
2415 ret = crypto_ahash_setkey(ctx->fback, key,
2416 crypto_ahash_blocksize(ctx->fback));
2417 memzero_explicit(key,
2418 crypto_ahash_blocksize(ctx->fback));
2419 ctx->fb_do_setkey = false;
2420 }
2421 ret = ret ?: crypto_ahash_init(subreq);
2422 ctx->fb_init_done = true;
2423 }
2424 }
2425 return ret;
2426 }
2427
2428 static int safexcel_sha3_update(struct ahash_request *req)
2429 {
2430 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2431 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2432 struct ahash_request *subreq = ahash_request_ctx(req);
2433
2434 ctx->do_fallback = true;
2435 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_update(subreq);
2436 }
2437
2438 static int safexcel_sha3_final(struct ahash_request *req)
2439 {
2440 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2441 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2442 struct ahash_request *subreq = ahash_request_ctx(req);
2443
2444 ctx->do_fallback = true;
2445 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_final(subreq);
2446 }
2447
2448 static int safexcel_sha3_finup(struct ahash_request *req)
2449 {
2450 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2451 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2452 struct ahash_request *subreq = ahash_request_ctx(req);
2453
2454 ctx->do_fallback |= !req->nbytes;
2455 if (ctx->do_fallback)
2456 /* Update or ex/import happened or len 0, cannot use the HW */
2457 return safexcel_sha3_fbcheck(req) ?:
2458 crypto_ahash_finup(subreq);
2459 else
2460 return safexcel_ahash_finup(req);
2461 }
2462
2463 static int safexcel_sha3_digest_fallback(struct ahash_request *req)
2464 {
2465 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2466 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2467 struct ahash_request *subreq = ahash_request_ctx(req);
2468
2469 ctx->do_fallback = true;
2470 ctx->fb_init_done = false;
2471 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_finup(subreq);
2472 }
2473
2474 static int safexcel_sha3_224_digest(struct ahash_request *req)
2475 {
2476 if (req->nbytes)
2477 return safexcel_sha3_224_init(req) ?: safexcel_ahash_finup(req);
2478
2479 /* HW cannot do zero length hash, use fallback instead */
2480 return safexcel_sha3_digest_fallback(req);
2481 }
2482
2483 static int safexcel_sha3_export(struct ahash_request *req, void *out)
2484 {
2485 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2486 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2487 struct ahash_request *subreq = ahash_request_ctx(req);
2488
2489 ctx->do_fallback = true;
2490 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_export(subreq, out);
2491 }
2492
2493 static int safexcel_sha3_import(struct ahash_request *req, const void *in)
2494 {
2495 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2496 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2497 struct ahash_request *subreq = ahash_request_ctx(req);
2498
2499 ctx->do_fallback = true;
2500 return safexcel_sha3_fbcheck(req) ?: crypto_ahash_import(subreq, in);
2501 // return safexcel_ahash_import(req, in);
2502 }
2503
2504 static int safexcel_sha3_cra_init(struct crypto_tfm *tfm)
2505 {
2506 struct crypto_ahash *ahash = __crypto_ahash_cast(tfm);
2507 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2508
2509 safexcel_ahash_cra_init(tfm);
2510
2511 /* Allocate fallback implementation */
2512 ctx->fback = crypto_alloc_ahash(crypto_tfm_alg_name(tfm), 0,
2513 CRYPTO_ALG_ASYNC |
2514 CRYPTO_ALG_NEED_FALLBACK);
2515 if (IS_ERR(ctx->fback))
2516 return PTR_ERR(ctx->fback);
2517
2518 /* Update statesize from fallback algorithm! */
2519 crypto_hash_alg_common(ahash)->statesize =
2520 crypto_ahash_statesize(ctx->fback);
2521 crypto_ahash_set_reqsize(ahash, max(sizeof(struct safexcel_ahash_req),
2522 sizeof(struct ahash_request) +
2523 crypto_ahash_reqsize(ctx->fback)));
2524 return 0;
2525 }
2526
2527 static void safexcel_sha3_cra_exit(struct crypto_tfm *tfm)
2528 {
2529 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2530
2531 crypto_free_ahash(ctx->fback);
2532 safexcel_ahash_cra_exit(tfm);
2533 }
2534
2535 struct safexcel_alg_template safexcel_alg_sha3_224 = {
2536 .type = SAFEXCEL_ALG_TYPE_AHASH,
2537 .algo_mask = SAFEXCEL_ALG_SHA3,
2538 .alg.ahash = {
2539 .init = safexcel_sha3_224_init,
2540 .update = safexcel_sha3_update,
2541 .final = safexcel_sha3_final,
2542 .finup = safexcel_sha3_finup,
2543 .digest = safexcel_sha3_224_digest,
2544 .export = safexcel_sha3_export,
2545 .import = safexcel_sha3_import,
2546 .halg = {
2547 .digestsize = SHA3_224_DIGEST_SIZE,
2548 .statesize = sizeof(struct safexcel_ahash_export_state),
2549 .base = {
2550 .cra_name = "sha3-224",
2551 .cra_driver_name = "safexcel-sha3-224",
2552 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2553 .cra_flags = CRYPTO_ALG_ASYNC |
2554 CRYPTO_ALG_KERN_DRIVER_ONLY |
2555 CRYPTO_ALG_NEED_FALLBACK,
2556 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2557 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2558 .cra_init = safexcel_sha3_cra_init,
2559 .cra_exit = safexcel_sha3_cra_exit,
2560 .cra_module = THIS_MODULE,
2561 },
2562 },
2563 },
2564 };
2565
2566 static int safexcel_sha3_256_init(struct ahash_request *areq)
2567 {
2568 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2569 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2570 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2571
2572 memset(req, 0, sizeof(*req));
2573
2574 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2575 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2576 req->state_sz = SHA3_256_DIGEST_SIZE;
2577 req->digest_sz = SHA3_256_DIGEST_SIZE;
2578 req->block_sz = SHA3_256_BLOCK_SIZE;
2579 ctx->do_fallback = false;
2580 ctx->fb_init_done = false;
2581 return 0;
2582 }
2583
2584 static int safexcel_sha3_256_digest(struct ahash_request *req)
2585 {
2586 if (req->nbytes)
2587 return safexcel_sha3_256_init(req) ?: safexcel_ahash_finup(req);
2588
2589 /* HW cannot do zero length hash, use fallback instead */
2590 return safexcel_sha3_digest_fallback(req);
2591 }
2592
2593 struct safexcel_alg_template safexcel_alg_sha3_256 = {
2594 .type = SAFEXCEL_ALG_TYPE_AHASH,
2595 .algo_mask = SAFEXCEL_ALG_SHA3,
2596 .alg.ahash = {
2597 .init = safexcel_sha3_256_init,
2598 .update = safexcel_sha3_update,
2599 .final = safexcel_sha3_final,
2600 .finup = safexcel_sha3_finup,
2601 .digest = safexcel_sha3_256_digest,
2602 .export = safexcel_sha3_export,
2603 .import = safexcel_sha3_import,
2604 .halg = {
2605 .digestsize = SHA3_256_DIGEST_SIZE,
2606 .statesize = sizeof(struct safexcel_ahash_export_state),
2607 .base = {
2608 .cra_name = "sha3-256",
2609 .cra_driver_name = "safexcel-sha3-256",
2610 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2611 .cra_flags = CRYPTO_ALG_ASYNC |
2612 CRYPTO_ALG_KERN_DRIVER_ONLY |
2613 CRYPTO_ALG_NEED_FALLBACK,
2614 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2615 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2616 .cra_init = safexcel_sha3_cra_init,
2617 .cra_exit = safexcel_sha3_cra_exit,
2618 .cra_module = THIS_MODULE,
2619 },
2620 },
2621 },
2622 };
2623
2624 static int safexcel_sha3_384_init(struct ahash_request *areq)
2625 {
2626 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2627 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2628 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2629
2630 memset(req, 0, sizeof(*req));
2631
2632 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2633 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2634 req->state_sz = SHA3_384_DIGEST_SIZE;
2635 req->digest_sz = SHA3_384_DIGEST_SIZE;
2636 req->block_sz = SHA3_384_BLOCK_SIZE;
2637 ctx->do_fallback = false;
2638 ctx->fb_init_done = false;
2639 return 0;
2640 }
2641
2642 static int safexcel_sha3_384_digest(struct ahash_request *req)
2643 {
2644 if (req->nbytes)
2645 return safexcel_sha3_384_init(req) ?: safexcel_ahash_finup(req);
2646
2647 /* HW cannot do zero length hash, use fallback instead */
2648 return safexcel_sha3_digest_fallback(req);
2649 }
2650
2651 struct safexcel_alg_template safexcel_alg_sha3_384 = {
2652 .type = SAFEXCEL_ALG_TYPE_AHASH,
2653 .algo_mask = SAFEXCEL_ALG_SHA3,
2654 .alg.ahash = {
2655 .init = safexcel_sha3_384_init,
2656 .update = safexcel_sha3_update,
2657 .final = safexcel_sha3_final,
2658 .finup = safexcel_sha3_finup,
2659 .digest = safexcel_sha3_384_digest,
2660 .export = safexcel_sha3_export,
2661 .import = safexcel_sha3_import,
2662 .halg = {
2663 .digestsize = SHA3_384_DIGEST_SIZE,
2664 .statesize = sizeof(struct safexcel_ahash_export_state),
2665 .base = {
2666 .cra_name = "sha3-384",
2667 .cra_driver_name = "safexcel-sha3-384",
2668 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2669 .cra_flags = CRYPTO_ALG_ASYNC |
2670 CRYPTO_ALG_KERN_DRIVER_ONLY |
2671 CRYPTO_ALG_NEED_FALLBACK,
2672 .cra_blocksize = SHA3_384_BLOCK_SIZE,
2673 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2674 .cra_init = safexcel_sha3_cra_init,
2675 .cra_exit = safexcel_sha3_cra_exit,
2676 .cra_module = THIS_MODULE,
2677 },
2678 },
2679 },
2680 };
2681
2682 static int safexcel_sha3_512_init(struct ahash_request *areq)
2683 {
2684 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2685 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2686 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2687
2688 memset(req, 0, sizeof(*req));
2689
2690 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
2691 req->digest = CONTEXT_CONTROL_DIGEST_INITIAL;
2692 req->state_sz = SHA3_512_DIGEST_SIZE;
2693 req->digest_sz = SHA3_512_DIGEST_SIZE;
2694 req->block_sz = SHA3_512_BLOCK_SIZE;
2695 ctx->do_fallback = false;
2696 ctx->fb_init_done = false;
2697 return 0;
2698 }
2699
2700 static int safexcel_sha3_512_digest(struct ahash_request *req)
2701 {
2702 if (req->nbytes)
2703 return safexcel_sha3_512_init(req) ?: safexcel_ahash_finup(req);
2704
2705 /* HW cannot do zero length hash, use fallback instead */
2706 return safexcel_sha3_digest_fallback(req);
2707 }
2708
2709 struct safexcel_alg_template safexcel_alg_sha3_512 = {
2710 .type = SAFEXCEL_ALG_TYPE_AHASH,
2711 .algo_mask = SAFEXCEL_ALG_SHA3,
2712 .alg.ahash = {
2713 .init = safexcel_sha3_512_init,
2714 .update = safexcel_sha3_update,
2715 .final = safexcel_sha3_final,
2716 .finup = safexcel_sha3_finup,
2717 .digest = safexcel_sha3_512_digest,
2718 .export = safexcel_sha3_export,
2719 .import = safexcel_sha3_import,
2720 .halg = {
2721 .digestsize = SHA3_512_DIGEST_SIZE,
2722 .statesize = sizeof(struct safexcel_ahash_export_state),
2723 .base = {
2724 .cra_name = "sha3-512",
2725 .cra_driver_name = "safexcel-sha3-512",
2726 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2727 .cra_flags = CRYPTO_ALG_ASYNC |
2728 CRYPTO_ALG_KERN_DRIVER_ONLY |
2729 CRYPTO_ALG_NEED_FALLBACK,
2730 .cra_blocksize = SHA3_512_BLOCK_SIZE,
2731 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2732 .cra_init = safexcel_sha3_cra_init,
2733 .cra_exit = safexcel_sha3_cra_exit,
2734 .cra_module = THIS_MODULE,
2735 },
2736 },
2737 },
2738 };
2739
2740 static int safexcel_hmac_sha3_cra_init(struct crypto_tfm *tfm, const char *alg)
2741 {
2742 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2743 int ret;
2744
2745 ret = safexcel_sha3_cra_init(tfm);
2746 if (ret)
2747 return ret;
2748
2749 /* Allocate precalc basic digest implementation */
2750 ctx->shpre = crypto_alloc_shash(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
2751 if (IS_ERR(ctx->shpre))
2752 return PTR_ERR(ctx->shpre);
2753
2754 ctx->shdesc = kmalloc(sizeof(*ctx->shdesc) +
2755 crypto_shash_descsize(ctx->shpre), GFP_KERNEL);
2756 if (!ctx->shdesc) {
2757 crypto_free_shash(ctx->shpre);
2758 return -ENOMEM;
2759 }
2760 ctx->shdesc->tfm = ctx->shpre;
2761 return 0;
2762 }
2763
2764 static void safexcel_hmac_sha3_cra_exit(struct crypto_tfm *tfm)
2765 {
2766 struct safexcel_ahash_ctx *ctx = crypto_tfm_ctx(tfm);
2767
2768 crypto_free_ahash(ctx->fback);
2769 crypto_free_shash(ctx->shpre);
2770 kfree(ctx->shdesc);
2771 safexcel_ahash_cra_exit(tfm);
2772 }
2773
2774 static int safexcel_hmac_sha3_setkey(struct crypto_ahash *tfm, const u8 *key,
2775 unsigned int keylen)
2776 {
2777 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2778 int ret = 0;
2779
2780 if (keylen > crypto_ahash_blocksize(tfm)) {
2781 /*
2782 * If the key is larger than the blocksize, then hash it
2783 * first using our fallback cipher
2784 */
2785 ret = crypto_shash_digest(ctx->shdesc, key, keylen,
2786 (u8 *)ctx->ipad);
2787 keylen = crypto_shash_digestsize(ctx->shpre);
2788
2789 /*
2790 * If the digest is larger than half the blocksize, we need to
2791 * move the rest to opad due to the way our HMAC infra works.
2792 */
2793 if (keylen > crypto_ahash_blocksize(tfm) / 2)
2794 /* Buffers overlap, need to use memmove iso memcpy! */
2795 memmove(ctx->opad,
2796 (u8 *)ctx->ipad +
2797 crypto_ahash_blocksize(tfm) / 2,
2798 keylen - crypto_ahash_blocksize(tfm) / 2);
2799 } else {
2800 /*
2801 * Copy the key to our ipad & opad buffers
2802 * Note that ipad and opad each contain one half of the key,
2803 * to match the existing HMAC driver infrastructure.
2804 */
2805 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2806 memcpy(ctx->ipad, key, keylen);
2807 } else {
2808 memcpy(ctx->ipad, key,
2809 crypto_ahash_blocksize(tfm) / 2);
2810 memcpy(ctx->opad,
2811 key + crypto_ahash_blocksize(tfm) / 2,
2812 keylen - crypto_ahash_blocksize(tfm) / 2);
2813 }
2814 }
2815
2816 /* Pad key with zeroes */
2817 if (keylen <= crypto_ahash_blocksize(tfm) / 2) {
2818 memset((u8 *)ctx->ipad + keylen, 0,
2819 crypto_ahash_blocksize(tfm) / 2 - keylen);
2820 memset(ctx->opad, 0, crypto_ahash_blocksize(tfm) / 2);
2821 } else {
2822 memset((u8 *)ctx->opad + keylen -
2823 crypto_ahash_blocksize(tfm) / 2, 0,
2824 crypto_ahash_blocksize(tfm) - keylen);
2825 }
2826
2827 /* If doing fallback, still need to set the new key! */
2828 ctx->fb_do_setkey = true;
2829 return ret;
2830 }
2831
2832 static int safexcel_hmac_sha3_224_init(struct ahash_request *areq)
2833 {
2834 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2835 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2836 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2837
2838 memset(req, 0, sizeof(*req));
2839
2840 /* Copy (half of) the key */
2841 memcpy(req->state, ctx->ipad, SHA3_224_BLOCK_SIZE / 2);
2842 /* Start of HMAC should have len == processed == blocksize */
2843 req->len = SHA3_224_BLOCK_SIZE;
2844 req->processed = SHA3_224_BLOCK_SIZE;
2845 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_224;
2846 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2847 req->state_sz = SHA3_224_BLOCK_SIZE / 2;
2848 req->digest_sz = SHA3_224_DIGEST_SIZE;
2849 req->block_sz = SHA3_224_BLOCK_SIZE;
2850 req->hmac = true;
2851 ctx->do_fallback = false;
2852 ctx->fb_init_done = false;
2853 return 0;
2854 }
2855
2856 static int safexcel_hmac_sha3_224_digest(struct ahash_request *req)
2857 {
2858 if (req->nbytes)
2859 return safexcel_hmac_sha3_224_init(req) ?:
2860 safexcel_ahash_finup(req);
2861
2862 /* HW cannot do zero length HMAC, use fallback instead */
2863 return safexcel_sha3_digest_fallback(req);
2864 }
2865
2866 static int safexcel_hmac_sha3_224_cra_init(struct crypto_tfm *tfm)
2867 {
2868 return safexcel_hmac_sha3_cra_init(tfm, "sha3-224");
2869 }
2870
2871 struct safexcel_alg_template safexcel_alg_hmac_sha3_224 = {
2872 .type = SAFEXCEL_ALG_TYPE_AHASH,
2873 .algo_mask = SAFEXCEL_ALG_SHA3,
2874 .alg.ahash = {
2875 .init = safexcel_hmac_sha3_224_init,
2876 .update = safexcel_sha3_update,
2877 .final = safexcel_sha3_final,
2878 .finup = safexcel_sha3_finup,
2879 .digest = safexcel_hmac_sha3_224_digest,
2880 .setkey = safexcel_hmac_sha3_setkey,
2881 .export = safexcel_sha3_export,
2882 .import = safexcel_sha3_import,
2883 .halg = {
2884 .digestsize = SHA3_224_DIGEST_SIZE,
2885 .statesize = sizeof(struct safexcel_ahash_export_state),
2886 .base = {
2887 .cra_name = "hmac(sha3-224)",
2888 .cra_driver_name = "safexcel-hmac-sha3-224",
2889 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2890 .cra_flags = CRYPTO_ALG_ASYNC |
2891 CRYPTO_ALG_KERN_DRIVER_ONLY |
2892 CRYPTO_ALG_NEED_FALLBACK,
2893 .cra_blocksize = SHA3_224_BLOCK_SIZE,
2894 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2895 .cra_init = safexcel_hmac_sha3_224_cra_init,
2896 .cra_exit = safexcel_hmac_sha3_cra_exit,
2897 .cra_module = THIS_MODULE,
2898 },
2899 },
2900 },
2901 };
2902
2903 static int safexcel_hmac_sha3_256_init(struct ahash_request *areq)
2904 {
2905 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2906 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2907 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2908
2909 memset(req, 0, sizeof(*req));
2910
2911 /* Copy (half of) the key */
2912 memcpy(req->state, ctx->ipad, SHA3_256_BLOCK_SIZE / 2);
2913 /* Start of HMAC should have len == processed == blocksize */
2914 req->len = SHA3_256_BLOCK_SIZE;
2915 req->processed = SHA3_256_BLOCK_SIZE;
2916 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_256;
2917 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2918 req->state_sz = SHA3_256_BLOCK_SIZE / 2;
2919 req->digest_sz = SHA3_256_DIGEST_SIZE;
2920 req->block_sz = SHA3_256_BLOCK_SIZE;
2921 req->hmac = true;
2922 ctx->do_fallback = false;
2923 ctx->fb_init_done = false;
2924 return 0;
2925 }
2926
2927 static int safexcel_hmac_sha3_256_digest(struct ahash_request *req)
2928 {
2929 if (req->nbytes)
2930 return safexcel_hmac_sha3_256_init(req) ?:
2931 safexcel_ahash_finup(req);
2932
2933 /* HW cannot do zero length HMAC, use fallback instead */
2934 return safexcel_sha3_digest_fallback(req);
2935 }
2936
2937 static int safexcel_hmac_sha3_256_cra_init(struct crypto_tfm *tfm)
2938 {
2939 return safexcel_hmac_sha3_cra_init(tfm, "sha3-256");
2940 }
2941
2942 struct safexcel_alg_template safexcel_alg_hmac_sha3_256 = {
2943 .type = SAFEXCEL_ALG_TYPE_AHASH,
2944 .algo_mask = SAFEXCEL_ALG_SHA3,
2945 .alg.ahash = {
2946 .init = safexcel_hmac_sha3_256_init,
2947 .update = safexcel_sha3_update,
2948 .final = safexcel_sha3_final,
2949 .finup = safexcel_sha3_finup,
2950 .digest = safexcel_hmac_sha3_256_digest,
2951 .setkey = safexcel_hmac_sha3_setkey,
2952 .export = safexcel_sha3_export,
2953 .import = safexcel_sha3_import,
2954 .halg = {
2955 .digestsize = SHA3_256_DIGEST_SIZE,
2956 .statesize = sizeof(struct safexcel_ahash_export_state),
2957 .base = {
2958 .cra_name = "hmac(sha3-256)",
2959 .cra_driver_name = "safexcel-hmac-sha3-256",
2960 .cra_priority = SAFEXCEL_CRA_PRIORITY,
2961 .cra_flags = CRYPTO_ALG_ASYNC |
2962 CRYPTO_ALG_KERN_DRIVER_ONLY |
2963 CRYPTO_ALG_NEED_FALLBACK,
2964 .cra_blocksize = SHA3_256_BLOCK_SIZE,
2965 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
2966 .cra_init = safexcel_hmac_sha3_256_cra_init,
2967 .cra_exit = safexcel_hmac_sha3_cra_exit,
2968 .cra_module = THIS_MODULE,
2969 },
2970 },
2971 },
2972 };
2973
2974 static int safexcel_hmac_sha3_384_init(struct ahash_request *areq)
2975 {
2976 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2977 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
2978 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
2979
2980 memset(req, 0, sizeof(*req));
2981
2982 /* Copy (half of) the key */
2983 memcpy(req->state, ctx->ipad, SHA3_384_BLOCK_SIZE / 2);
2984 /* Start of HMAC should have len == processed == blocksize */
2985 req->len = SHA3_384_BLOCK_SIZE;
2986 req->processed = SHA3_384_BLOCK_SIZE;
2987 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_384;
2988 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
2989 req->state_sz = SHA3_384_BLOCK_SIZE / 2;
2990 req->digest_sz = SHA3_384_DIGEST_SIZE;
2991 req->block_sz = SHA3_384_BLOCK_SIZE;
2992 req->hmac = true;
2993 ctx->do_fallback = false;
2994 ctx->fb_init_done = false;
2995 return 0;
2996 }
2997
2998 static int safexcel_hmac_sha3_384_digest(struct ahash_request *req)
2999 {
3000 if (req->nbytes)
3001 return safexcel_hmac_sha3_384_init(req) ?:
3002 safexcel_ahash_finup(req);
3003
3004 /* HW cannot do zero length HMAC, use fallback instead */
3005 return safexcel_sha3_digest_fallback(req);
3006 }
3007
3008 static int safexcel_hmac_sha3_384_cra_init(struct crypto_tfm *tfm)
3009 {
3010 return safexcel_hmac_sha3_cra_init(tfm, "sha3-384");
3011 }
3012
3013 struct safexcel_alg_template safexcel_alg_hmac_sha3_384 = {
3014 .type = SAFEXCEL_ALG_TYPE_AHASH,
3015 .algo_mask = SAFEXCEL_ALG_SHA3,
3016 .alg.ahash = {
3017 .init = safexcel_hmac_sha3_384_init,
3018 .update = safexcel_sha3_update,
3019 .final = safexcel_sha3_final,
3020 .finup = safexcel_sha3_finup,
3021 .digest = safexcel_hmac_sha3_384_digest,
3022 .setkey = safexcel_hmac_sha3_setkey,
3023 .export = safexcel_sha3_export,
3024 .import = safexcel_sha3_import,
3025 .halg = {
3026 .digestsize = SHA3_384_DIGEST_SIZE,
3027 .statesize = sizeof(struct safexcel_ahash_export_state),
3028 .base = {
3029 .cra_name = "hmac(sha3-384)",
3030 .cra_driver_name = "safexcel-hmac-sha3-384",
3031 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3032 .cra_flags = CRYPTO_ALG_ASYNC |
3033 CRYPTO_ALG_KERN_DRIVER_ONLY |
3034 CRYPTO_ALG_NEED_FALLBACK,
3035 .cra_blocksize = SHA3_384_BLOCK_SIZE,
3036 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3037 .cra_init = safexcel_hmac_sha3_384_cra_init,
3038 .cra_exit = safexcel_hmac_sha3_cra_exit,
3039 .cra_module = THIS_MODULE,
3040 },
3041 },
3042 },
3043 };
3044
3045 static int safexcel_hmac_sha3_512_init(struct ahash_request *areq)
3046 {
3047 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
3048 struct safexcel_ahash_ctx *ctx = crypto_ahash_ctx(tfm);
3049 struct safexcel_ahash_req *req = ahash_request_ctx(areq);
3050
3051 memset(req, 0, sizeof(*req));
3052
3053 /* Copy (half of) the key */
3054 memcpy(req->state, ctx->ipad, SHA3_512_BLOCK_SIZE / 2);
3055 /* Start of HMAC should have len == processed == blocksize */
3056 req->len = SHA3_512_BLOCK_SIZE;
3057 req->processed = SHA3_512_BLOCK_SIZE;
3058 ctx->alg = CONTEXT_CONTROL_CRYPTO_ALG_SHA3_512;
3059 req->digest = CONTEXT_CONTROL_DIGEST_HMAC;
3060 req->state_sz = SHA3_512_BLOCK_SIZE / 2;
3061 req->digest_sz = SHA3_512_DIGEST_SIZE;
3062 req->block_sz = SHA3_512_BLOCK_SIZE;
3063 req->hmac = true;
3064 ctx->do_fallback = false;
3065 ctx->fb_init_done = false;
3066 return 0;
3067 }
3068
3069 static int safexcel_hmac_sha3_512_digest(struct ahash_request *req)
3070 {
3071 if (req->nbytes)
3072 return safexcel_hmac_sha3_512_init(req) ?:
3073 safexcel_ahash_finup(req);
3074
3075 /* HW cannot do zero length HMAC, use fallback instead */
3076 return safexcel_sha3_digest_fallback(req);
3077 }
3078
3079 static int safexcel_hmac_sha3_512_cra_init(struct crypto_tfm *tfm)
3080 {
3081 return safexcel_hmac_sha3_cra_init(tfm, "sha3-512");
3082 }
3083 struct safexcel_alg_template safexcel_alg_hmac_sha3_512 = {
3084 .type = SAFEXCEL_ALG_TYPE_AHASH,
3085 .algo_mask = SAFEXCEL_ALG_SHA3,
3086 .alg.ahash = {
3087 .init = safexcel_hmac_sha3_512_init,
3088 .update = safexcel_sha3_update,
3089 .final = safexcel_sha3_final,
3090 .finup = safexcel_sha3_finup,
3091 .digest = safexcel_hmac_sha3_512_digest,
3092 .setkey = safexcel_hmac_sha3_setkey,
3093 .export = safexcel_sha3_export,
3094 .import = safexcel_sha3_import,
3095 .halg = {
3096 .digestsize = SHA3_512_DIGEST_SIZE,
3097 .statesize = sizeof(struct safexcel_ahash_export_state),
3098 .base = {
3099 .cra_name = "hmac(sha3-512)",
3100 .cra_driver_name = "safexcel-hmac-sha3-512",
3101 .cra_priority = SAFEXCEL_CRA_PRIORITY,
3102 .cra_flags = CRYPTO_ALG_ASYNC |
3103 CRYPTO_ALG_KERN_DRIVER_ONLY |
3104 CRYPTO_ALG_NEED_FALLBACK,
3105 .cra_blocksize = SHA3_512_BLOCK_SIZE,
3106 .cra_ctxsize = sizeof(struct safexcel_ahash_ctx),
3107 .cra_init = safexcel_hmac_sha3_512_cra_init,
3108 .cra_exit = safexcel_hmac_sha3_cra_exit,
3109 .cra_module = THIS_MODULE,
3110 },
3111 },
3112 },
3113 };
Linux with added FPC-III support