?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
iwlwifi: mvm: fix version check for GEO_TX_POWER_LIMIT support
[linux/fpc-iii.git] / arch / x86 / crypto / sha1-mb / sha1_mb.c
blobb93805664c1dd0ad290e731bc145dfbcb33f5a1c
1 /*
2 * Multi buffer SHA1 algorithm Glue Code
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * Copyright(c) 2014 Intel Corporation.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of version 2 of the GNU General Public License as
13 * published by the Free Software Foundation.
14 *
15 * This program is distributed in the hope that it will be useful, but
16 * WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * General Public License for more details.
19 *
20 * Contact Information:
21 * Tim Chen <tim.c.chen@linux.intel.com>
22 *
23 * BSD LICENSE
24 *
25 * Copyright(c) 2014 Intel Corporation.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 *
31 * * Redistributions of source code must retain the above copyright
32 * notice, this list of conditions and the following disclaimer.
33 * * Redistributions in binary form must reproduce the above copyright
34 * notice, this list of conditions and the following disclaimer in
35 * the documentation and/or other materials provided with the
36 * distribution.
37 * * Neither the name of Intel Corporation nor the names of its
38 * contributors may be used to endorse or promote products derived
39 * from this software without specific prior written permission.
40 *
41 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
42 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
43 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
44 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
45 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
46 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
47 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
48 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
49 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
50 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
51 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
52 */
53
54 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
55
56 #include <crypto/internal/hash.h>
57 #include <linux/init.h>
58 #include <linux/module.h>
59 #include <linux/mm.h>
60 #include <linux/cryptohash.h>
61 #include <linux/types.h>
62 #include <linux/list.h>
63 #include <crypto/scatterwalk.h>
64 #include <crypto/sha.h>
65 #include <crypto/mcryptd.h>
66 #include <crypto/crypto_wq.h>
67 #include <asm/byteorder.h>
68 #include <linux/hardirq.h>
69 #include <asm/fpu/api.h>
70 #include "sha1_mb_ctx.h"
71
72 #define FLUSH_INTERVAL 1000 /* in usec */
73
74 static struct mcryptd_alg_state sha1_mb_alg_state;
75
76 struct sha1_mb_ctx {
77 struct mcryptd_ahash *mcryptd_tfm;
78 };
79
80 static inline struct mcryptd_hash_request_ctx
81 *cast_hash_to_mcryptd_ctx(struct sha1_hash_ctx *hash_ctx)
82 {
83 struct ahash_request *areq;
84
85 areq = container_of((void *) hash_ctx, struct ahash_request, __ctx);
86 return container_of(areq, struct mcryptd_hash_request_ctx, areq);
87 }
88
89 static inline struct ahash_request
90 *cast_mcryptd_ctx_to_req(struct mcryptd_hash_request_ctx *ctx)
91 {
92 return container_of((void *) ctx, struct ahash_request, __ctx);
93 }
94
95 static void req_ctx_init(struct mcryptd_hash_request_ctx *rctx,
96 struct ahash_request *areq)
97 {
98 rctx->flag = HASH_UPDATE;
99 }
100
101 static asmlinkage void (*sha1_job_mgr_init)(struct sha1_mb_mgr *state);
102 static asmlinkage struct job_sha1* (*sha1_job_mgr_submit)
103 (struct sha1_mb_mgr *state, struct job_sha1 *job);
104 static asmlinkage struct job_sha1* (*sha1_job_mgr_flush)
105 (struct sha1_mb_mgr *state);
106 static asmlinkage struct job_sha1* (*sha1_job_mgr_get_comp_job)
107 (struct sha1_mb_mgr *state);
108
109 static inline uint32_t sha1_pad(uint8_t padblock[SHA1_BLOCK_SIZE * 2],
110 uint64_t total_len)
111 {
112 uint32_t i = total_len & (SHA1_BLOCK_SIZE - 1);
113
114 memset(&padblock[i], 0, SHA1_BLOCK_SIZE);
115 padblock[i] = 0x80;
116
117 i += ((SHA1_BLOCK_SIZE - 1) &
118 (0 - (total_len + SHA1_PADLENGTHFIELD_SIZE + 1)))
119 + 1 + SHA1_PADLENGTHFIELD_SIZE;
120
121 #if SHA1_PADLENGTHFIELD_SIZE == 16
122 *((uint64_t *) &padblock[i - 16]) = 0;
123 #endif
124
125 *((uint64_t *) &padblock[i - 8]) = cpu_to_be64(total_len << 3);
126
127 /* Number of extra blocks to hash */
128 return i >> SHA1_LOG2_BLOCK_SIZE;
129 }
130
131 static struct sha1_hash_ctx *sha1_ctx_mgr_resubmit(struct sha1_ctx_mgr *mgr,
132 struct sha1_hash_ctx *ctx)
133 {
134 while (ctx) {
135 if (ctx->status & HASH_CTX_STS_COMPLETE) {
136 /* Clear PROCESSING bit */
137 ctx->status = HASH_CTX_STS_COMPLETE;
138 return ctx;
139 }
140
141 /*
142 * If the extra blocks are empty, begin hashing what remains
143 * in the user's buffer.
144 */
145 if (ctx->partial_block_buffer_length == 0 &&
146 ctx->incoming_buffer_length) {
147
148 const void *buffer = ctx->incoming_buffer;
149 uint32_t len = ctx->incoming_buffer_length;
150 uint32_t copy_len;
151
152 /*
153 * Only entire blocks can be hashed.
154 * Copy remainder to extra blocks buffer.
155 */
156 copy_len = len & (SHA1_BLOCK_SIZE-1);
157
158 if (copy_len) {
159 len -= copy_len;
160 memcpy(ctx->partial_block_buffer,
161 ((const char *) buffer + len),
162 copy_len);
163 ctx->partial_block_buffer_length = copy_len;
164 }
165
166 ctx->incoming_buffer_length = 0;
167
168 /* len should be a multiple of the block size now */
169 assert((len % SHA1_BLOCK_SIZE) == 0);
170
171 /* Set len to the number of blocks to be hashed */
172 len >>= SHA1_LOG2_BLOCK_SIZE;
173
174 if (len) {
175
176 ctx->job.buffer = (uint8_t *) buffer;
177 ctx->job.len = len;
178 ctx = (struct sha1_hash_ctx *)sha1_job_mgr_submit(&mgr->mgr,
179 &ctx->job);
180 continue;
181 }
182 }
183
184 /*
185 * If the extra blocks are not empty, then we are
186 * either on the last block(s) or we need more
187 * user input before continuing.
188 */
189 if (ctx->status & HASH_CTX_STS_LAST) {
190
191 uint8_t *buf = ctx->partial_block_buffer;
192 uint32_t n_extra_blocks =
193 sha1_pad(buf, ctx->total_length);
194
195 ctx->status = (HASH_CTX_STS_PROCESSING |
196 HASH_CTX_STS_COMPLETE);
197 ctx->job.buffer = buf;
198 ctx->job.len = (uint32_t) n_extra_blocks;
199 ctx = (struct sha1_hash_ctx *)
200 sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
201 continue;
202 }
203
204 ctx->status = HASH_CTX_STS_IDLE;
205 return ctx;
206 }
207
208 return NULL;
209 }
210
211 static struct sha1_hash_ctx
212 *sha1_ctx_mgr_get_comp_ctx(struct sha1_ctx_mgr *mgr)
213 {
214 /*
215 * If get_comp_job returns NULL, there are no jobs complete.
216 * If get_comp_job returns a job, verify that it is safe to return to
217 * the user.
218 * If it is not ready, resubmit the job to finish processing.
219 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be returned.
220 * Otherwise, all jobs currently being managed by the hash_ctx_mgr
221 * still need processing.
222 */
223 struct sha1_hash_ctx *ctx;
224
225 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_get_comp_job(&mgr->mgr);
226 return sha1_ctx_mgr_resubmit(mgr, ctx);
227 }
228
229 static void sha1_ctx_mgr_init(struct sha1_ctx_mgr *mgr)
230 {
231 sha1_job_mgr_init(&mgr->mgr);
232 }
233
234 static struct sha1_hash_ctx *sha1_ctx_mgr_submit(struct sha1_ctx_mgr *mgr,
235 struct sha1_hash_ctx *ctx,
236 const void *buffer,
237 uint32_t len,
238 int flags)
239 {
240 if (flags & ~(HASH_UPDATE | HASH_LAST)) {
241 /* User should not pass anything other than UPDATE or LAST */
242 ctx->error = HASH_CTX_ERROR_INVALID_FLAGS;
243 return ctx;
244 }
245
246 if (ctx->status & HASH_CTX_STS_PROCESSING) {
247 /* Cannot submit to a currently processing job. */
248 ctx->error = HASH_CTX_ERROR_ALREADY_PROCESSING;
249 return ctx;
250 }
251
252 if (ctx->status & HASH_CTX_STS_COMPLETE) {
253 /* Cannot update a finished job. */
254 ctx->error = HASH_CTX_ERROR_ALREADY_COMPLETED;
255 return ctx;
256 }
257
258 /*
259 * If we made it here, there were no errors during this call to
260 * submit
261 */
262 ctx->error = HASH_CTX_ERROR_NONE;
263
264 /* Store buffer ptr info from user */
265 ctx->incoming_buffer = buffer;
266 ctx->incoming_buffer_length = len;
267
268 /*
269 * Store the user's request flags and mark this ctx as currently
270 * being processed.
271 */
272 ctx->status = (flags & HASH_LAST) ?
273 (HASH_CTX_STS_PROCESSING | HASH_CTX_STS_LAST) :
274 HASH_CTX_STS_PROCESSING;
275
276 /* Advance byte counter */
277 ctx->total_length += len;
278
279 /*
280 * If there is anything currently buffered in the extra blocks,
281 * append to it until it contains a whole block.
282 * Or if the user's buffer contains less than a whole block,
283 * append as much as possible to the extra block.
284 */
285 if (ctx->partial_block_buffer_length || len < SHA1_BLOCK_SIZE) {
286 /*
287 * Compute how many bytes to copy from user buffer into
288 * extra block
289 */
290 uint32_t copy_len = SHA1_BLOCK_SIZE -
291 ctx->partial_block_buffer_length;
292 if (len < copy_len)
293 copy_len = len;
294
295 if (copy_len) {
296 /* Copy and update relevant pointers and counters */
297 memcpy(&ctx->partial_block_buffer[ctx->partial_block_buffer_length],
298 buffer, copy_len);
299
300 ctx->partial_block_buffer_length += copy_len;
301 ctx->incoming_buffer = (const void *)
302 ((const char *)buffer + copy_len);
303 ctx->incoming_buffer_length = len - copy_len;
304 }
305
306 /*
307 * The extra block should never contain more than 1 block
308 * here
309 */
310 assert(ctx->partial_block_buffer_length <= SHA1_BLOCK_SIZE);
311
312 /*
313 * If the extra block buffer contains exactly 1 block, it can
314 * be hashed.
315 */
316 if (ctx->partial_block_buffer_length >= SHA1_BLOCK_SIZE) {
317 ctx->partial_block_buffer_length = 0;
318
319 ctx->job.buffer = ctx->partial_block_buffer;
320 ctx->job.len = 1;
321 ctx = (struct sha1_hash_ctx *)
322 sha1_job_mgr_submit(&mgr->mgr, &ctx->job);
323 }
324 }
325
326 return sha1_ctx_mgr_resubmit(mgr, ctx);
327 }
328
329 static struct sha1_hash_ctx *sha1_ctx_mgr_flush(struct sha1_ctx_mgr *mgr)
330 {
331 struct sha1_hash_ctx *ctx;
332
333 while (1) {
334 ctx = (struct sha1_hash_ctx *) sha1_job_mgr_flush(&mgr->mgr);
335
336 /* If flush returned 0, there are no more jobs in flight. */
337 if (!ctx)
338 return NULL;
339
340 /*
341 * If flush returned a job, resubmit the job to finish
342 * processing.
343 */
344 ctx = sha1_ctx_mgr_resubmit(mgr, ctx);
345
346 /*
347 * If sha1_ctx_mgr_resubmit returned a job, it is ready to be
348 * returned. Otherwise, all jobs currently being managed by the
349 * sha1_ctx_mgr still need processing. Loop.
350 */
351 if (ctx)
352 return ctx;
353 }
354 }
355
356 static int sha1_mb_init(struct ahash_request *areq)
357 {
358 struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
359
360 hash_ctx_init(sctx);
361 sctx->job.result_digest[0] = SHA1_H0;
362 sctx->job.result_digest[1] = SHA1_H1;
363 sctx->job.result_digest[2] = SHA1_H2;
364 sctx->job.result_digest[3] = SHA1_H3;
365 sctx->job.result_digest[4] = SHA1_H4;
366 sctx->total_length = 0;
367 sctx->partial_block_buffer_length = 0;
368 sctx->status = HASH_CTX_STS_IDLE;
369
370 return 0;
371 }
372
373 static int sha1_mb_set_results(struct mcryptd_hash_request_ctx *rctx)
374 {
375 int i;
376 struct sha1_hash_ctx *sctx = ahash_request_ctx(&rctx->areq);
377 __be32 *dst = (__be32 *) rctx->out;
378
379 for (i = 0; i < 5; ++i)
380 dst[i] = cpu_to_be32(sctx->job.result_digest[i]);
381
382 return 0;
383 }
384
385 static int sha_finish_walk(struct mcryptd_hash_request_ctx **ret_rctx,
386 struct mcryptd_alg_cstate *cstate, bool flush)
387 {
388 int flag = HASH_UPDATE;
389 int nbytes, err = 0;
390 struct mcryptd_hash_request_ctx *rctx = *ret_rctx;
391 struct sha1_hash_ctx *sha_ctx;
392
393 /* more work ? */
394 while (!(rctx->flag & HASH_DONE)) {
395 nbytes = crypto_ahash_walk_done(&rctx->walk, 0);
396 if (nbytes < 0) {
397 err = nbytes;
398 goto out;
399 }
400 /* check if the walk is done */
401 if (crypto_ahash_walk_last(&rctx->walk)) {
402 rctx->flag |= HASH_DONE;
403 if (rctx->flag & HASH_FINAL)
404 flag |= HASH_LAST;
405
406 }
407 sha_ctx = (struct sha1_hash_ctx *)
408 ahash_request_ctx(&rctx->areq);
409 kernel_fpu_begin();
410 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx,
411 rctx->walk.data, nbytes, flag);
412 if (!sha_ctx) {
413 if (flush)
414 sha_ctx = sha1_ctx_mgr_flush(cstate->mgr);
415 }
416 kernel_fpu_end();
417 if (sha_ctx)
418 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
419 else {
420 rctx = NULL;
421 goto out;
422 }
423 }
424
425 /* copy the results */
426 if (rctx->flag & HASH_FINAL)
427 sha1_mb_set_results(rctx);
428
429 out:
430 *ret_rctx = rctx;
431 return err;
432 }
433
434 static int sha_complete_job(struct mcryptd_hash_request_ctx *rctx,
435 struct mcryptd_alg_cstate *cstate,
436 int err)
437 {
438 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
439 struct sha1_hash_ctx *sha_ctx;
440 struct mcryptd_hash_request_ctx *req_ctx;
441 int ret;
442
443 /* remove from work list */
444 spin_lock(&cstate->work_lock);
445 list_del(&rctx->waiter);
446 spin_unlock(&cstate->work_lock);
447
448 if (irqs_disabled())
449 rctx->complete(&req->base, err);
450 else {
451 local_bh_disable();
452 rctx->complete(&req->base, err);
453 local_bh_enable();
454 }
455
456 /* check to see if there are other jobs that are done */
457 sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
458 while (sha_ctx) {
459 req_ctx = cast_hash_to_mcryptd_ctx(sha_ctx);
460 ret = sha_finish_walk(&req_ctx, cstate, false);
461 if (req_ctx) {
462 spin_lock(&cstate->work_lock);
463 list_del(&req_ctx->waiter);
464 spin_unlock(&cstate->work_lock);
465
466 req = cast_mcryptd_ctx_to_req(req_ctx);
467 if (irqs_disabled())
468 req_ctx->complete(&req->base, ret);
469 else {
470 local_bh_disable();
471 req_ctx->complete(&req->base, ret);
472 local_bh_enable();
473 }
474 }
475 sha_ctx = sha1_ctx_mgr_get_comp_ctx(cstate->mgr);
476 }
477
478 return 0;
479 }
480
481 static void sha1_mb_add_list(struct mcryptd_hash_request_ctx *rctx,
482 struct mcryptd_alg_cstate *cstate)
483 {
484 unsigned long next_flush;
485 unsigned long delay = usecs_to_jiffies(FLUSH_INTERVAL);
486
487 /* initialize tag */
488 rctx->tag.arrival = jiffies; /* tag the arrival time */
489 rctx->tag.seq_num = cstate->next_seq_num++;
490 next_flush = rctx->tag.arrival + delay;
491 rctx->tag.expire = next_flush;
492
493 spin_lock(&cstate->work_lock);
494 list_add_tail(&rctx->waiter, &cstate->work_list);
495 spin_unlock(&cstate->work_lock);
496
497 mcryptd_arm_flusher(cstate, delay);
498 }
499
500 static int sha1_mb_update(struct ahash_request *areq)
501 {
502 struct mcryptd_hash_request_ctx *rctx =
503 container_of(areq, struct mcryptd_hash_request_ctx, areq);
504 struct mcryptd_alg_cstate *cstate =
505 this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
506
507 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
508 struct sha1_hash_ctx *sha_ctx;
509 int ret = 0, nbytes;
510
511
512 /* sanity check */
513 if (rctx->tag.cpu != smp_processor_id()) {
514 pr_err("mcryptd error: cpu clash\n");
515 goto done;
516 }
517
518 /* need to init context */
519 req_ctx_init(rctx, areq);
520
521 nbytes = crypto_ahash_walk_first(req, &rctx->walk);
522
523 if (nbytes < 0) {
524 ret = nbytes;
525 goto done;
526 }
527
528 if (crypto_ahash_walk_last(&rctx->walk))
529 rctx->flag |= HASH_DONE;
530
531 /* submit */
532 sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
533 sha1_mb_add_list(rctx, cstate);
534 kernel_fpu_begin();
535 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
536 nbytes, HASH_UPDATE);
537 kernel_fpu_end();
538
539 /* check if anything is returned */
540 if (!sha_ctx)
541 return -EINPROGRESS;
542
543 if (sha_ctx->error) {
544 ret = sha_ctx->error;
545 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
546 goto done;
547 }
548
549 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
550 ret = sha_finish_walk(&rctx, cstate, false);
551
552 if (!rctx)
553 return -EINPROGRESS;
554 done:
555 sha_complete_job(rctx, cstate, ret);
556 return ret;
557 }
558
559 static int sha1_mb_finup(struct ahash_request *areq)
560 {
561 struct mcryptd_hash_request_ctx *rctx =
562 container_of(areq, struct mcryptd_hash_request_ctx, areq);
563 struct mcryptd_alg_cstate *cstate =
564 this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
565
566 struct ahash_request *req = cast_mcryptd_ctx_to_req(rctx);
567 struct sha1_hash_ctx *sha_ctx;
568 int ret = 0, flag = HASH_UPDATE, nbytes;
569
570 /* sanity check */
571 if (rctx->tag.cpu != smp_processor_id()) {
572 pr_err("mcryptd error: cpu clash\n");
573 goto done;
574 }
575
576 /* need to init context */
577 req_ctx_init(rctx, areq);
578
579 nbytes = crypto_ahash_walk_first(req, &rctx->walk);
580
581 if (nbytes < 0) {
582 ret = nbytes;
583 goto done;
584 }
585
586 if (crypto_ahash_walk_last(&rctx->walk)) {
587 rctx->flag |= HASH_DONE;
588 flag = HASH_LAST;
589 }
590
591 /* submit */
592 rctx->flag |= HASH_FINAL;
593 sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
594 sha1_mb_add_list(rctx, cstate);
595
596 kernel_fpu_begin();
597 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, rctx->walk.data,
598 nbytes, flag);
599 kernel_fpu_end();
600
601 /* check if anything is returned */
602 if (!sha_ctx)
603 return -EINPROGRESS;
604
605 if (sha_ctx->error) {
606 ret = sha_ctx->error;
607 goto done;
608 }
609
610 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
611 ret = sha_finish_walk(&rctx, cstate, false);
612 if (!rctx)
613 return -EINPROGRESS;
614 done:
615 sha_complete_job(rctx, cstate, ret);
616 return ret;
617 }
618
619 static int sha1_mb_final(struct ahash_request *areq)
620 {
621 struct mcryptd_hash_request_ctx *rctx =
622 container_of(areq, struct mcryptd_hash_request_ctx, areq);
623 struct mcryptd_alg_cstate *cstate =
624 this_cpu_ptr(sha1_mb_alg_state.alg_cstate);
625
626 struct sha1_hash_ctx *sha_ctx;
627 int ret = 0;
628 u8 data;
629
630 /* sanity check */
631 if (rctx->tag.cpu != smp_processor_id()) {
632 pr_err("mcryptd error: cpu clash\n");
633 goto done;
634 }
635
636 /* need to init context */
637 req_ctx_init(rctx, areq);
638
639 rctx->flag |= HASH_DONE | HASH_FINAL;
640
641 sha_ctx = (struct sha1_hash_ctx *) ahash_request_ctx(areq);
642 /* flag HASH_FINAL and 0 data size */
643 sha1_mb_add_list(rctx, cstate);
644 kernel_fpu_begin();
645 sha_ctx = sha1_ctx_mgr_submit(cstate->mgr, sha_ctx, &data, 0,
646 HASH_LAST);
647 kernel_fpu_end();
648
649 /* check if anything is returned */
650 if (!sha_ctx)
651 return -EINPROGRESS;
652
653 if (sha_ctx->error) {
654 ret = sha_ctx->error;
655 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
656 goto done;
657 }
658
659 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
660 ret = sha_finish_walk(&rctx, cstate, false);
661 if (!rctx)
662 return -EINPROGRESS;
663 done:
664 sha_complete_job(rctx, cstate, ret);
665 return ret;
666 }
667
668 static int sha1_mb_export(struct ahash_request *areq, void *out)
669 {
670 struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
671
672 memcpy(out, sctx, sizeof(*sctx));
673
674 return 0;
675 }
676
677 static int sha1_mb_import(struct ahash_request *areq, const void *in)
678 {
679 struct sha1_hash_ctx *sctx = ahash_request_ctx(areq);
680
681 memcpy(sctx, in, sizeof(*sctx));
682
683 return 0;
684 }
685
686 static int sha1_mb_async_init_tfm(struct crypto_tfm *tfm)
687 {
688 struct mcryptd_ahash *mcryptd_tfm;
689 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
690 struct mcryptd_hash_ctx *mctx;
691
692 mcryptd_tfm = mcryptd_alloc_ahash("__intel_sha1-mb",
693 CRYPTO_ALG_INTERNAL,
694 CRYPTO_ALG_INTERNAL);
695 if (IS_ERR(mcryptd_tfm))
696 return PTR_ERR(mcryptd_tfm);
697 mctx = crypto_ahash_ctx(&mcryptd_tfm->base);
698 mctx->alg_state = &sha1_mb_alg_state;
699 ctx->mcryptd_tfm = mcryptd_tfm;
700 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
701 sizeof(struct ahash_request) +
702 crypto_ahash_reqsize(&mcryptd_tfm->base));
703
704 return 0;
705 }
706
707 static void sha1_mb_async_exit_tfm(struct crypto_tfm *tfm)
708 {
709 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
710
711 mcryptd_free_ahash(ctx->mcryptd_tfm);
712 }
713
714 static int sha1_mb_areq_init_tfm(struct crypto_tfm *tfm)
715 {
716 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
717 sizeof(struct ahash_request) +
718 sizeof(struct sha1_hash_ctx));
719
720 return 0;
721 }
722
723 static void sha1_mb_areq_exit_tfm(struct crypto_tfm *tfm)
724 {
725 struct sha1_mb_ctx *ctx = crypto_tfm_ctx(tfm);
726
727 mcryptd_free_ahash(ctx->mcryptd_tfm);
728 }
729
730 static struct ahash_alg sha1_mb_areq_alg = {
731 .init = sha1_mb_init,
732 .update = sha1_mb_update,
733 .final = sha1_mb_final,
734 .finup = sha1_mb_finup,
735 .export = sha1_mb_export,
736 .import = sha1_mb_import,
737 .halg = {
738 .digestsize = SHA1_DIGEST_SIZE,
739 .statesize = sizeof(struct sha1_hash_ctx),
740 .base = {
741 .cra_name = "__sha1-mb",
742 .cra_driver_name = "__intel_sha1-mb",
743 .cra_priority = 100,
744 /*
745 * use ASYNC flag as some buffers in multi-buffer
746 * algo may not have completed before hashing thread
747 * sleep
748 */
749 .cra_flags = CRYPTO_ALG_ASYNC |
750 CRYPTO_ALG_INTERNAL,
751 .cra_blocksize = SHA1_BLOCK_SIZE,
752 .cra_module = THIS_MODULE,
753 .cra_list = LIST_HEAD_INIT
754 (sha1_mb_areq_alg.halg.base.cra_list),
755 .cra_init = sha1_mb_areq_init_tfm,
756 .cra_exit = sha1_mb_areq_exit_tfm,
757 .cra_ctxsize = sizeof(struct sha1_hash_ctx),
758 }
759 }
760 };
761
762 static int sha1_mb_async_init(struct ahash_request *req)
763 {
764 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
765 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
766 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
767 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
768
769 memcpy(mcryptd_req, req, sizeof(*req));
770 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
771 return crypto_ahash_init(mcryptd_req);
772 }
773
774 static int sha1_mb_async_update(struct ahash_request *req)
775 {
776 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
777
778 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
779 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
780 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
781
782 memcpy(mcryptd_req, req, sizeof(*req));
783 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
784 return crypto_ahash_update(mcryptd_req);
785 }
786
787 static int sha1_mb_async_finup(struct ahash_request *req)
788 {
789 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
790
791 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
792 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
793 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
794
795 memcpy(mcryptd_req, req, sizeof(*req));
796 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
797 return crypto_ahash_finup(mcryptd_req);
798 }
799
800 static int sha1_mb_async_final(struct ahash_request *req)
801 {
802 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
803
804 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
805 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
806 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
807
808 memcpy(mcryptd_req, req, sizeof(*req));
809 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
810 return crypto_ahash_final(mcryptd_req);
811 }
812
813 static int sha1_mb_async_digest(struct ahash_request *req)
814 {
815 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
816 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
817 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
818 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
819
820 memcpy(mcryptd_req, req, sizeof(*req));
821 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
822 return crypto_ahash_digest(mcryptd_req);
823 }
824
825 static int sha1_mb_async_export(struct ahash_request *req, void *out)
826 {
827 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
828 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
829 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
830 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
831
832 memcpy(mcryptd_req, req, sizeof(*req));
833 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
834 return crypto_ahash_export(mcryptd_req, out);
835 }
836
837 static int sha1_mb_async_import(struct ahash_request *req, const void *in)
838 {
839 struct ahash_request *mcryptd_req = ahash_request_ctx(req);
840 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
841 struct sha1_mb_ctx *ctx = crypto_ahash_ctx(tfm);
842 struct mcryptd_ahash *mcryptd_tfm = ctx->mcryptd_tfm;
843 struct crypto_ahash *child = mcryptd_ahash_child(mcryptd_tfm);
844 struct mcryptd_hash_request_ctx *rctx;
845 struct ahash_request *areq;
846
847 memcpy(mcryptd_req, req, sizeof(*req));
848 ahash_request_set_tfm(mcryptd_req, &mcryptd_tfm->base);
849 rctx = ahash_request_ctx(mcryptd_req);
850 areq = &rctx->areq;
851
852 ahash_request_set_tfm(areq, child);
853 ahash_request_set_callback(areq, CRYPTO_TFM_REQ_MAY_SLEEP,
854 rctx->complete, req);
855
856 return crypto_ahash_import(mcryptd_req, in);
857 }
858
859 static struct ahash_alg sha1_mb_async_alg = {
860 .init = sha1_mb_async_init,
861 .update = sha1_mb_async_update,
862 .final = sha1_mb_async_final,
863 .finup = sha1_mb_async_finup,
864 .digest = sha1_mb_async_digest,
865 .export = sha1_mb_async_export,
866 .import = sha1_mb_async_import,
867 .halg = {
868 .digestsize = SHA1_DIGEST_SIZE,
869 .statesize = sizeof(struct sha1_hash_ctx),
870 .base = {
871 .cra_name = "sha1",
872 .cra_driver_name = "sha1_mb",
873 /*
874 * Low priority, since with few concurrent hash requests
875 * this is extremely slow due to the flush delay. Users
876 * whose workloads would benefit from this can request
877 * it explicitly by driver name, or can increase its
878 * priority at runtime using NETLINK_CRYPTO.
879 */
880 .cra_priority = 50,
881 .cra_flags = CRYPTO_ALG_ASYNC,
882 .cra_blocksize = SHA1_BLOCK_SIZE,
883 .cra_module = THIS_MODULE,
884 .cra_list = LIST_HEAD_INIT(sha1_mb_async_alg.halg.base.cra_list),
885 .cra_init = sha1_mb_async_init_tfm,
886 .cra_exit = sha1_mb_async_exit_tfm,
887 .cra_ctxsize = sizeof(struct sha1_mb_ctx),
888 .cra_alignmask = 0,
889 },
890 },
891 };
892
893 static unsigned long sha1_mb_flusher(struct mcryptd_alg_cstate *cstate)
894 {
895 struct mcryptd_hash_request_ctx *rctx;
896 unsigned long cur_time;
897 unsigned long next_flush = 0;
898 struct sha1_hash_ctx *sha_ctx;
899
900
901 cur_time = jiffies;
902
903 while (!list_empty(&cstate->work_list)) {
904 rctx = list_entry(cstate->work_list.next,
905 struct mcryptd_hash_request_ctx, waiter);
906 if (time_before(cur_time, rctx->tag.expire))
907 break;
908 kernel_fpu_begin();
909 sha_ctx = (struct sha1_hash_ctx *)
910 sha1_ctx_mgr_flush(cstate->mgr);
911 kernel_fpu_end();
912 if (!sha_ctx) {
913 pr_err("sha1_mb error: nothing got flushed for non-empty list\n");
914 break;
915 }
916 rctx = cast_hash_to_mcryptd_ctx(sha_ctx);
917 sha_finish_walk(&rctx, cstate, true);
918 sha_complete_job(rctx, cstate, 0);
919 }
920
921 if (!list_empty(&cstate->work_list)) {
922 rctx = list_entry(cstate->work_list.next,
923 struct mcryptd_hash_request_ctx, waiter);
924 /* get the hash context and then flush time */
925 next_flush = rctx->tag.expire;
926 mcryptd_arm_flusher(cstate, get_delay(next_flush));
927 }
928 return next_flush;
929 }
930
931 static int __init sha1_mb_mod_init(void)
932 {
933
934 int cpu;
935 int err;
936 struct mcryptd_alg_cstate *cpu_state;
937
938 /* check for dependent cpu features */
939 if (!boot_cpu_has(X86_FEATURE_AVX2) ||
940 !boot_cpu_has(X86_FEATURE_BMI2))
941 return -ENODEV;
942
943 /* initialize multibuffer structures */
944 sha1_mb_alg_state.alg_cstate = alloc_percpu(struct mcryptd_alg_cstate);
945
946 sha1_job_mgr_init = sha1_mb_mgr_init_avx2;
947 sha1_job_mgr_submit = sha1_mb_mgr_submit_avx2;
948 sha1_job_mgr_flush = sha1_mb_mgr_flush_avx2;
949 sha1_job_mgr_get_comp_job = sha1_mb_mgr_get_comp_job_avx2;
950
951 if (!sha1_mb_alg_state.alg_cstate)
952 return -ENOMEM;
953 for_each_possible_cpu(cpu) {
954 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
955 cpu_state->next_flush = 0;
956 cpu_state->next_seq_num = 0;
957 cpu_state->flusher_engaged = false;
958 INIT_DELAYED_WORK(&cpu_state->flush, mcryptd_flusher);
959 cpu_state->cpu = cpu;
960 cpu_state->alg_state = &sha1_mb_alg_state;
961 cpu_state->mgr = kzalloc(sizeof(struct sha1_ctx_mgr),
962 GFP_KERNEL);
963 if (!cpu_state->mgr)
964 goto err2;
965 sha1_ctx_mgr_init(cpu_state->mgr);
966 INIT_LIST_HEAD(&cpu_state->work_list);
967 spin_lock_init(&cpu_state->work_lock);
968 }
969 sha1_mb_alg_state.flusher = &sha1_mb_flusher;
970
971 err = crypto_register_ahash(&sha1_mb_areq_alg);
972 if (err)
973 goto err2;
974 err = crypto_register_ahash(&sha1_mb_async_alg);
975 if (err)
976 goto err1;
977
978
979 return 0;
980 err1:
981 crypto_unregister_ahash(&sha1_mb_areq_alg);
982 err2:
983 for_each_possible_cpu(cpu) {
984 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
985 kfree(cpu_state->mgr);
986 }
987 free_percpu(sha1_mb_alg_state.alg_cstate);
988 return -ENODEV;
989 }
990
991 static void __exit sha1_mb_mod_fini(void)
992 {
993 int cpu;
994 struct mcryptd_alg_cstate *cpu_state;
995
996 crypto_unregister_ahash(&sha1_mb_async_alg);
997 crypto_unregister_ahash(&sha1_mb_areq_alg);
998 for_each_possible_cpu(cpu) {
999 cpu_state = per_cpu_ptr(sha1_mb_alg_state.alg_cstate, cpu);
1000 kfree(cpu_state->mgr);
1001 }
1002 free_percpu(sha1_mb_alg_state.alg_cstate);
1003 }
1004
1005 module_init(sha1_mb_mod_init);
1006 module_exit(sha1_mb_mod_fini);
1007
1008 MODULE_LICENSE("GPL");
1009 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, multi buffer accelerated");
1010
1011 MODULE_ALIAS_CRYPTO("sha1");
Linux with added FPC-III support