?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / crypto / chelsio / chcr_algo.c
blob34a02d6905489ee8459a6c64daead1b25c4c658e
1 /*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42 #define pr_fmt(fmt) "chcr:" fmt
43
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/cryptohash.h>
48 #include <linux/skbuff.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/highmem.h>
51 #include <linux/scatterlist.h>
52
53 #include <crypto/aes.h>
54 #include <crypto/algapi.h>
55 #include <crypto/hash.h>
56 #include <crypto/gcm.h>
57 #include <crypto/sha.h>
58 #include <crypto/authenc.h>
59 #include <crypto/ctr.h>
60 #include <crypto/gf128mul.h>
61 #include <crypto/internal/aead.h>
62 #include <crypto/null.h>
63 #include <crypto/internal/skcipher.h>
64 #include <crypto/aead.h>
65 #include <crypto/scatterwalk.h>
66 #include <crypto/internal/hash.h>
67
68 #include "t4fw_api.h"
69 #include "t4_msg.h"
70 #include "chcr_core.h"
71 #include "chcr_algo.h"
72 #include "chcr_crypto.h"
73
74 #define IV AES_BLOCK_SIZE
75
76 static unsigned int sgl_ent_len[] = {
77 0, 0, 16, 24, 40, 48, 64, 72, 88,
78 96, 112, 120, 136, 144, 160, 168, 184,
79 192, 208, 216, 232, 240, 256, 264, 280,
80 288, 304, 312, 328, 336, 352, 360, 376
81 };
82
83 static unsigned int dsgl_ent_len[] = {
84 0, 32, 32, 48, 48, 64, 64, 80, 80,
85 112, 112, 128, 128, 144, 144, 160, 160,
86 192, 192, 208, 208, 224, 224, 240, 240,
87 272, 272, 288, 288, 304, 304, 320, 320
88 };
89
90 static u32 round_constant[11] = {
91 0x01000000, 0x02000000, 0x04000000, 0x08000000,
92 0x10000000, 0x20000000, 0x40000000, 0x80000000,
93 0x1B000000, 0x36000000, 0x6C000000
94 };
95
96 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
97 unsigned char *input, int err);
98
99 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
100 {
101 return ctx->crypto_ctx->aeadctx;
102 }
103
104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
105 {
106 return ctx->crypto_ctx->ablkctx;
107 }
108
109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
110 {
111 return ctx->crypto_ctx->hmacctx;
112 }
113
114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
115 {
116 return gctx->ctx->gcm;
117 }
118
119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
120 {
121 return gctx->ctx->authenc;
122 }
123
124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
125 {
126 return ctx->dev->u_ctx;
127 }
128
129 static inline int is_ofld_imm(const struct sk_buff *skb)
130 {
131 return (skb->len <= SGE_MAX_WR_LEN);
132 }
133
134 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
135 unsigned int entlen,
136 unsigned int skip)
137 {
138 int nents = 0;
139 unsigned int less;
140 unsigned int skip_len = 0;
141
142 while (sg && skip) {
143 if (sg_dma_len(sg) <= skip) {
144 skip -= sg_dma_len(sg);
145 skip_len = 0;
146 sg = sg_next(sg);
147 } else {
148 skip_len = skip;
149 skip = 0;
150 }
151 }
152
153 while (sg && reqlen) {
154 less = min(reqlen, sg_dma_len(sg) - skip_len);
155 nents += DIV_ROUND_UP(less, entlen);
156 reqlen -= less;
157 skip_len = 0;
158 sg = sg_next(sg);
159 }
160 return nents;
161 }
162
163 static inline void chcr_handle_ahash_resp(struct ahash_request *req,
164 unsigned char *input,
165 int err)
166 {
167 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
168 int digestsize, updated_digestsize;
169 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
170 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
171
172 if (input == NULL)
173 goto out;
174 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
175 if (reqctx->is_sg_map)
176 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
177 if (reqctx->dma_addr)
178 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->dma_addr,
179 reqctx->dma_len, DMA_TO_DEVICE);
180 reqctx->dma_addr = 0;
181 updated_digestsize = digestsize;
182 if (digestsize == SHA224_DIGEST_SIZE)
183 updated_digestsize = SHA256_DIGEST_SIZE;
184 else if (digestsize == SHA384_DIGEST_SIZE)
185 updated_digestsize = SHA512_DIGEST_SIZE;
186 if (reqctx->result == 1) {
187 reqctx->result = 0;
188 memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
189 digestsize);
190 } else {
191 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
192 updated_digestsize);
193 }
194 out:
195 req->base.complete(&req->base, err);
196 }
197
198 static inline int get_aead_subtype(struct crypto_aead *aead)
199 {
200 struct aead_alg *alg = crypto_aead_alg(aead);
201 struct chcr_alg_template *chcr_crypto_alg =
202 container_of(alg, struct chcr_alg_template, alg.aead);
203 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
204 }
205
206 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
207 {
208 u8 temp[SHA512_DIGEST_SIZE];
209 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
210 int authsize = crypto_aead_authsize(tfm);
211 struct cpl_fw6_pld *fw6_pld;
212 int cmp = 0;
213
214 fw6_pld = (struct cpl_fw6_pld *)input;
215 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
216 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
217 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
218 } else {
219
220 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
221 authsize, req->assoclen +
222 req->cryptlen - authsize);
223 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
224 }
225 if (cmp)
226 *err = -EBADMSG;
227 else
228 *err = 0;
229 }
230
231 static inline void chcr_handle_aead_resp(struct aead_request *req,
232 unsigned char *input,
233 int err)
234 {
235 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
236 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
237 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
238
239 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
240 if (reqctx->b0_dma)
241 dma_unmap_single(&u_ctx->lldi.pdev->dev, reqctx->b0_dma,
242 reqctx->b0_len, DMA_BIDIRECTIONAL);
243 if (reqctx->verify == VERIFY_SW) {
244 chcr_verify_tag(req, input, &err);
245 reqctx->verify = VERIFY_HW;
246 }
247 req->base.complete(&req->base, err);
248 }
249
250 /*
251 * chcr_handle_resp - Unmap the DMA buffers associated with the request
252 * @req: crypto request
253 */
254 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
255 int err)
256 {
257 struct crypto_tfm *tfm = req->tfm;
258 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
259 struct adapter *adap = padap(ctx->dev);
260
261 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
262 case CRYPTO_ALG_TYPE_AEAD:
263 chcr_handle_aead_resp(aead_request_cast(req), input, err);
264 break;
265
266 case CRYPTO_ALG_TYPE_ABLKCIPHER:
267 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
268 input, err);
269 break;
270
271 case CRYPTO_ALG_TYPE_AHASH:
272 chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
273 }
274 atomic_inc(&adap->chcr_stats.complete);
275 return err;
276 }
277
278 static void get_aes_decrypt_key(unsigned char *dec_key,
279 const unsigned char *key,
280 unsigned int keylength)
281 {
282 u32 temp;
283 u32 w_ring[MAX_NK];
284 int i, j, k;
285 u8 nr, nk;
286
287 switch (keylength) {
288 case AES_KEYLENGTH_128BIT:
289 nk = KEYLENGTH_4BYTES;
290 nr = NUMBER_OF_ROUNDS_10;
291 break;
292 case AES_KEYLENGTH_192BIT:
293 nk = KEYLENGTH_6BYTES;
294 nr = NUMBER_OF_ROUNDS_12;
295 break;
296 case AES_KEYLENGTH_256BIT:
297 nk = KEYLENGTH_8BYTES;
298 nr = NUMBER_OF_ROUNDS_14;
299 break;
300 default:
301 return;
302 }
303 for (i = 0; i < nk; i++)
304 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
305
306 i = 0;
307 temp = w_ring[nk - 1];
308 while (i + nk < (nr + 1) * 4) {
309 if (!(i % nk)) {
310 /* RotWord(temp) */
311 temp = (temp << 8) | (temp >> 24);
312 temp = aes_ks_subword(temp);
313 temp ^= round_constant[i / nk];
314 } else if (nk == 8 && (i % 4 == 0)) {
315 temp = aes_ks_subword(temp);
316 }
317 w_ring[i % nk] ^= temp;
318 temp = w_ring[i % nk];
319 i++;
320 }
321 i--;
322 for (k = 0, j = i % nk; k < nk; k++) {
323 *((u32 *)dec_key + k) = htonl(w_ring[j]);
324 j--;
325 if (j < 0)
326 j += nk;
327 }
328 }
329
330 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
331 {
332 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
333
334 switch (ds) {
335 case SHA1_DIGEST_SIZE:
336 base_hash = crypto_alloc_shash("sha1", 0, 0);
337 break;
338 case SHA224_DIGEST_SIZE:
339 base_hash = crypto_alloc_shash("sha224", 0, 0);
340 break;
341 case SHA256_DIGEST_SIZE:
342 base_hash = crypto_alloc_shash("sha256", 0, 0);
343 break;
344 case SHA384_DIGEST_SIZE:
345 base_hash = crypto_alloc_shash("sha384", 0, 0);
346 break;
347 case SHA512_DIGEST_SIZE:
348 base_hash = crypto_alloc_shash("sha512", 0, 0);
349 break;
350 }
351
352 return base_hash;
353 }
354
355 static int chcr_compute_partial_hash(struct shash_desc *desc,
356 char *iopad, char *result_hash,
357 int digest_size)
358 {
359 struct sha1_state sha1_st;
360 struct sha256_state sha256_st;
361 struct sha512_state sha512_st;
362 int error;
363
364 if (digest_size == SHA1_DIGEST_SIZE) {
365 error = crypto_shash_init(desc) ?:
366 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
367 crypto_shash_export(desc, (void *)&sha1_st);
368 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
369 } else if (digest_size == SHA224_DIGEST_SIZE) {
370 error = crypto_shash_init(desc) ?:
371 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
372 crypto_shash_export(desc, (void *)&sha256_st);
373 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
374
375 } else if (digest_size == SHA256_DIGEST_SIZE) {
376 error = crypto_shash_init(desc) ?:
377 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
378 crypto_shash_export(desc, (void *)&sha256_st);
379 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
380
381 } else if (digest_size == SHA384_DIGEST_SIZE) {
382 error = crypto_shash_init(desc) ?:
383 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
384 crypto_shash_export(desc, (void *)&sha512_st);
385 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
386
387 } else if (digest_size == SHA512_DIGEST_SIZE) {
388 error = crypto_shash_init(desc) ?:
389 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
390 crypto_shash_export(desc, (void *)&sha512_st);
391 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
392 } else {
393 error = -EINVAL;
394 pr_err("Unknown digest size %d\n", digest_size);
395 }
396 return error;
397 }
398
399 static void chcr_change_order(char *buf, int ds)
400 {
401 int i;
402
403 if (ds == SHA512_DIGEST_SIZE) {
404 for (i = 0; i < (ds / sizeof(u64)); i++)
405 *((__be64 *)buf + i) =
406 cpu_to_be64(*((u64 *)buf + i));
407 } else {
408 for (i = 0; i < (ds / sizeof(u32)); i++)
409 *((__be32 *)buf + i) =
410 cpu_to_be32(*((u32 *)buf + i));
411 }
412 }
413
414 static inline int is_hmac(struct crypto_tfm *tfm)
415 {
416 struct crypto_alg *alg = tfm->__crt_alg;
417 struct chcr_alg_template *chcr_crypto_alg =
418 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
419 alg.hash);
420 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
421 return 1;
422 return 0;
423 }
424
425 static inline void dsgl_walk_init(struct dsgl_walk *walk,
426 struct cpl_rx_phys_dsgl *dsgl)
427 {
428 walk->dsgl = dsgl;
429 walk->nents = 0;
430 walk->to = (struct phys_sge_pairs *)(dsgl + 1);
431 }
432
433 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid)
434 {
435 struct cpl_rx_phys_dsgl *phys_cpl;
436
437 phys_cpl = walk->dsgl;
438
439 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
440 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
441 phys_cpl->pcirlxorder_to_noofsgentr =
442 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
443 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
444 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
445 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
446 CPL_RX_PHYS_DSGL_DCAID_V(0) |
447 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
448 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
449 phys_cpl->rss_hdr_int.qid = htons(qid);
450 phys_cpl->rss_hdr_int.hash_val = 0;
451 }
452
453 static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
454 size_t size,
455 dma_addr_t *addr)
456 {
457 int j;
458
459 if (!size)
460 return;
461 j = walk->nents;
462 walk->to->len[j % 8] = htons(size);
463 walk->to->addr[j % 8] = cpu_to_be64(*addr);
464 j++;
465 if ((j % 8) == 0)
466 walk->to++;
467 walk->nents = j;
468 }
469
470 static void dsgl_walk_add_sg(struct dsgl_walk *walk,
471 struct scatterlist *sg,
472 unsigned int slen,
473 unsigned int skip)
474 {
475 int skip_len = 0;
476 unsigned int left_size = slen, len = 0;
477 unsigned int j = walk->nents;
478 int offset, ent_len;
479
480 if (!slen)
481 return;
482 while (sg && skip) {
483 if (sg_dma_len(sg) <= skip) {
484 skip -= sg_dma_len(sg);
485 skip_len = 0;
486 sg = sg_next(sg);
487 } else {
488 skip_len = skip;
489 skip = 0;
490 }
491 }
492
493 while (left_size && sg) {
494 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
495 offset = 0;
496 while (len) {
497 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
498 walk->to->len[j % 8] = htons(ent_len);
499 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
500 offset + skip_len);
501 offset += ent_len;
502 len -= ent_len;
503 j++;
504 if ((j % 8) == 0)
505 walk->to++;
506 }
507 walk->last_sg = sg;
508 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
509 skip_len) + skip_len;
510 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
511 skip_len = 0;
512 sg = sg_next(sg);
513 }
514 walk->nents = j;
515 }
516
517 static inline void ulptx_walk_init(struct ulptx_walk *walk,
518 struct ulptx_sgl *ulp)
519 {
520 walk->sgl = ulp;
521 walk->nents = 0;
522 walk->pair_idx = 0;
523 walk->pair = ulp->sge;
524 walk->last_sg = NULL;
525 walk->last_sg_len = 0;
526 }
527
528 static inline void ulptx_walk_end(struct ulptx_walk *walk)
529 {
530 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
531 ULPTX_NSGE_V(walk->nents));
532 }
533
534
535 static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
536 size_t size,
537 dma_addr_t *addr)
538 {
539 if (!size)
540 return;
541
542 if (walk->nents == 0) {
543 walk->sgl->len0 = cpu_to_be32(size);
544 walk->sgl->addr0 = cpu_to_be64(*addr);
545 } else {
546 walk->pair->addr[walk->pair_idx] = cpu_to_be64(*addr);
547 walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
548 walk->pair_idx = !walk->pair_idx;
549 if (!walk->pair_idx)
550 walk->pair++;
551 }
552 walk->nents++;
553 }
554
555 static void ulptx_walk_add_sg(struct ulptx_walk *walk,
556 struct scatterlist *sg,
557 unsigned int len,
558 unsigned int skip)
559 {
560 int small;
561 int skip_len = 0;
562 unsigned int sgmin;
563
564 if (!len)
565 return;
566
567 while (sg && skip) {
568 if (sg_dma_len(sg) <= skip) {
569 skip -= sg_dma_len(sg);
570 skip_len = 0;
571 sg = sg_next(sg);
572 } else {
573 skip_len = skip;
574 skip = 0;
575 }
576 }
577 WARN(!sg, "SG should not be null here\n");
578 if (sg && (walk->nents == 0)) {
579 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
580 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
581 walk->sgl->len0 = cpu_to_be32(sgmin);
582 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
583 walk->nents++;
584 len -= sgmin;
585 walk->last_sg = sg;
586 walk->last_sg_len = sgmin + skip_len;
587 skip_len += sgmin;
588 if (sg_dma_len(sg) == skip_len) {
589 sg = sg_next(sg);
590 skip_len = 0;
591 }
592 }
593
594 while (sg && len) {
595 small = min(sg_dma_len(sg) - skip_len, len);
596 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
597 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
598 walk->pair->addr[walk->pair_idx] =
599 cpu_to_be64(sg_dma_address(sg) + skip_len);
600 walk->pair_idx = !walk->pair_idx;
601 walk->nents++;
602 if (!walk->pair_idx)
603 walk->pair++;
604 len -= sgmin;
605 skip_len += sgmin;
606 walk->last_sg = sg;
607 walk->last_sg_len = skip_len;
608 if (sg_dma_len(sg) == skip_len) {
609 sg = sg_next(sg);
610 skip_len = 0;
611 }
612 }
613 }
614
615 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
616 {
617 struct crypto_alg *alg = tfm->__crt_alg;
618 struct chcr_alg_template *chcr_crypto_alg =
619 container_of(alg, struct chcr_alg_template, alg.crypto);
620
621 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
622 }
623
624 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
625 {
626 struct adapter *adap = netdev2adap(dev);
627 struct sge_uld_txq_info *txq_info =
628 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
629 struct sge_uld_txq *txq;
630 int ret = 0;
631
632 local_bh_disable();
633 txq = &txq_info->uldtxq[idx];
634 spin_lock(&txq->sendq.lock);
635 if (txq->full)
636 ret = -1;
637 spin_unlock(&txq->sendq.lock);
638 local_bh_enable();
639 return ret;
640 }
641
642 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
643 struct _key_ctx *key_ctx)
644 {
645 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
646 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
647 } else {
648 memcpy(key_ctx->key,
649 ablkctx->key + (ablkctx->enckey_len >> 1),
650 ablkctx->enckey_len >> 1);
651 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
652 ablkctx->rrkey, ablkctx->enckey_len >> 1);
653 }
654 return 0;
655 }
656 static int chcr_sg_ent_in_wr(struct scatterlist *src,
657 struct scatterlist *dst,
658 unsigned int minsg,
659 unsigned int space,
660 unsigned int srcskip,
661 unsigned int dstskip)
662 {
663 int srclen = 0, dstlen = 0;
664 int srcsg = minsg, dstsg = minsg;
665 int offset = 0, less;
666
667 if (sg_dma_len(src) == srcskip) {
668 src = sg_next(src);
669 srcskip = 0;
670 }
671
672 if (sg_dma_len(dst) == dstskip) {
673 dst = sg_next(dst);
674 dstskip = 0;
675 }
676
677 while (src && dst &&
678 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
679 srclen += (sg_dma_len(src) - srcskip);
680 srcsg++;
681 offset = 0;
682 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
683 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
684 if (srclen <= dstlen)
685 break;
686 less = min_t(unsigned int, sg_dma_len(dst) - offset -
687 dstskip, CHCR_DST_SG_SIZE);
688 dstlen += less;
689 offset += less;
690 if (offset == sg_dma_len(dst)) {
691 dst = sg_next(dst);
692 offset = 0;
693 }
694 dstsg++;
695 dstskip = 0;
696 }
697 src = sg_next(src);
698 srcskip = 0;
699 }
700 return min(srclen, dstlen);
701 }
702
703 static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
704 u32 flags,
705 struct scatterlist *src,
706 struct scatterlist *dst,
707 unsigned int nbytes,
708 u8 *iv,
709 unsigned short op_type)
710 {
711 int err;
712
713 SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
714 skcipher_request_set_tfm(subreq, cipher);
715 skcipher_request_set_callback(subreq, flags, NULL, NULL);
716 skcipher_request_set_crypt(subreq, src, dst,
717 nbytes, iv);
718
719 err = op_type ? crypto_skcipher_decrypt(subreq) :
720 crypto_skcipher_encrypt(subreq);
721 skcipher_request_zero(subreq);
722
723 return err;
724
725 }
726 static inline void create_wreq(struct chcr_context *ctx,
727 struct chcr_wr *chcr_req,
728 struct crypto_async_request *req,
729 unsigned int imm,
730 int hash_sz,
731 unsigned int len16,
732 unsigned int sc_len,
733 unsigned int lcb)
734 {
735 struct uld_ctx *u_ctx = ULD_CTX(ctx);
736 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
737
738
739 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
740 chcr_req->wreq.pld_size_hash_size =
741 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
742 chcr_req->wreq.len16_pkd =
743 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
744 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
745 chcr_req->wreq.rx_chid_to_rx_q_id =
746 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
747 !!lcb, ctx->tx_qidx);
748
749 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
750 qid);
751 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
752 ((sizeof(chcr_req->wreq)) >> 4)));
753
754 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
755 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
756 sizeof(chcr_req->key_ctx) + sc_len);
757 }
758
759 /**
760 * create_cipher_wr - form the WR for cipher operations
761 * @req: cipher req.
762 * @ctx: crypto driver context of the request.
763 * @qid: ingress qid where response of this WR should be received.
764 * @op_type: encryption or decryption
765 */
766 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
767 {
768 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
769 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
770 struct sk_buff *skb = NULL;
771 struct chcr_wr *chcr_req;
772 struct cpl_rx_phys_dsgl *phys_cpl;
773 struct ulptx_sgl *ulptx;
774 struct chcr_blkcipher_req_ctx *reqctx =
775 ablkcipher_request_ctx(wrparam->req);
776 unsigned int temp = 0, transhdr_len, dst_size;
777 int error;
778 int nents;
779 unsigned int kctx_len;
780 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
781 GFP_KERNEL : GFP_ATOMIC;
782 struct adapter *adap = padap(c_ctx(tfm)->dev);
783
784 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE,
785 reqctx->dst_ofst);
786 dst_size = get_space_for_phys_dsgl(nents + 1);
787 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
788 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
789 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
790 CHCR_SRC_SG_SIZE, reqctx->src_ofst);
791 temp = reqctx->imm ? (DIV_ROUND_UP((IV + wrparam->req->nbytes), 16)
792 * 16) : (sgl_len(nents + MIN_CIPHER_SG) * 8);
793 transhdr_len += temp;
794 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
795 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
796 if (!skb) {
797 error = -ENOMEM;
798 goto err;
799 }
800 chcr_req = __skb_put_zero(skb, transhdr_len);
801 chcr_req->sec_cpl.op_ivinsrtofst =
802 FILL_SEC_CPL_OP_IVINSR(c_ctx(tfm)->dev->rx_channel_id, 2, 1);
803
804 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
805 chcr_req->sec_cpl.aadstart_cipherstop_hi =
806 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
807
808 chcr_req->sec_cpl.cipherstop_lo_authinsert =
809 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
810 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
811 ablkctx->ciph_mode,
812 0, 0, IV >> 1);
813 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
814 0, 0, dst_size);
815
816 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
817 if ((reqctx->op == CHCR_DECRYPT_OP) &&
818 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
819 CRYPTO_ALG_SUB_TYPE_CTR)) &&
820 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
821 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
822 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
823 } else {
824 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
825 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
826 memcpy(chcr_req->key_ctx.key, ablkctx->key,
827 ablkctx->enckey_len);
828 } else {
829 memcpy(chcr_req->key_ctx.key, ablkctx->key +
830 (ablkctx->enckey_len >> 1),
831 ablkctx->enckey_len >> 1);
832 memcpy(chcr_req->key_ctx.key +
833 (ablkctx->enckey_len >> 1),
834 ablkctx->key,
835 ablkctx->enckey_len >> 1);
836 }
837 }
838 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
839 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
840 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
841 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
842
843 atomic_inc(&adap->chcr_stats.cipher_rqst);
844 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len
845 +(reqctx->imm ? (IV + wrparam->bytes) : 0);
846 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
847 transhdr_len, temp,
848 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
849 reqctx->skb = skb;
850 return skb;
851 err:
852 return ERR_PTR(error);
853 }
854
855 static inline int chcr_keyctx_ck_size(unsigned int keylen)
856 {
857 int ck_size = 0;
858
859 if (keylen == AES_KEYSIZE_128)
860 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
861 else if (keylen == AES_KEYSIZE_192)
862 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
863 else if (keylen == AES_KEYSIZE_256)
864 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
865 else
866 ck_size = 0;
867
868 return ck_size;
869 }
870 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
871 const u8 *key,
872 unsigned int keylen)
873 {
874 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
875 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
876 int err = 0;
877
878 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
879 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
880 CRYPTO_TFM_REQ_MASK);
881 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
882 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
883 tfm->crt_flags |=
884 crypto_skcipher_get_flags(ablkctx->sw_cipher) &
885 CRYPTO_TFM_RES_MASK;
886 return err;
887 }
888
889 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
890 const u8 *key,
891 unsigned int keylen)
892 {
893 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
894 unsigned int ck_size, context_size;
895 u16 alignment = 0;
896 int err;
897
898 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
899 if (err)
900 goto badkey_err;
901
902 ck_size = chcr_keyctx_ck_size(keylen);
903 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
904 memcpy(ablkctx->key, key, keylen);
905 ablkctx->enckey_len = keylen;
906 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
907 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
908 keylen + alignment) >> 4;
909
910 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
911 0, 0, context_size);
912 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
913 return 0;
914 badkey_err:
915 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
916 ablkctx->enckey_len = 0;
917
918 return err;
919 }
920
921 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
922 const u8 *key,
923 unsigned int keylen)
924 {
925 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
926 unsigned int ck_size, context_size;
927 u16 alignment = 0;
928 int err;
929
930 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
931 if (err)
932 goto badkey_err;
933 ck_size = chcr_keyctx_ck_size(keylen);
934 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
935 memcpy(ablkctx->key, key, keylen);
936 ablkctx->enckey_len = keylen;
937 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
938 keylen + alignment) >> 4;
939
940 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
941 0, 0, context_size);
942 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
943
944 return 0;
945 badkey_err:
946 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
947 ablkctx->enckey_len = 0;
948
949 return err;
950 }
951
952 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
953 const u8 *key,
954 unsigned int keylen)
955 {
956 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
957 unsigned int ck_size, context_size;
958 u16 alignment = 0;
959 int err;
960
961 if (keylen < CTR_RFC3686_NONCE_SIZE)
962 return -EINVAL;
963 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
964 CTR_RFC3686_NONCE_SIZE);
965
966 keylen -= CTR_RFC3686_NONCE_SIZE;
967 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
968 if (err)
969 goto badkey_err;
970
971 ck_size = chcr_keyctx_ck_size(keylen);
972 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
973 memcpy(ablkctx->key, key, keylen);
974 ablkctx->enckey_len = keylen;
975 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
976 keylen + alignment) >> 4;
977
978 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
979 0, 0, context_size);
980 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
981
982 return 0;
983 badkey_err:
984 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
985 ablkctx->enckey_len = 0;
986
987 return err;
988 }
989 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
990 {
991 unsigned int size = AES_BLOCK_SIZE;
992 __be32 *b = (__be32 *)(dstiv + size);
993 u32 c, prev;
994
995 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
996 for (; size >= 4; size -= 4) {
997 prev = be32_to_cpu(*--b);
998 c = prev + add;
999 *b = cpu_to_be32(c);
1000 if (prev < c)
1001 break;
1002 add = 1;
1003 }
1004
1005 }
1006
1007 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1008 {
1009 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1010 u64 c;
1011 u32 temp = be32_to_cpu(*--b);
1012
1013 temp = ~temp;
1014 c = (u64)temp + 1; // No of block can processed withou overflow
1015 if ((bytes / AES_BLOCK_SIZE) > c)
1016 bytes = c * AES_BLOCK_SIZE;
1017 return bytes;
1018 }
1019
1020 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv,
1021 u32 isfinal)
1022 {
1023 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1024 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1025 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1026 struct crypto_cipher *cipher;
1027 int ret, i;
1028 u8 *key;
1029 unsigned int keylen;
1030 int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1031 int round8 = round / 8;
1032
1033 cipher = ablkctx->aes_generic;
1034 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1035
1036 keylen = ablkctx->enckey_len / 2;
1037 key = ablkctx->key + keylen;
1038 ret = crypto_cipher_setkey(cipher, key, keylen);
1039 if (ret)
1040 goto out;
1041 /*H/W sends the encrypted IV in dsgl when AADIVDROP bit is 0*/
1042 for (i = 0; i < round8; i++)
1043 gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1044
1045 for (i = 0; i < (round % 8); i++)
1046 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1047
1048 if (!isfinal)
1049 crypto_cipher_decrypt_one(cipher, iv, iv);
1050 out:
1051 return ret;
1052 }
1053
1054 static int chcr_update_cipher_iv(struct ablkcipher_request *req,
1055 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1056 {
1057 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1058 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1059 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
1060 int ret = 0;
1061
1062 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1063 ctr_add_iv(iv, req->info, (reqctx->processed /
1064 AES_BLOCK_SIZE));
1065 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1066 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1067 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1068 AES_BLOCK_SIZE) + 1);
1069 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1070 ret = chcr_update_tweak(req, iv, 0);
1071 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1072 if (reqctx->op)
1073 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
1074 16,
1075 reqctx->processed - AES_BLOCK_SIZE);
1076 else
1077 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1078 }
1079
1080 return ret;
1081
1082 }
1083
1084 /* We need separate function for final iv because in rfc3686 Initial counter
1085 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1086 * for subsequent update requests
1087 */
1088
1089 static int chcr_final_cipher_iv(struct ablkcipher_request *req,
1090 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1091 {
1092 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1093 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1094 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
1095 int ret = 0;
1096
1097 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1098 ctr_add_iv(iv, req->info, (reqctx->processed /
1099 AES_BLOCK_SIZE));
1100 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1101 ret = chcr_update_tweak(req, iv, 1);
1102 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1103 if (reqctx->op)
1104 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
1105 16,
1106 reqctx->processed - AES_BLOCK_SIZE);
1107 else
1108 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1109
1110 }
1111 return ret;
1112
1113 }
1114
1115 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
1116 unsigned char *input, int err)
1117 {
1118 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1119 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1120 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1121 struct sk_buff *skb;
1122 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1123 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1124 struct cipher_wr_param wrparam;
1125 int bytes;
1126
1127 if (err)
1128 goto unmap;
1129 if (req->nbytes == reqctx->processed) {
1130 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1131 req);
1132 err = chcr_final_cipher_iv(req, fw6_pld, req->info);
1133 goto complete;
1134 }
1135
1136 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1137 c_ctx(tfm)->tx_qidx))) {
1138 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1139 err = -EBUSY;
1140 goto unmap;
1141 }
1142
1143 }
1144 if (!reqctx->imm) {
1145 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 1,
1146 SPACE_LEFT(ablkctx->enckey_len),
1147 reqctx->src_ofst, reqctx->dst_ofst);
1148 if ((bytes + reqctx->processed) >= req->nbytes)
1149 bytes = req->nbytes - reqctx->processed;
1150 else
1151 bytes = ROUND_16(bytes);
1152 } else {
1153 /*CTR mode counter overfloa*/
1154 bytes = req->nbytes - reqctx->processed;
1155 }
1156 dma_sync_single_for_cpu(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1157 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
1158 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1159 dma_sync_single_for_device(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1160 reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
1161 if (err)
1162 goto unmap;
1163
1164 if (unlikely(bytes == 0)) {
1165 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1166 req);
1167 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1168 req->base.flags,
1169 req->src,
1170 req->dst,
1171 req->nbytes,
1172 req->info,
1173 reqctx->op);
1174 goto complete;
1175 }
1176
1177 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1178 CRYPTO_ALG_SUB_TYPE_CTR)
1179 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1180 wrparam.qid = u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx];
1181 wrparam.req = req;
1182 wrparam.bytes = bytes;
1183 skb = create_cipher_wr(&wrparam);
1184 if (IS_ERR(skb)) {
1185 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1186 err = PTR_ERR(skb);
1187 goto unmap;
1188 }
1189 skb->dev = u_ctx->lldi.ports[0];
1190 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
1191 chcr_send_wr(skb);
1192 reqctx->last_req_len = bytes;
1193 reqctx->processed += bytes;
1194 return 0;
1195 unmap:
1196 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1197 complete:
1198 req->base.complete(&req->base, err);
1199 return err;
1200 }
1201
1202 static int process_cipher(struct ablkcipher_request *req,
1203 unsigned short qid,
1204 struct sk_buff **skb,
1205 unsigned short op_type)
1206 {
1207 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1208 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
1209 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1210 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1211 struct cipher_wr_param wrparam;
1212 int bytes, err = -EINVAL;
1213
1214 reqctx->processed = 0;
1215 if (!req->info)
1216 goto error;
1217 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1218 (req->nbytes == 0) ||
1219 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
1220 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1221 ablkctx->enckey_len, req->nbytes, ivsize);
1222 goto error;
1223 }
1224 chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1225 if (req->nbytes < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1226 AES_MIN_KEY_SIZE +
1227 sizeof(struct cpl_rx_phys_dsgl) +
1228 /*Min dsgl size*/
1229 32))) {
1230 /* Can be sent as Imm*/
1231 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1232
1233 dnents = sg_nents_xlen(req->dst, req->nbytes,
1234 CHCR_DST_SG_SIZE, 0);
1235 dnents += 1; // IV
1236 phys_dsgl = get_space_for_phys_dsgl(dnents);
1237 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
1238 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1239 reqctx->imm = (transhdr_len + IV + req->nbytes) <=
1240 SGE_MAX_WR_LEN;
1241 bytes = IV + req->nbytes;
1242
1243 } else {
1244 reqctx->imm = 0;
1245 }
1246
1247 if (!reqctx->imm) {
1248 bytes = chcr_sg_ent_in_wr(req->src, req->dst,
1249 MIN_CIPHER_SG,
1250 SPACE_LEFT(ablkctx->enckey_len),
1251 0, 0);
1252 if ((bytes + reqctx->processed) >= req->nbytes)
1253 bytes = req->nbytes - reqctx->processed;
1254 else
1255 bytes = ROUND_16(bytes);
1256 } else {
1257 bytes = req->nbytes;
1258 }
1259 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1260 CRYPTO_ALG_SUB_TYPE_CTR) {
1261 bytes = adjust_ctr_overflow(req->info, bytes);
1262 }
1263 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1264 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1265 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1266 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
1267 CTR_RFC3686_IV_SIZE);
1268
1269 /* initialize counter portion of counter block */
1270 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1271 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1272
1273 } else {
1274
1275 memcpy(reqctx->iv, req->info, IV);
1276 }
1277 if (unlikely(bytes == 0)) {
1278 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1279 req);
1280 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1281 req->base.flags,
1282 req->src,
1283 req->dst,
1284 req->nbytes,
1285 req->info,
1286 op_type);
1287 goto error;
1288 }
1289 reqctx->op = op_type;
1290 reqctx->srcsg = req->src;
1291 reqctx->dstsg = req->dst;
1292 reqctx->src_ofst = 0;
1293 reqctx->dst_ofst = 0;
1294 wrparam.qid = qid;
1295 wrparam.req = req;
1296 wrparam.bytes = bytes;
1297 *skb = create_cipher_wr(&wrparam);
1298 if (IS_ERR(*skb)) {
1299 err = PTR_ERR(*skb);
1300 goto unmap;
1301 }
1302 reqctx->processed = bytes;
1303 reqctx->last_req_len = bytes;
1304
1305 return 0;
1306 unmap:
1307 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1308 error:
1309 return err;
1310 }
1311
1312 static int chcr_aes_encrypt(struct ablkcipher_request *req)
1313 {
1314 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1315 struct sk_buff *skb = NULL;
1316 int err;
1317 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1318
1319 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1320 c_ctx(tfm)->tx_qidx))) {
1321 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1322 return -EBUSY;
1323 }
1324
1325 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
1326 &skb, CHCR_ENCRYPT_OP);
1327 if (err || !skb)
1328 return err;
1329 skb->dev = u_ctx->lldi.ports[0];
1330 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
1331 chcr_send_wr(skb);
1332 return -EINPROGRESS;
1333 }
1334
1335 static int chcr_aes_decrypt(struct ablkcipher_request *req)
1336 {
1337 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1338 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1339 struct sk_buff *skb = NULL;
1340 int err;
1341
1342 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1343 c_ctx(tfm)->tx_qidx))) {
1344 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1345 return -EBUSY;
1346 }
1347
1348 err = process_cipher(req, u_ctx->lldi.rxq_ids[c_ctx(tfm)->rx_qidx],
1349 &skb, CHCR_DECRYPT_OP);
1350 if (err || !skb)
1351 return err;
1352 skb->dev = u_ctx->lldi.ports[0];
1353 set_wr_txq(skb, CPL_PRIORITY_DATA, c_ctx(tfm)->tx_qidx);
1354 chcr_send_wr(skb);
1355 return -EINPROGRESS;
1356 }
1357
1358 static int chcr_device_init(struct chcr_context *ctx)
1359 {
1360 struct uld_ctx *u_ctx = NULL;
1361 struct adapter *adap;
1362 unsigned int id;
1363 int txq_perchan, txq_idx, ntxq;
1364 int err = 0, rxq_perchan, rxq_idx;
1365
1366 id = smp_processor_id();
1367 if (!ctx->dev) {
1368 u_ctx = assign_chcr_device();
1369 if (!u_ctx) {
1370 pr_err("chcr device assignment fails\n");
1371 goto out;
1372 }
1373 ctx->dev = u_ctx->dev;
1374 adap = padap(ctx->dev);
1375 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
1376 adap->vres.ncrypto_fc);
1377 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1378 txq_perchan = ntxq / u_ctx->lldi.nchan;
1379 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
1380 rxq_idx += id % rxq_perchan;
1381 txq_idx = ctx->dev->tx_channel_id * txq_perchan;
1382 txq_idx += id % txq_perchan;
1383 spin_lock(&ctx->dev->lock_chcr_dev);
1384 ctx->rx_qidx = rxq_idx;
1385 ctx->tx_qidx = txq_idx;
1386 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
1387 ctx->dev->rx_channel_id = 0;
1388 spin_unlock(&ctx->dev->lock_chcr_dev);
1389 }
1390 out:
1391 return err;
1392 }
1393
1394 static int chcr_cra_init(struct crypto_tfm *tfm)
1395 {
1396 struct crypto_alg *alg = tfm->__crt_alg;
1397 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1398 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1399
1400 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
1401 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1402 if (IS_ERR(ablkctx->sw_cipher)) {
1403 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1404 return PTR_ERR(ablkctx->sw_cipher);
1405 }
1406
1407 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) {
1408 /* To update tweak*/
1409 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0);
1410 if (IS_ERR(ablkctx->aes_generic)) {
1411 pr_err("failed to allocate aes cipher for tweak\n");
1412 return PTR_ERR(ablkctx->aes_generic);
1413 }
1414 } else
1415 ablkctx->aes_generic = NULL;
1416
1417 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1418 return chcr_device_init(crypto_tfm_ctx(tfm));
1419 }
1420
1421 static int chcr_rfc3686_init(struct crypto_tfm *tfm)
1422 {
1423 struct crypto_alg *alg = tfm->__crt_alg;
1424 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1425 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1426
1427 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1428 * cannot be used as fallback in chcr_handle_cipher_response
1429 */
1430 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1431 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1432 if (IS_ERR(ablkctx->sw_cipher)) {
1433 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1434 return PTR_ERR(ablkctx->sw_cipher);
1435 }
1436 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1437 return chcr_device_init(crypto_tfm_ctx(tfm));
1438 }
1439
1440
1441 static void chcr_cra_exit(struct crypto_tfm *tfm)
1442 {
1443 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1444 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1445
1446 crypto_free_skcipher(ablkctx->sw_cipher);
1447 if (ablkctx->aes_generic)
1448 crypto_free_cipher(ablkctx->aes_generic);
1449 }
1450
1451 static int get_alg_config(struct algo_param *params,
1452 unsigned int auth_size)
1453 {
1454 switch (auth_size) {
1455 case SHA1_DIGEST_SIZE:
1456 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1457 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1458 params->result_size = SHA1_DIGEST_SIZE;
1459 break;
1460 case SHA224_DIGEST_SIZE:
1461 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1462 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1463 params->result_size = SHA256_DIGEST_SIZE;
1464 break;
1465 case SHA256_DIGEST_SIZE:
1466 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1467 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1468 params->result_size = SHA256_DIGEST_SIZE;
1469 break;
1470 case SHA384_DIGEST_SIZE:
1471 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1472 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1473 params->result_size = SHA512_DIGEST_SIZE;
1474 break;
1475 case SHA512_DIGEST_SIZE:
1476 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1477 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1478 params->result_size = SHA512_DIGEST_SIZE;
1479 break;
1480 default:
1481 pr_err("chcr : ERROR, unsupported digest size\n");
1482 return -EINVAL;
1483 }
1484 return 0;
1485 }
1486
1487 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1488 {
1489 crypto_free_shash(base_hash);
1490 }
1491
1492 /**
1493 * create_hash_wr - Create hash work request
1494 * @req - Cipher req base
1495 */
1496 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1497 struct hash_wr_param *param)
1498 {
1499 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1500 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1501 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
1502 struct sk_buff *skb = NULL;
1503 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
1504 struct chcr_wr *chcr_req;
1505 struct ulptx_sgl *ulptx;
1506 unsigned int nents = 0, transhdr_len, iopad_alignment = 0;
1507 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1508 unsigned int kctx_len = 0, temp = 0;
1509 u8 hash_size_in_response = 0;
1510 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1511 GFP_ATOMIC;
1512 struct adapter *adap = padap(h_ctx(tfm)->dev);
1513 int error = 0;
1514
1515 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
1516 kctx_len = param->alg_prm.result_size + iopad_alignment;
1517 if (param->opad_needed)
1518 kctx_len += param->alg_prm.result_size + iopad_alignment;
1519
1520 if (req_ctx->result)
1521 hash_size_in_response = digestsize;
1522 else
1523 hash_size_in_response = param->alg_prm.result_size;
1524 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
1525 req_ctx->imm = (transhdr_len + param->bfr_len + param->sg_len) <=
1526 SGE_MAX_WR_LEN;
1527 nents = sg_nents_xlen(req->src, param->sg_len, CHCR_SRC_SG_SIZE, 0);
1528 nents += param->bfr_len ? 1 : 0;
1529 transhdr_len += req_ctx->imm ? (DIV_ROUND_UP((param->bfr_len +
1530 param->sg_len), 16) * 16) :
1531 (sgl_len(nents) * 8);
1532 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
1533
1534 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
1535 if (!skb)
1536 return ERR_PTR(-ENOMEM);
1537 chcr_req = __skb_put_zero(skb, transhdr_len);
1538
1539 chcr_req->sec_cpl.op_ivinsrtofst =
1540 FILL_SEC_CPL_OP_IVINSR(h_ctx(tfm)->dev->rx_channel_id, 2, 0);
1541 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1542
1543 chcr_req->sec_cpl.aadstart_cipherstop_hi =
1544 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1545 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1546 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1547 chcr_req->sec_cpl.seqno_numivs =
1548 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1549 param->opad_needed, 0);
1550
1551 chcr_req->sec_cpl.ivgen_hdrlen =
1552 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1553
1554 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1555 param->alg_prm.result_size);
1556
1557 if (param->opad_needed)
1558 memcpy(chcr_req->key_ctx.key +
1559 ((param->alg_prm.result_size <= 32) ? 32 :
1560 CHCR_HASH_MAX_DIGEST_SIZE),
1561 hmacctx->opad, param->alg_prm.result_size);
1562
1563 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1564 param->alg_prm.mk_size, 0,
1565 param->opad_needed,
1566 ((kctx_len +
1567 sizeof(chcr_req->key_ctx)) >> 4));
1568 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1569 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + kctx_len +
1570 DUMMY_BYTES);
1571 if (param->bfr_len != 0) {
1572 req_ctx->dma_addr = dma_map_single(&u_ctx->lldi.pdev->dev,
1573 req_ctx->reqbfr, param->bfr_len,
1574 DMA_TO_DEVICE);
1575 if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
1576 req_ctx->dma_addr)) {
1577 error = -ENOMEM;
1578 goto err;
1579 }
1580 req_ctx->dma_len = param->bfr_len;
1581 } else {
1582 req_ctx->dma_addr = 0;
1583 }
1584 chcr_add_hash_src_ent(req, ulptx, param);
1585 /* Request upto max wr size */
1586 temp = kctx_len + DUMMY_BYTES + (req_ctx->imm ? (param->sg_len
1587 + param->bfr_len) : 0);
1588 atomic_inc(&adap->chcr_stats.digest_rqst);
1589 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->imm,
1590 hash_size_in_response, transhdr_len,
1591 temp, 0);
1592 req_ctx->skb = skb;
1593 return skb;
1594 err:
1595 kfree_skb(skb);
1596 return ERR_PTR(error);
1597 }
1598
1599 static int chcr_ahash_update(struct ahash_request *req)
1600 {
1601 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1602 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1603 struct uld_ctx *u_ctx = NULL;
1604 struct sk_buff *skb;
1605 u8 remainder = 0, bs;
1606 unsigned int nbytes = req->nbytes;
1607 struct hash_wr_param params;
1608 int error;
1609
1610 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1611
1612 u_ctx = ULD_CTX(h_ctx(rtfm));
1613 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1614 h_ctx(rtfm)->tx_qidx))) {
1615 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1616 return -EBUSY;
1617 }
1618
1619 if (nbytes + req_ctx->reqlen >= bs) {
1620 remainder = (nbytes + req_ctx->reqlen) % bs;
1621 nbytes = nbytes + req_ctx->reqlen - remainder;
1622 } else {
1623 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1624 + req_ctx->reqlen, nbytes, 0);
1625 req_ctx->reqlen += nbytes;
1626 return 0;
1627 }
1628 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1629 if (error)
1630 return -ENOMEM;
1631 params.opad_needed = 0;
1632 params.more = 1;
1633 params.last = 0;
1634 params.sg_len = nbytes - req_ctx->reqlen;
1635 params.bfr_len = req_ctx->reqlen;
1636 params.scmd1 = 0;
1637 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1638 req_ctx->result = 0;
1639 req_ctx->data_len += params.sg_len + params.bfr_len;
1640 skb = create_hash_wr(req, &params);
1641 if (IS_ERR(skb)) {
1642 error = PTR_ERR(skb);
1643 goto unmap;
1644 }
1645
1646 if (remainder) {
1647 /* Swap buffers */
1648 swap(req_ctx->reqbfr, req_ctx->skbfr);
1649 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1650 req_ctx->reqbfr, remainder, req->nbytes -
1651 remainder);
1652 }
1653 req_ctx->reqlen = remainder;
1654 skb->dev = u_ctx->lldi.ports[0];
1655 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1656 chcr_send_wr(skb);
1657
1658 return -EINPROGRESS;
1659 unmap:
1660 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1661 return error;
1662 }
1663
1664 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1665 {
1666 memset(bfr_ptr, 0, bs);
1667 *bfr_ptr = 0x80;
1668 if (bs == 64)
1669 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1670 else
1671 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1672 }
1673
1674 static int chcr_ahash_final(struct ahash_request *req)
1675 {
1676 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1677 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1678 struct hash_wr_param params;
1679 struct sk_buff *skb;
1680 struct uld_ctx *u_ctx = NULL;
1681 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1682
1683 u_ctx = ULD_CTX(h_ctx(rtfm));
1684 if (is_hmac(crypto_ahash_tfm(rtfm)))
1685 params.opad_needed = 1;
1686 else
1687 params.opad_needed = 0;
1688 params.sg_len = 0;
1689 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1690 req_ctx->result = 1;
1691 params.bfr_len = req_ctx->reqlen;
1692 req_ctx->data_len += params.bfr_len + params.sg_len;
1693 if (req_ctx->reqlen == 0) {
1694 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1695 params.last = 0;
1696 params.more = 1;
1697 params.scmd1 = 0;
1698 params.bfr_len = bs;
1699
1700 } else {
1701 params.scmd1 = req_ctx->data_len;
1702 params.last = 1;
1703 params.more = 0;
1704 }
1705 skb = create_hash_wr(req, &params);
1706 if (IS_ERR(skb))
1707 return PTR_ERR(skb);
1708
1709 skb->dev = u_ctx->lldi.ports[0];
1710 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1711 chcr_send_wr(skb);
1712 return -EINPROGRESS;
1713 }
1714
1715 static int chcr_ahash_finup(struct ahash_request *req)
1716 {
1717 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1718 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1719 struct uld_ctx *u_ctx = NULL;
1720 struct sk_buff *skb;
1721 struct hash_wr_param params;
1722 u8 bs;
1723 int error;
1724
1725 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1726 u_ctx = ULD_CTX(h_ctx(rtfm));
1727
1728 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1729 h_ctx(rtfm)->tx_qidx))) {
1730 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1731 return -EBUSY;
1732 }
1733
1734 if (is_hmac(crypto_ahash_tfm(rtfm)))
1735 params.opad_needed = 1;
1736 else
1737 params.opad_needed = 0;
1738
1739 params.sg_len = req->nbytes;
1740 params.bfr_len = req_ctx->reqlen;
1741 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1742 req_ctx->data_len += params.bfr_len + params.sg_len;
1743 req_ctx->result = 1;
1744 if ((req_ctx->reqlen + req->nbytes) == 0) {
1745 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1746 params.last = 0;
1747 params.more = 1;
1748 params.scmd1 = 0;
1749 params.bfr_len = bs;
1750 } else {
1751 params.scmd1 = req_ctx->data_len;
1752 params.last = 1;
1753 params.more = 0;
1754 }
1755 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1756 if (error)
1757 return -ENOMEM;
1758
1759 skb = create_hash_wr(req, &params);
1760 if (IS_ERR(skb)) {
1761 error = PTR_ERR(skb);
1762 goto unmap;
1763 }
1764 skb->dev = u_ctx->lldi.ports[0];
1765 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1766 chcr_send_wr(skb);
1767
1768 return -EINPROGRESS;
1769 unmap:
1770 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1771 return error;
1772 }
1773
1774 static int chcr_ahash_digest(struct ahash_request *req)
1775 {
1776 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1777 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1778 struct uld_ctx *u_ctx = NULL;
1779 struct sk_buff *skb;
1780 struct hash_wr_param params;
1781 u8 bs;
1782 int error;
1783
1784 rtfm->init(req);
1785 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1786
1787 u_ctx = ULD_CTX(h_ctx(rtfm));
1788 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1789 h_ctx(rtfm)->tx_qidx))) {
1790 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1791 return -EBUSY;
1792 }
1793
1794 if (is_hmac(crypto_ahash_tfm(rtfm)))
1795 params.opad_needed = 1;
1796 else
1797 params.opad_needed = 0;
1798 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1799 if (error)
1800 return -ENOMEM;
1801
1802 params.last = 0;
1803 params.more = 0;
1804 params.sg_len = req->nbytes;
1805 params.bfr_len = 0;
1806 params.scmd1 = 0;
1807 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1808 req_ctx->result = 1;
1809 req_ctx->data_len += params.bfr_len + params.sg_len;
1810
1811 if (req->nbytes == 0) {
1812 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1813 params.more = 1;
1814 params.bfr_len = bs;
1815 }
1816
1817 skb = create_hash_wr(req, &params);
1818 if (IS_ERR(skb)) {
1819 error = PTR_ERR(skb);
1820 goto unmap;
1821 }
1822 skb->dev = u_ctx->lldi.ports[0];
1823 set_wr_txq(skb, CPL_PRIORITY_DATA, h_ctx(rtfm)->tx_qidx);
1824 chcr_send_wr(skb);
1825 return -EINPROGRESS;
1826 unmap:
1827 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1828 return error;
1829 }
1830
1831 static int chcr_ahash_export(struct ahash_request *areq, void *out)
1832 {
1833 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1834 struct chcr_ahash_req_ctx *state = out;
1835
1836 state->reqlen = req_ctx->reqlen;
1837 state->data_len = req_ctx->data_len;
1838 state->is_sg_map = 0;
1839 state->result = 0;
1840 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
1841 memcpy(state->partial_hash, req_ctx->partial_hash,
1842 CHCR_HASH_MAX_DIGEST_SIZE);
1843 return 0;
1844 }
1845
1846 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1847 {
1848 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1849 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1850
1851 req_ctx->reqlen = state->reqlen;
1852 req_ctx->data_len = state->data_len;
1853 req_ctx->reqbfr = req_ctx->bfr1;
1854 req_ctx->skbfr = req_ctx->bfr2;
1855 req_ctx->is_sg_map = 0;
1856 req_ctx->result = 0;
1857 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
1858 memcpy(req_ctx->partial_hash, state->partial_hash,
1859 CHCR_HASH_MAX_DIGEST_SIZE);
1860 return 0;
1861 }
1862
1863 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1864 unsigned int keylen)
1865 {
1866 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
1867 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1868 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1869 unsigned int i, err = 0, updated_digestsize;
1870
1871 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1872
1873 /* use the key to calculate the ipad and opad. ipad will sent with the
1874 * first request's data. opad will be sent with the final hash result
1875 * ipad in hmacctx->ipad and opad in hmacctx->opad location
1876 */
1877 shash->tfm = hmacctx->base_hash;
1878 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
1879 if (keylen > bs) {
1880 err = crypto_shash_digest(shash, key, keylen,
1881 hmacctx->ipad);
1882 if (err)
1883 goto out;
1884 keylen = digestsize;
1885 } else {
1886 memcpy(hmacctx->ipad, key, keylen);
1887 }
1888 memset(hmacctx->ipad + keylen, 0, bs - keylen);
1889 memcpy(hmacctx->opad, hmacctx->ipad, bs);
1890
1891 for (i = 0; i < bs / sizeof(int); i++) {
1892 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1893 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1894 }
1895
1896 updated_digestsize = digestsize;
1897 if (digestsize == SHA224_DIGEST_SIZE)
1898 updated_digestsize = SHA256_DIGEST_SIZE;
1899 else if (digestsize == SHA384_DIGEST_SIZE)
1900 updated_digestsize = SHA512_DIGEST_SIZE;
1901 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
1902 hmacctx->ipad, digestsize);
1903 if (err)
1904 goto out;
1905 chcr_change_order(hmacctx->ipad, updated_digestsize);
1906
1907 err = chcr_compute_partial_hash(shash, hmacctx->opad,
1908 hmacctx->opad, digestsize);
1909 if (err)
1910 goto out;
1911 chcr_change_order(hmacctx->opad, updated_digestsize);
1912 out:
1913 return err;
1914 }
1915
1916 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1917 unsigned int key_len)
1918 {
1919 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
1920 unsigned short context_size = 0;
1921 int err;
1922
1923 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
1924 if (err)
1925 goto badkey_err;
1926
1927 memcpy(ablkctx->key, key, key_len);
1928 ablkctx->enckey_len = key_len;
1929 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1930 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1931 ablkctx->key_ctx_hdr =
1932 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1933 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1934 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1935 CHCR_KEYCTX_NO_KEY, 1,
1936 0, context_size);
1937 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1938 return 0;
1939 badkey_err:
1940 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1941 ablkctx->enckey_len = 0;
1942
1943 return err;
1944 }
1945
1946 static int chcr_sha_init(struct ahash_request *areq)
1947 {
1948 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1949 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1950 int digestsize = crypto_ahash_digestsize(tfm);
1951
1952 req_ctx->data_len = 0;
1953 req_ctx->reqlen = 0;
1954 req_ctx->reqbfr = req_ctx->bfr1;
1955 req_ctx->skbfr = req_ctx->bfr2;
1956 req_ctx->skb = NULL;
1957 req_ctx->result = 0;
1958 req_ctx->is_sg_map = 0;
1959 copy_hash_init_values(req_ctx->partial_hash, digestsize);
1960 return 0;
1961 }
1962
1963 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1964 {
1965 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1966 sizeof(struct chcr_ahash_req_ctx));
1967 return chcr_device_init(crypto_tfm_ctx(tfm));
1968 }
1969
1970 static int chcr_hmac_init(struct ahash_request *areq)
1971 {
1972 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1973 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1974 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
1975 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1976 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1977
1978 chcr_sha_init(areq);
1979 req_ctx->data_len = bs;
1980 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1981 if (digestsize == SHA224_DIGEST_SIZE)
1982 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1983 SHA256_DIGEST_SIZE);
1984 else if (digestsize == SHA384_DIGEST_SIZE)
1985 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1986 SHA512_DIGEST_SIZE);
1987 else
1988 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1989 digestsize);
1990 }
1991 return 0;
1992 }
1993
1994 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1995 {
1996 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1997 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1998 unsigned int digestsize =
1999 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
2000
2001 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2002 sizeof(struct chcr_ahash_req_ctx));
2003 hmacctx->base_hash = chcr_alloc_shash(digestsize);
2004 if (IS_ERR(hmacctx->base_hash))
2005 return PTR_ERR(hmacctx->base_hash);
2006 return chcr_device_init(crypto_tfm_ctx(tfm));
2007 }
2008
2009 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
2010 {
2011 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2012 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2013
2014 if (hmacctx->base_hash) {
2015 chcr_free_shash(hmacctx->base_hash);
2016 hmacctx->base_hash = NULL;
2017 }
2018 }
2019
2020 static int chcr_aead_common_init(struct aead_request *req,
2021 unsigned short op_type)
2022 {
2023 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2024 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2025 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2026 int error = -EINVAL;
2027 unsigned int authsize = crypto_aead_authsize(tfm);
2028
2029 /* validate key size */
2030 if (aeadctx->enckey_len == 0)
2031 goto err;
2032 if (op_type && req->cryptlen < authsize)
2033 goto err;
2034 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2035 op_type);
2036 if (error) {
2037 error = -ENOMEM;
2038 goto err;
2039 }
2040 reqctx->aad_nents = sg_nents_xlen(req->src, req->assoclen,
2041 CHCR_SRC_SG_SIZE, 0);
2042 reqctx->src_nents = sg_nents_xlen(req->src, req->cryptlen,
2043 CHCR_SRC_SG_SIZE, req->assoclen);
2044 return 0;
2045 err:
2046 return error;
2047 }
2048
2049 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
2050 int aadmax, int wrlen,
2051 unsigned short op_type)
2052 {
2053 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2054
2055 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2056 dst_nents > MAX_DSGL_ENT ||
2057 (req->assoclen > aadmax) ||
2058 (wrlen > SGE_MAX_WR_LEN))
2059 return 1;
2060 return 0;
2061 }
2062
2063 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2064 {
2065 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2066 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2067 struct aead_request *subreq = aead_request_ctx(req);
2068
2069 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2070 aead_request_set_callback(subreq, req->base.flags,
2071 req->base.complete, req->base.data);
2072 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2073 req->iv);
2074 aead_request_set_ad(subreq, req->assoclen);
2075 return op_type ? crypto_aead_decrypt(subreq) :
2076 crypto_aead_encrypt(subreq);
2077 }
2078
2079 static struct sk_buff *create_authenc_wr(struct aead_request *req,
2080 unsigned short qid,
2081 int size,
2082 unsigned short op_type)
2083 {
2084 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2085 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2086 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2087 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2088 struct sk_buff *skb = NULL;
2089 struct chcr_wr *chcr_req;
2090 struct cpl_rx_phys_dsgl *phys_cpl;
2091 struct ulptx_sgl *ulptx;
2092 unsigned int transhdr_len;
2093 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
2094 unsigned int kctx_len = 0, dnents;
2095 unsigned int assoclen = req->assoclen;
2096 unsigned int authsize = crypto_aead_authsize(tfm);
2097 int error = -EINVAL;
2098 int null = 0;
2099 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2100 GFP_ATOMIC;
2101 struct adapter *adap = padap(a_ctx(tfm)->dev);
2102
2103 if (req->cryptlen == 0)
2104 return NULL;
2105
2106 reqctx->b0_dma = 0;
2107 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
2108 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2109 null = 1;
2110 assoclen = 0;
2111 }
2112 error = chcr_aead_common_init(req, op_type);
2113 if (error)
2114 return ERR_PTR(error);
2115 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2116 dnents += sg_nents_xlen(req->dst, req->cryptlen +
2117 (op_type ? -authsize : authsize), CHCR_DST_SG_SIZE,
2118 req->assoclen);
2119 dnents += MIN_AUTH_SG; // For IV
2120
2121 dst_size = get_space_for_phys_dsgl(dnents);
2122 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
2123 - sizeof(chcr_req->key_ctx);
2124 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2125 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <
2126 SGE_MAX_WR_LEN;
2127 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen), 16)
2128 * 16) : (sgl_len(reqctx->src_nents + reqctx->aad_nents
2129 + MIN_GCM_SG) * 8);
2130 transhdr_len += temp;
2131 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
2132
2133 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2134 transhdr_len, op_type)) {
2135 atomic_inc(&adap->chcr_stats.fallback);
2136 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2137 op_type);
2138 return ERR_PTR(chcr_aead_fallback(req, op_type));
2139 }
2140 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2141 if (!skb) {
2142 error = -ENOMEM;
2143 goto err;
2144 }
2145
2146 chcr_req = __skb_put_zero(skb, transhdr_len);
2147
2148 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2149
2150 /*
2151 * Input order is AAD,IV and Payload. where IV should be included as
2152 * the part of authdata. All other fields should be filled according
2153 * to the hardware spec
2154 */
2155 chcr_req->sec_cpl.op_ivinsrtofst =
2156 FILL_SEC_CPL_OP_IVINSR(a_ctx(tfm)->dev->rx_channel_id, 2,
2157 assoclen + 1);
2158 chcr_req->sec_cpl.pldlen = htonl(assoclen + IV + req->cryptlen);
2159 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2160 assoclen ? 1 : 0, assoclen,
2161 assoclen + IV + 1,
2162 (temp & 0x1F0) >> 4);
2163 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2164 temp & 0xF,
2165 null ? 0 : assoclen + IV + 1,
2166 temp, temp);
2167 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2168 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2169 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2170 else
2171 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
2172 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2173 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
2174 temp,
2175 actx->auth_mode, aeadctx->hmac_ctrl,
2176 IV >> 1);
2177 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2178 0, 0, dst_size);
2179
2180 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2181 if (op_type == CHCR_ENCRYPT_OP ||
2182 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2183 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2184 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2185 aeadctx->enckey_len);
2186 else
2187 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2188 aeadctx->enckey_len);
2189
2190 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
2191 4), actx->h_iopad, kctx_len -
2192 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
2193 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2194 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2195 memcpy(reqctx->iv, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2196 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
2197 CTR_RFC3686_IV_SIZE);
2198 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
2199 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2200 } else {
2201 memcpy(reqctx->iv, req->iv, IV);
2202 }
2203 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2204 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2205 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
2206 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
2207 atomic_inc(&adap->chcr_stats.cipher_rqst);
2208 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2209 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
2210 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2211 transhdr_len, temp, 0);
2212 reqctx->skb = skb;
2213 reqctx->op = op_type;
2214
2215 return skb;
2216 err:
2217 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2218 op_type);
2219
2220 return ERR_PTR(error);
2221 }
2222
2223 int chcr_aead_dma_map(struct device *dev,
2224 struct aead_request *req,
2225 unsigned short op_type)
2226 {
2227 int error;
2228 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2229 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2230 unsigned int authsize = crypto_aead_authsize(tfm);
2231 int dst_size;
2232
2233 dst_size = req->assoclen + req->cryptlen + (op_type ?
2234 -authsize : authsize);
2235 if (!req->cryptlen || !dst_size)
2236 return 0;
2237 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
2238 DMA_BIDIRECTIONAL);
2239 if (dma_mapping_error(dev, reqctx->iv_dma))
2240 return -ENOMEM;
2241
2242 if (req->src == req->dst) {
2243 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2244 DMA_BIDIRECTIONAL);
2245 if (!error)
2246 goto err;
2247 } else {
2248 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2249 DMA_TO_DEVICE);
2250 if (!error)
2251 goto err;
2252 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2253 DMA_FROM_DEVICE);
2254 if (!error) {
2255 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2256 DMA_TO_DEVICE);
2257 goto err;
2258 }
2259 }
2260
2261 return 0;
2262 err:
2263 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2264 return -ENOMEM;
2265 }
2266
2267 void chcr_aead_dma_unmap(struct device *dev,
2268 struct aead_request *req,
2269 unsigned short op_type)
2270 {
2271 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2272 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2273 unsigned int authsize = crypto_aead_authsize(tfm);
2274 int dst_size;
2275
2276 dst_size = req->assoclen + req->cryptlen + (op_type ?
2277 -authsize : authsize);
2278 if (!req->cryptlen || !dst_size)
2279 return;
2280
2281 dma_unmap_single(dev, reqctx->iv_dma, IV,
2282 DMA_BIDIRECTIONAL);
2283 if (req->src == req->dst) {
2284 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2285 DMA_BIDIRECTIONAL);
2286 } else {
2287 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2288 DMA_TO_DEVICE);
2289 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2290 DMA_FROM_DEVICE);
2291 }
2292 }
2293
2294 void chcr_add_aead_src_ent(struct aead_request *req,
2295 struct ulptx_sgl *ulptx,
2296 unsigned int assoclen,
2297 unsigned short op_type)
2298 {
2299 struct ulptx_walk ulp_walk;
2300 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2301
2302 if (reqctx->imm) {
2303 u8 *buf = (u8 *)ulptx;
2304
2305 if (reqctx->b0_dma) {
2306 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2307 buf += reqctx->b0_len;
2308 }
2309 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2310 buf, assoclen, 0);
2311 buf += assoclen;
2312 memcpy(buf, reqctx->iv, IV);
2313 buf += IV;
2314 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2315 buf, req->cryptlen, req->assoclen);
2316 } else {
2317 ulptx_walk_init(&ulp_walk, ulptx);
2318 if (reqctx->b0_dma)
2319 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
2320 &reqctx->b0_dma);
2321 ulptx_walk_add_sg(&ulp_walk, req->src, assoclen, 0);
2322 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
2323 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen,
2324 req->assoclen);
2325 ulptx_walk_end(&ulp_walk);
2326 }
2327 }
2328
2329 void chcr_add_aead_dst_ent(struct aead_request *req,
2330 struct cpl_rx_phys_dsgl *phys_cpl,
2331 unsigned int assoclen,
2332 unsigned short op_type,
2333 unsigned short qid)
2334 {
2335 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2336 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2337 struct dsgl_walk dsgl_walk;
2338 unsigned int authsize = crypto_aead_authsize(tfm);
2339 u32 temp;
2340
2341 dsgl_walk_init(&dsgl_walk, phys_cpl);
2342 if (reqctx->b0_dma)
2343 dsgl_walk_add_page(&dsgl_walk, reqctx->b0_len, &reqctx->b0_dma);
2344 dsgl_walk_add_sg(&dsgl_walk, req->dst, assoclen, 0);
2345 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
2346 temp = req->cryptlen + (op_type ? -authsize : authsize);
2347 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, req->assoclen);
2348 dsgl_walk_end(&dsgl_walk, qid);
2349 }
2350
2351 void chcr_add_cipher_src_ent(struct ablkcipher_request *req,
2352 struct ulptx_sgl *ulptx,
2353 struct cipher_wr_param *wrparam)
2354 {
2355 struct ulptx_walk ulp_walk;
2356 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2357
2358 if (reqctx->imm) {
2359 u8 *buf = (u8 *)ulptx;
2360
2361 memcpy(buf, reqctx->iv, IV);
2362 buf += IV;
2363 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2364 buf, wrparam->bytes, reqctx->processed);
2365 } else {
2366 ulptx_walk_init(&ulp_walk, ulptx);
2367 ulptx_walk_add_page(&ulp_walk, IV, &reqctx->iv_dma);
2368 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2369 reqctx->src_ofst);
2370 reqctx->srcsg = ulp_walk.last_sg;
2371 reqctx->src_ofst = ulp_walk.last_sg_len;
2372 ulptx_walk_end(&ulp_walk);
2373 }
2374 }
2375
2376 void chcr_add_cipher_dst_ent(struct ablkcipher_request *req,
2377 struct cpl_rx_phys_dsgl *phys_cpl,
2378 struct cipher_wr_param *wrparam,
2379 unsigned short qid)
2380 {
2381 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2382 struct dsgl_walk dsgl_walk;
2383
2384 dsgl_walk_init(&dsgl_walk, phys_cpl);
2385 dsgl_walk_add_page(&dsgl_walk, IV, &reqctx->iv_dma);
2386 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2387 reqctx->dst_ofst);
2388 reqctx->dstsg = dsgl_walk.last_sg;
2389 reqctx->dst_ofst = dsgl_walk.last_sg_len;
2390
2391 dsgl_walk_end(&dsgl_walk, qid);
2392 }
2393
2394 void chcr_add_hash_src_ent(struct ahash_request *req,
2395 struct ulptx_sgl *ulptx,
2396 struct hash_wr_param *param)
2397 {
2398 struct ulptx_walk ulp_walk;
2399 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2400
2401 if (reqctx->imm) {
2402 u8 *buf = (u8 *)ulptx;
2403
2404 if (param->bfr_len) {
2405 memcpy(buf, reqctx->reqbfr, param->bfr_len);
2406 buf += param->bfr_len;
2407 }
2408 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2409 buf, param->sg_len, 0);
2410 } else {
2411 ulptx_walk_init(&ulp_walk, ulptx);
2412 if (param->bfr_len)
2413 ulptx_walk_add_page(&ulp_walk, param->bfr_len,
2414 &reqctx->dma_addr);
2415 ulptx_walk_add_sg(&ulp_walk, req->src, param->sg_len,
2416 0);
2417 ulptx_walk_end(&ulp_walk);
2418 }
2419 }
2420
2421 int chcr_hash_dma_map(struct device *dev,
2422 struct ahash_request *req)
2423 {
2424 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2425 int error = 0;
2426
2427 if (!req->nbytes)
2428 return 0;
2429 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2430 DMA_TO_DEVICE);
2431 if (!error)
2432 return -ENOMEM;
2433 req_ctx->is_sg_map = 1;
2434 return 0;
2435 }
2436
2437 void chcr_hash_dma_unmap(struct device *dev,
2438 struct ahash_request *req)
2439 {
2440 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2441
2442 if (!req->nbytes)
2443 return;
2444
2445 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2446 DMA_TO_DEVICE);
2447 req_ctx->is_sg_map = 0;
2448
2449 }
2450
2451 int chcr_cipher_dma_map(struct device *dev,
2452 struct ablkcipher_request *req)
2453 {
2454 int error;
2455 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2456
2457 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, IV,
2458 DMA_BIDIRECTIONAL);
2459 if (dma_mapping_error(dev, reqctx->iv_dma))
2460 return -ENOMEM;
2461
2462 if (req->src == req->dst) {
2463 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2464 DMA_BIDIRECTIONAL);
2465 if (!error)
2466 goto err;
2467 } else {
2468 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2469 DMA_TO_DEVICE);
2470 if (!error)
2471 goto err;
2472 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2473 DMA_FROM_DEVICE);
2474 if (!error) {
2475 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2476 DMA_TO_DEVICE);
2477 goto err;
2478 }
2479 }
2480
2481 return 0;
2482 err:
2483 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2484 return -ENOMEM;
2485 }
2486
2487 void chcr_cipher_dma_unmap(struct device *dev,
2488 struct ablkcipher_request *req)
2489 {
2490 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
2491
2492 dma_unmap_single(dev, reqctx->iv_dma, IV,
2493 DMA_BIDIRECTIONAL);
2494 if (req->src == req->dst) {
2495 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2496 DMA_BIDIRECTIONAL);
2497 } else {
2498 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2499 DMA_TO_DEVICE);
2500 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2501 DMA_FROM_DEVICE);
2502 }
2503 }
2504
2505 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2506 {
2507 __be32 data;
2508
2509 memset(block, 0, csize);
2510 block += csize;
2511
2512 if (csize >= 4)
2513 csize = 4;
2514 else if (msglen > (unsigned int)(1 << (8 * csize)))
2515 return -EOVERFLOW;
2516
2517 data = cpu_to_be32(msglen);
2518 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2519
2520 return 0;
2521 }
2522
2523 static void generate_b0(struct aead_request *req,
2524 struct chcr_aead_ctx *aeadctx,
2525 unsigned short op_type)
2526 {
2527 unsigned int l, lp, m;
2528 int rc;
2529 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2530 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2531 u8 *b0 = reqctx->scratch_pad;
2532
2533 m = crypto_aead_authsize(aead);
2534
2535 memcpy(b0, reqctx->iv, 16);
2536
2537 lp = b0[0];
2538 l = lp + 1;
2539
2540 /* set m, bits 3-5 */
2541 *b0 |= (8 * ((m - 2) / 2));
2542
2543 /* set adata, bit 6, if associated data is used */
2544 if (req->assoclen)
2545 *b0 |= 64;
2546 rc = set_msg_len(b0 + 16 - l,
2547 (op_type == CHCR_DECRYPT_OP) ?
2548 req->cryptlen - m : req->cryptlen, l);
2549 }
2550
2551 static inline int crypto_ccm_check_iv(const u8 *iv)
2552 {
2553 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2554 if (iv[0] < 1 || iv[0] > 7)
2555 return -EINVAL;
2556
2557 return 0;
2558 }
2559
2560 static int ccm_format_packet(struct aead_request *req,
2561 struct chcr_aead_ctx *aeadctx,
2562 unsigned int sub_type,
2563 unsigned short op_type)
2564 {
2565 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2566 int rc = 0;
2567
2568 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2569 reqctx->iv[0] = 3;
2570 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
2571 memcpy(reqctx->iv + 4, req->iv, 8);
2572 memset(reqctx->iv + 12, 0, 4);
2573 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2574 htons(req->assoclen - 8);
2575 } else {
2576 memcpy(reqctx->iv, req->iv, 16);
2577 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2578 htons(req->assoclen);
2579 }
2580 generate_b0(req, aeadctx, op_type);
2581 /* zero the ctr value */
2582 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
2583 return rc;
2584 }
2585
2586 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2587 unsigned int dst_size,
2588 struct aead_request *req,
2589 unsigned short op_type)
2590 {
2591 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2592 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2593 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2594 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2595 unsigned int c_id = a_ctx(tfm)->dev->rx_channel_id;
2596 unsigned int ccm_xtra;
2597 unsigned char tag_offset = 0, auth_offset = 0;
2598 unsigned int assoclen;
2599
2600 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2601 assoclen = req->assoclen - 8;
2602 else
2603 assoclen = req->assoclen;
2604 ccm_xtra = CCM_B0_SIZE +
2605 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2606
2607 auth_offset = req->cryptlen ?
2608 (assoclen + IV + 1 + ccm_xtra) : 0;
2609 if (op_type == CHCR_DECRYPT_OP) {
2610 if (crypto_aead_authsize(tfm) != req->cryptlen)
2611 tag_offset = crypto_aead_authsize(tfm);
2612 else
2613 auth_offset = 0;
2614 }
2615
2616
2617 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
2618 2, assoclen + 1 + ccm_xtra);
2619 sec_cpl->pldlen =
2620 htonl(assoclen + IV + req->cryptlen + ccm_xtra);
2621 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2622 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2623 1, assoclen + ccm_xtra, assoclen
2624 + IV + 1 + ccm_xtra, 0);
2625
2626 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2627 auth_offset, tag_offset,
2628 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2629 crypto_aead_authsize(tfm));
2630 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2631 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2632 cipher_mode, mac_mode,
2633 aeadctx->hmac_ctrl, IV >> 1);
2634
2635 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2636 0, dst_size);
2637 }
2638
2639 int aead_ccm_validate_input(unsigned short op_type,
2640 struct aead_request *req,
2641 struct chcr_aead_ctx *aeadctx,
2642 unsigned int sub_type)
2643 {
2644 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2645 if (crypto_ccm_check_iv(req->iv)) {
2646 pr_err("CCM: IV check fails\n");
2647 return -EINVAL;
2648 }
2649 } else {
2650 if (req->assoclen != 16 && req->assoclen != 20) {
2651 pr_err("RFC4309: Invalid AAD length %d\n",
2652 req->assoclen);
2653 return -EINVAL;
2654 }
2655 }
2656 return 0;
2657 }
2658
2659 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2660 unsigned short qid,
2661 int size,
2662 unsigned short op_type)
2663 {
2664 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2665 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2666 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2667 struct sk_buff *skb = NULL;
2668 struct chcr_wr *chcr_req;
2669 struct cpl_rx_phys_dsgl *phys_cpl;
2670 struct ulptx_sgl *ulptx;
2671 unsigned int transhdr_len;
2672 unsigned int dst_size = 0, kctx_len, dnents, temp;
2673 unsigned int sub_type, assoclen = req->assoclen;
2674 unsigned int authsize = crypto_aead_authsize(tfm);
2675 int error = -EINVAL;
2676 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2677 GFP_ATOMIC;
2678 struct adapter *adap = padap(a_ctx(tfm)->dev);
2679
2680 reqctx->b0_dma = 0;
2681 sub_type = get_aead_subtype(tfm);
2682 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2683 assoclen -= 8;
2684 error = chcr_aead_common_init(req, op_type);
2685 if (error)
2686 return ERR_PTR(error);
2687
2688
2689 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2690 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
2691 if (error)
2692 goto err;
2693 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2694 dnents += sg_nents_xlen(req->dst, req->cryptlen
2695 + (op_type ? -authsize : authsize),
2696 CHCR_DST_SG_SIZE, req->assoclen);
2697 dnents += MIN_CCM_SG; // For IV and B0
2698 dst_size = get_space_for_phys_dsgl(dnents);
2699 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
2700 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2701 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen +
2702 reqctx->b0_len) <= SGE_MAX_WR_LEN;
2703 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV + req->cryptlen +
2704 reqctx->b0_len), 16) * 16) :
2705 (sgl_len(reqctx->src_nents + reqctx->aad_nents +
2706 MIN_CCM_SG) * 8);
2707 transhdr_len += temp;
2708 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
2709
2710 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
2711 reqctx->b0_len, transhdr_len, op_type)) {
2712 atomic_inc(&adap->chcr_stats.fallback);
2713 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2714 op_type);
2715 return ERR_PTR(chcr_aead_fallback(req, op_type));
2716 }
2717 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2718
2719 if (!skb) {
2720 error = -ENOMEM;
2721 goto err;
2722 }
2723
2724 chcr_req = (struct chcr_wr *) __skb_put_zero(skb, transhdr_len);
2725
2726 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type);
2727
2728 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2729 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2730 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2731 16), aeadctx->key, aeadctx->enckey_len);
2732
2733 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2734 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2735 error = ccm_format_packet(req, aeadctx, sub_type, op_type);
2736 if (error)
2737 goto dstmap_fail;
2738
2739 reqctx->b0_dma = dma_map_single(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
2740 &reqctx->scratch_pad, reqctx->b0_len,
2741 DMA_BIDIRECTIONAL);
2742 if (dma_mapping_error(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev,
2743 reqctx->b0_dma)) {
2744 error = -ENOMEM;
2745 goto dstmap_fail;
2746 }
2747
2748 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
2749 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
2750
2751 atomic_inc(&adap->chcr_stats.aead_rqst);
2752 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2753 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen +
2754 reqctx->b0_len) : 0);
2755 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
2756 transhdr_len, temp, 0);
2757 reqctx->skb = skb;
2758 reqctx->op = op_type;
2759
2760 return skb;
2761 dstmap_fail:
2762 kfree_skb(skb);
2763 err:
2764 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
2765 return ERR_PTR(error);
2766 }
2767
2768 static struct sk_buff *create_gcm_wr(struct aead_request *req,
2769 unsigned short qid,
2770 int size,
2771 unsigned short op_type)
2772 {
2773 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2774 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2775 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2776 struct sk_buff *skb = NULL;
2777 struct chcr_wr *chcr_req;
2778 struct cpl_rx_phys_dsgl *phys_cpl;
2779 struct ulptx_sgl *ulptx;
2780 unsigned int transhdr_len, dnents = 0;
2781 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
2782 unsigned int authsize = crypto_aead_authsize(tfm);
2783 int error = -EINVAL;
2784 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2785 GFP_ATOMIC;
2786 struct adapter *adap = padap(a_ctx(tfm)->dev);
2787
2788 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
2789 assoclen = req->assoclen - 8;
2790
2791 reqctx->b0_dma = 0;
2792 error = chcr_aead_common_init(req, op_type);
2793 if (error)
2794 return ERR_PTR(error);
2795 dnents = sg_nents_xlen(req->dst, assoclen, CHCR_DST_SG_SIZE, 0);
2796 dnents += sg_nents_xlen(req->dst, req->cryptlen +
2797 (op_type ? -authsize : authsize),
2798 CHCR_DST_SG_SIZE, req->assoclen);
2799 dnents += MIN_GCM_SG; // For IV
2800 dst_size = get_space_for_phys_dsgl(dnents);
2801 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
2802 AEAD_H_SIZE;
2803 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2804 reqctx->imm = (transhdr_len + assoclen + IV + req->cryptlen) <=
2805 SGE_MAX_WR_LEN;
2806 temp = reqctx->imm ? (DIV_ROUND_UP((assoclen + IV +
2807 req->cryptlen), 16) * 16) : (sgl_len(reqctx->src_nents +
2808 reqctx->aad_nents + MIN_GCM_SG) * 8);
2809 transhdr_len += temp;
2810 transhdr_len = DIV_ROUND_UP(transhdr_len, 16) * 16;
2811 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2812 transhdr_len, op_type)) {
2813 atomic_inc(&adap->chcr_stats.fallback);
2814 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2815 op_type);
2816 return ERR_PTR(chcr_aead_fallback(req, op_type));
2817 }
2818 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
2819 if (!skb) {
2820 error = -ENOMEM;
2821 goto err;
2822 }
2823
2824 chcr_req = __skb_put_zero(skb, transhdr_len);
2825
2826 //Offset of tag from end
2827 temp = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2828 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
2829 a_ctx(tfm)->dev->rx_channel_id, 2,
2830 (assoclen + 1));
2831 chcr_req->sec_cpl.pldlen =
2832 htonl(assoclen + IV + req->cryptlen);
2833 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2834 assoclen ? 1 : 0, assoclen,
2835 assoclen + IV + 1, 0);
2836 chcr_req->sec_cpl.cipherstop_lo_authinsert =
2837 FILL_SEC_CPL_AUTHINSERT(0, assoclen + IV + 1,
2838 temp, temp);
2839 chcr_req->sec_cpl.seqno_numivs =
2840 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
2841 CHCR_ENCRYPT_OP) ? 1 : 0,
2842 CHCR_SCMD_CIPHER_MODE_AES_GCM,
2843 CHCR_SCMD_AUTH_MODE_GHASH,
2844 aeadctx->hmac_ctrl, IV >> 1);
2845 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2846 0, 0, dst_size);
2847 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2848 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2849 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2850 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2851
2852 /* prepare a 16 byte iv */
2853 /* S A L T | IV | 0x00000001 */
2854 if (get_aead_subtype(tfm) ==
2855 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
2856 memcpy(reqctx->iv, aeadctx->salt, 4);
2857 memcpy(reqctx->iv + 4, req->iv, GCM_RFC4106_IV_SIZE);
2858 } else {
2859 memcpy(reqctx->iv, req->iv, GCM_AES_IV_SIZE);
2860 }
2861 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
2862
2863 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2864 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
2865
2866 chcr_add_aead_dst_ent(req, phys_cpl, assoclen, op_type, qid);
2867 chcr_add_aead_src_ent(req, ulptx, assoclen, op_type);
2868 atomic_inc(&adap->chcr_stats.aead_rqst);
2869 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size +
2870 kctx_len + (reqctx->imm ? (assoclen + IV + req->cryptlen) : 0);
2871 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2872 transhdr_len, temp, reqctx->verify);
2873 reqctx->skb = skb;
2874 reqctx->op = op_type;
2875 return skb;
2876
2877 err:
2878 chcr_aead_dma_unmap(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req, op_type);
2879 return ERR_PTR(error);
2880 }
2881
2882
2883
2884 static int chcr_aead_cra_init(struct crypto_aead *tfm)
2885 {
2886 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2887 struct aead_alg *alg = crypto_aead_alg(tfm);
2888
2889 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
2890 CRYPTO_ALG_NEED_FALLBACK |
2891 CRYPTO_ALG_ASYNC);
2892 if (IS_ERR(aeadctx->sw_cipher))
2893 return PTR_ERR(aeadctx->sw_cipher);
2894 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
2895 sizeof(struct aead_request) +
2896 crypto_aead_reqsize(aeadctx->sw_cipher)));
2897 return chcr_device_init(a_ctx(tfm));
2898 }
2899
2900 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
2901 {
2902 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2903
2904 crypto_free_aead(aeadctx->sw_cipher);
2905 }
2906
2907 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
2908 unsigned int authsize)
2909 {
2910 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2911
2912 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
2913 aeadctx->mayverify = VERIFY_HW;
2914 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2915 }
2916 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
2917 unsigned int authsize)
2918 {
2919 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2920 u32 maxauth = crypto_aead_maxauthsize(tfm);
2921
2922 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
2923 * true for sha1. authsize == 12 condition should be before
2924 * authsize == (maxauth >> 1)
2925 */
2926 if (authsize == ICV_4) {
2927 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2928 aeadctx->mayverify = VERIFY_HW;
2929 } else if (authsize == ICV_6) {
2930 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2931 aeadctx->mayverify = VERIFY_HW;
2932 } else if (authsize == ICV_10) {
2933 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2934 aeadctx->mayverify = VERIFY_HW;
2935 } else if (authsize == ICV_12) {
2936 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2937 aeadctx->mayverify = VERIFY_HW;
2938 } else if (authsize == ICV_14) {
2939 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2940 aeadctx->mayverify = VERIFY_HW;
2941 } else if (authsize == (maxauth >> 1)) {
2942 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2943 aeadctx->mayverify = VERIFY_HW;
2944 } else if (authsize == maxauth) {
2945 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2946 aeadctx->mayverify = VERIFY_HW;
2947 } else {
2948 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2949 aeadctx->mayverify = VERIFY_SW;
2950 }
2951 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2952 }
2953
2954
2955 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2956 {
2957 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2958
2959 switch (authsize) {
2960 case ICV_4:
2961 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2962 aeadctx->mayverify = VERIFY_HW;
2963 break;
2964 case ICV_8:
2965 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2966 aeadctx->mayverify = VERIFY_HW;
2967 break;
2968 case ICV_12:
2969 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2970 aeadctx->mayverify = VERIFY_HW;
2971 break;
2972 case ICV_14:
2973 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2974 aeadctx->mayverify = VERIFY_HW;
2975 break;
2976 case ICV_16:
2977 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2978 aeadctx->mayverify = VERIFY_HW;
2979 break;
2980 case ICV_13:
2981 case ICV_15:
2982 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2983 aeadctx->mayverify = VERIFY_SW;
2984 break;
2985 default:
2986
2987 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2988 CRYPTO_TFM_RES_BAD_KEY_LEN);
2989 return -EINVAL;
2990 }
2991 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2992 }
2993
2994 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2995 unsigned int authsize)
2996 {
2997 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2998
2999 switch (authsize) {
3000 case ICV_8:
3001 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3002 aeadctx->mayverify = VERIFY_HW;
3003 break;
3004 case ICV_12:
3005 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3006 aeadctx->mayverify = VERIFY_HW;
3007 break;
3008 case ICV_16:
3009 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3010 aeadctx->mayverify = VERIFY_HW;
3011 break;
3012 default:
3013 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
3014 CRYPTO_TFM_RES_BAD_KEY_LEN);
3015 return -EINVAL;
3016 }
3017 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3018 }
3019
3020 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3021 unsigned int authsize)
3022 {
3023 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3024
3025 switch (authsize) {
3026 case ICV_4:
3027 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3028 aeadctx->mayverify = VERIFY_HW;
3029 break;
3030 case ICV_6:
3031 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3032 aeadctx->mayverify = VERIFY_HW;
3033 break;
3034 case ICV_8:
3035 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3036 aeadctx->mayverify = VERIFY_HW;
3037 break;
3038 case ICV_10:
3039 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3040 aeadctx->mayverify = VERIFY_HW;
3041 break;
3042 case ICV_12:
3043 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3044 aeadctx->mayverify = VERIFY_HW;
3045 break;
3046 case ICV_14:
3047 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3048 aeadctx->mayverify = VERIFY_HW;
3049 break;
3050 case ICV_16:
3051 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3052 aeadctx->mayverify = VERIFY_HW;
3053 break;
3054 default:
3055 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
3056 CRYPTO_TFM_RES_BAD_KEY_LEN);
3057 return -EINVAL;
3058 }
3059 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3060 }
3061
3062 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
3063 const u8 *key,
3064 unsigned int keylen)
3065 {
3066 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3067 unsigned char ck_size, mk_size;
3068 int key_ctx_size = 0;
3069
3070 key_ctx_size = sizeof(struct _key_ctx) +
3071 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2;
3072 if (keylen == AES_KEYSIZE_128) {
3073 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3074 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3075 } else if (keylen == AES_KEYSIZE_192) {
3076 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3077 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3078 } else if (keylen == AES_KEYSIZE_256) {
3079 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3080 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3081 } else {
3082 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3083 CRYPTO_TFM_RES_BAD_KEY_LEN);
3084 aeadctx->enckey_len = 0;
3085 return -EINVAL;
3086 }
3087 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3088 key_ctx_size >> 4);
3089 memcpy(aeadctx->key, key, keylen);
3090 aeadctx->enckey_len = keylen;
3091
3092 return 0;
3093 }
3094
3095 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3096 const u8 *key,
3097 unsigned int keylen)
3098 {
3099 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3100 int error;
3101
3102 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3103 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3104 CRYPTO_TFM_REQ_MASK);
3105 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3106 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3107 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3108 CRYPTO_TFM_RES_MASK);
3109 if (error)
3110 return error;
3111 return chcr_ccm_common_setkey(aead, key, keylen);
3112 }
3113
3114 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3115 unsigned int keylen)
3116 {
3117 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3118 int error;
3119
3120 if (keylen < 3) {
3121 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3122 CRYPTO_TFM_RES_BAD_KEY_LEN);
3123 aeadctx->enckey_len = 0;
3124 return -EINVAL;
3125 }
3126 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3127 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3128 CRYPTO_TFM_REQ_MASK);
3129 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3130 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3131 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3132 CRYPTO_TFM_RES_MASK);
3133 if (error)
3134 return error;
3135 keylen -= 3;
3136 memcpy(aeadctx->salt, key + keylen, 3);
3137 return chcr_ccm_common_setkey(aead, key, keylen);
3138 }
3139
3140 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3141 unsigned int keylen)
3142 {
3143 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3144 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
3145 struct crypto_cipher *cipher;
3146 unsigned int ck_size;
3147 int ret = 0, key_ctx_size = 0;
3148
3149 aeadctx->enckey_len = 0;
3150 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3151 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3152 & CRYPTO_TFM_REQ_MASK);
3153 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3154 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
3155 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
3156 CRYPTO_TFM_RES_MASK);
3157 if (ret)
3158 goto out;
3159
3160 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3161 keylen > 3) {
3162 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
3163 memcpy(aeadctx->salt, key + keylen, 4);
3164 }
3165 if (keylen == AES_KEYSIZE_128) {
3166 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3167 } else if (keylen == AES_KEYSIZE_192) {
3168 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3169 } else if (keylen == AES_KEYSIZE_256) {
3170 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3171 } else {
3172 crypto_tfm_set_flags((struct crypto_tfm *)aead,
3173 CRYPTO_TFM_RES_BAD_KEY_LEN);
3174 pr_err("GCM: Invalid key length %d\n", keylen);
3175 ret = -EINVAL;
3176 goto out;
3177 }
3178
3179 memcpy(aeadctx->key, key, keylen);
3180 aeadctx->enckey_len = keylen;
3181 key_ctx_size = sizeof(struct _key_ctx) +
3182 ((DIV_ROUND_UP(keylen, 16)) << 4) +
3183 AEAD_H_SIZE;
3184 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
3185 CHCR_KEYCTX_MAC_KEY_SIZE_128,
3186 0, 0,
3187 key_ctx_size >> 4);
3188 /* Calculate the H = CIPH(K, 0 repeated 16 times).
3189 * It will go in key context
3190 */
3191 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
3192 if (IS_ERR(cipher)) {
3193 aeadctx->enckey_len = 0;
3194 ret = -ENOMEM;
3195 goto out;
3196 }
3197
3198 ret = crypto_cipher_setkey(cipher, key, keylen);
3199 if (ret) {
3200 aeadctx->enckey_len = 0;
3201 goto out1;
3202 }
3203 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
3204 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
3205
3206 out1:
3207 crypto_free_cipher(cipher);
3208 out:
3209 return ret;
3210 }
3211
3212 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3213 unsigned int keylen)
3214 {
3215 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3216 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3217 /* it contains auth and cipher key both*/
3218 struct crypto_authenc_keys keys;
3219 unsigned int bs, subtype;
3220 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3221 int err = 0, i, key_ctx_len = 0;
3222 unsigned char ck_size = 0;
3223 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
3224 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
3225 struct algo_param param;
3226 int align;
3227 u8 *o_ptr = NULL;
3228
3229 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3230 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3231 & CRYPTO_TFM_REQ_MASK);
3232 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3233 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3234 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3235 & CRYPTO_TFM_RES_MASK);
3236 if (err)
3237 goto out;
3238
3239 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3240 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3241 goto out;
3242 }
3243
3244 if (get_alg_config(&param, max_authsize)) {
3245 pr_err("chcr : Unsupported digest size\n");
3246 goto out;
3247 }
3248 subtype = get_aead_subtype(authenc);
3249 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3250 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3251 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3252 goto out;
3253 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3254 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3255 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3256 }
3257 if (keys.enckeylen == AES_KEYSIZE_128) {
3258 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3259 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3260 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3261 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3262 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3263 } else {
3264 pr_err("chcr : Unsupported cipher key\n");
3265 goto out;
3266 }
3267
3268 /* Copy only encryption key. We use authkey to generate h(ipad) and
3269 * h(opad) so authkey is not needed again. authkeylen size have the
3270 * size of the hash digest size.
3271 */
3272 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3273 aeadctx->enckey_len = keys.enckeylen;
3274 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3275 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3276
3277 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3278 aeadctx->enckey_len << 3);
3279 }
3280 base_hash = chcr_alloc_shash(max_authsize);
3281 if (IS_ERR(base_hash)) {
3282 pr_err("chcr : Base driver cannot be loaded\n");
3283 aeadctx->enckey_len = 0;
3284 return -EINVAL;
3285 }
3286 {
3287 SHASH_DESC_ON_STACK(shash, base_hash);
3288 shash->tfm = base_hash;
3289 shash->flags = crypto_shash_get_flags(base_hash);
3290 bs = crypto_shash_blocksize(base_hash);
3291 align = KEYCTX_ALIGN_PAD(max_authsize);
3292 o_ptr = actx->h_iopad + param.result_size + align;
3293
3294 if (keys.authkeylen > bs) {
3295 err = crypto_shash_digest(shash, keys.authkey,
3296 keys.authkeylen,
3297 o_ptr);
3298 if (err) {
3299 pr_err("chcr : Base driver cannot be loaded\n");
3300 goto out;
3301 }
3302 keys.authkeylen = max_authsize;
3303 } else
3304 memcpy(o_ptr, keys.authkey, keys.authkeylen);
3305
3306 /* Compute the ipad-digest*/
3307 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3308 memcpy(pad, o_ptr, keys.authkeylen);
3309 for (i = 0; i < bs >> 2; i++)
3310 *((unsigned int *)pad + i) ^= IPAD_DATA;
3311
3312 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3313 max_authsize))
3314 goto out;
3315 /* Compute the opad-digest */
3316 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3317 memcpy(pad, o_ptr, keys.authkeylen);
3318 for (i = 0; i < bs >> 2; i++)
3319 *((unsigned int *)pad + i) ^= OPAD_DATA;
3320
3321 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3322 goto out;
3323
3324 /* convert the ipad and opad digest to network order */
3325 chcr_change_order(actx->h_iopad, param.result_size);
3326 chcr_change_order(o_ptr, param.result_size);
3327 key_ctx_len = sizeof(struct _key_ctx) +
3328 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
3329 (param.result_size + align) * 2;
3330 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3331 0, 1, key_ctx_len >> 4);
3332 actx->auth_mode = param.auth_mode;
3333 chcr_free_shash(base_hash);
3334
3335 return 0;
3336 }
3337 out:
3338 aeadctx->enckey_len = 0;
3339 if (!IS_ERR(base_hash))
3340 chcr_free_shash(base_hash);
3341 return -EINVAL;
3342 }
3343
3344 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3345 const u8 *key, unsigned int keylen)
3346 {
3347 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3348 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3349 struct crypto_authenc_keys keys;
3350 int err;
3351 /* it contains auth and cipher key both*/
3352 unsigned int subtype;
3353 int key_ctx_len = 0;
3354 unsigned char ck_size = 0;
3355
3356 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3357 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3358 & CRYPTO_TFM_REQ_MASK);
3359 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3360 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3361 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3362 & CRYPTO_TFM_RES_MASK);
3363 if (err)
3364 goto out;
3365
3366 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3367 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3368 goto out;
3369 }
3370 subtype = get_aead_subtype(authenc);
3371 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3372 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3373 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3374 goto out;
3375 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3376 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3377 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3378 }
3379 if (keys.enckeylen == AES_KEYSIZE_128) {
3380 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3381 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3382 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3383 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3384 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3385 } else {
3386 pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen);
3387 goto out;
3388 }
3389 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3390 aeadctx->enckey_len = keys.enckeylen;
3391 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3392 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3393 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3394 aeadctx->enckey_len << 3);
3395 }
3396 key_ctx_len = sizeof(struct _key_ctx)
3397 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
3398
3399 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3400 0, key_ctx_len >> 4);
3401 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3402 return 0;
3403 out:
3404 aeadctx->enckey_len = 0;
3405 return -EINVAL;
3406 }
3407
3408 static int chcr_aead_op(struct aead_request *req,
3409 unsigned short op_type,
3410 int size,
3411 create_wr_t create_wr_fn)
3412 {
3413 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3414 struct uld_ctx *u_ctx;
3415 struct sk_buff *skb;
3416
3417 if (!a_ctx(tfm)->dev) {
3418 pr_err("chcr : %s : No crypto device.\n", __func__);
3419 return -ENXIO;
3420 }
3421 u_ctx = ULD_CTX(a_ctx(tfm));
3422 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3423 a_ctx(tfm)->tx_qidx)) {
3424 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3425 return -EBUSY;
3426 }
3427
3428 /* Form a WR from req */
3429 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[a_ctx(tfm)->rx_qidx], size,
3430 op_type);
3431
3432 if (IS_ERR(skb) || !skb)
3433 return PTR_ERR(skb);
3434
3435 skb->dev = u_ctx->lldi.ports[0];
3436 set_wr_txq(skb, CPL_PRIORITY_DATA, a_ctx(tfm)->tx_qidx);
3437 chcr_send_wr(skb);
3438 return -EINPROGRESS;
3439 }
3440
3441 static int chcr_aead_encrypt(struct aead_request *req)
3442 {
3443 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3444 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3445
3446 reqctx->verify = VERIFY_HW;
3447
3448 switch (get_aead_subtype(tfm)) {
3449 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3450 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3451 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3452 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3453 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3454 create_authenc_wr);
3455 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3456 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3457 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3458 create_aead_ccm_wr);
3459 default:
3460 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3461 create_gcm_wr);
3462 }
3463 }
3464
3465 static int chcr_aead_decrypt(struct aead_request *req)
3466 {
3467 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3468 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3469 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3470 int size;
3471
3472 if (aeadctx->mayverify == VERIFY_SW) {
3473 size = crypto_aead_maxauthsize(tfm);
3474 reqctx->verify = VERIFY_SW;
3475 } else {
3476 size = 0;
3477 reqctx->verify = VERIFY_HW;
3478 }
3479
3480 switch (get_aead_subtype(tfm)) {
3481 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3482 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3483 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3484 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3485 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3486 create_authenc_wr);
3487 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3488 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3489 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3490 create_aead_ccm_wr);
3491 default:
3492 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3493 create_gcm_wr);
3494 }
3495 }
3496
3497 static struct chcr_alg_template driver_algs[] = {
3498 /* AES-CBC */
3499 {
3500 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3501 .is_registered = 0,
3502 .alg.crypto = {
3503 .cra_name = "cbc(aes)",
3504 .cra_driver_name = "cbc-aes-chcr",
3505 .cra_blocksize = AES_BLOCK_SIZE,
3506 .cra_init = chcr_cra_init,
3507 .cra_exit = chcr_cra_exit,
3508 .cra_u.ablkcipher = {
3509 .min_keysize = AES_MIN_KEY_SIZE,
3510 .max_keysize = AES_MAX_KEY_SIZE,
3511 .ivsize = AES_BLOCK_SIZE,
3512 .setkey = chcr_aes_cbc_setkey,
3513 .encrypt = chcr_aes_encrypt,
3514 .decrypt = chcr_aes_decrypt,
3515 }
3516 }
3517 },
3518 {
3519 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3520 .is_registered = 0,
3521 .alg.crypto = {
3522 .cra_name = "xts(aes)",
3523 .cra_driver_name = "xts-aes-chcr",
3524 .cra_blocksize = AES_BLOCK_SIZE,
3525 .cra_init = chcr_cra_init,
3526 .cra_exit = NULL,
3527 .cra_u .ablkcipher = {
3528 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3529 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3530 .ivsize = AES_BLOCK_SIZE,
3531 .setkey = chcr_aes_xts_setkey,
3532 .encrypt = chcr_aes_encrypt,
3533 .decrypt = chcr_aes_decrypt,
3534 }
3535 }
3536 },
3537 {
3538 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3539 .is_registered = 0,
3540 .alg.crypto = {
3541 .cra_name = "ctr(aes)",
3542 .cra_driver_name = "ctr-aes-chcr",
3543 .cra_blocksize = 1,
3544 .cra_init = chcr_cra_init,
3545 .cra_exit = chcr_cra_exit,
3546 .cra_u.ablkcipher = {
3547 .min_keysize = AES_MIN_KEY_SIZE,
3548 .max_keysize = AES_MAX_KEY_SIZE,
3549 .ivsize = AES_BLOCK_SIZE,
3550 .setkey = chcr_aes_ctr_setkey,
3551 .encrypt = chcr_aes_encrypt,
3552 .decrypt = chcr_aes_decrypt,
3553 }
3554 }
3555 },
3556 {
3557 .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
3558 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3559 .is_registered = 0,
3560 .alg.crypto = {
3561 .cra_name = "rfc3686(ctr(aes))",
3562 .cra_driver_name = "rfc3686-ctr-aes-chcr",
3563 .cra_blocksize = 1,
3564 .cra_init = chcr_rfc3686_init,
3565 .cra_exit = chcr_cra_exit,
3566 .cra_u.ablkcipher = {
3567 .min_keysize = AES_MIN_KEY_SIZE +
3568 CTR_RFC3686_NONCE_SIZE,
3569 .max_keysize = AES_MAX_KEY_SIZE +
3570 CTR_RFC3686_NONCE_SIZE,
3571 .ivsize = CTR_RFC3686_IV_SIZE,
3572 .setkey = chcr_aes_rfc3686_setkey,
3573 .encrypt = chcr_aes_encrypt,
3574 .decrypt = chcr_aes_decrypt,
3575 .geniv = "seqiv",
3576 }
3577 }
3578 },
3579 /* SHA */
3580 {
3581 .type = CRYPTO_ALG_TYPE_AHASH,
3582 .is_registered = 0,
3583 .alg.hash = {
3584 .halg.digestsize = SHA1_DIGEST_SIZE,
3585 .halg.base = {
3586 .cra_name = "sha1",
3587 .cra_driver_name = "sha1-chcr",
3588 .cra_blocksize = SHA1_BLOCK_SIZE,
3589 }
3590 }
3591 },
3592 {
3593 .type = CRYPTO_ALG_TYPE_AHASH,
3594 .is_registered = 0,
3595 .alg.hash = {
3596 .halg.digestsize = SHA256_DIGEST_SIZE,
3597 .halg.base = {
3598 .cra_name = "sha256",
3599 .cra_driver_name = "sha256-chcr",
3600 .cra_blocksize = SHA256_BLOCK_SIZE,
3601 }
3602 }
3603 },
3604 {
3605 .type = CRYPTO_ALG_TYPE_AHASH,
3606 .is_registered = 0,
3607 .alg.hash = {
3608 .halg.digestsize = SHA224_DIGEST_SIZE,
3609 .halg.base = {
3610 .cra_name = "sha224",
3611 .cra_driver_name = "sha224-chcr",
3612 .cra_blocksize = SHA224_BLOCK_SIZE,
3613 }
3614 }
3615 },
3616 {
3617 .type = CRYPTO_ALG_TYPE_AHASH,
3618 .is_registered = 0,
3619 .alg.hash = {
3620 .halg.digestsize = SHA384_DIGEST_SIZE,
3621 .halg.base = {
3622 .cra_name = "sha384",
3623 .cra_driver_name = "sha384-chcr",
3624 .cra_blocksize = SHA384_BLOCK_SIZE,
3625 }
3626 }
3627 },
3628 {
3629 .type = CRYPTO_ALG_TYPE_AHASH,
3630 .is_registered = 0,
3631 .alg.hash = {
3632 .halg.digestsize = SHA512_DIGEST_SIZE,
3633 .halg.base = {
3634 .cra_name = "sha512",
3635 .cra_driver_name = "sha512-chcr",
3636 .cra_blocksize = SHA512_BLOCK_SIZE,
3637 }
3638 }
3639 },
3640 /* HMAC */
3641 {
3642 .type = CRYPTO_ALG_TYPE_HMAC,
3643 .is_registered = 0,
3644 .alg.hash = {
3645 .halg.digestsize = SHA1_DIGEST_SIZE,
3646 .halg.base = {
3647 .cra_name = "hmac(sha1)",
3648 .cra_driver_name = "hmac-sha1-chcr",
3649 .cra_blocksize = SHA1_BLOCK_SIZE,
3650 }
3651 }
3652 },
3653 {
3654 .type = CRYPTO_ALG_TYPE_HMAC,
3655 .is_registered = 0,
3656 .alg.hash = {
3657 .halg.digestsize = SHA224_DIGEST_SIZE,
3658 .halg.base = {
3659 .cra_name = "hmac(sha224)",
3660 .cra_driver_name = "hmac-sha224-chcr",
3661 .cra_blocksize = SHA224_BLOCK_SIZE,
3662 }
3663 }
3664 },
3665 {
3666 .type = CRYPTO_ALG_TYPE_HMAC,
3667 .is_registered = 0,
3668 .alg.hash = {
3669 .halg.digestsize = SHA256_DIGEST_SIZE,
3670 .halg.base = {
3671 .cra_name = "hmac(sha256)",
3672 .cra_driver_name = "hmac-sha256-chcr",
3673 .cra_blocksize = SHA256_BLOCK_SIZE,
3674 }
3675 }
3676 },
3677 {
3678 .type = CRYPTO_ALG_TYPE_HMAC,
3679 .is_registered = 0,
3680 .alg.hash = {
3681 .halg.digestsize = SHA384_DIGEST_SIZE,
3682 .halg.base = {
3683 .cra_name = "hmac(sha384)",
3684 .cra_driver_name = "hmac-sha384-chcr",
3685 .cra_blocksize = SHA384_BLOCK_SIZE,
3686 }
3687 }
3688 },
3689 {
3690 .type = CRYPTO_ALG_TYPE_HMAC,
3691 .is_registered = 0,
3692 .alg.hash = {
3693 .halg.digestsize = SHA512_DIGEST_SIZE,
3694 .halg.base = {
3695 .cra_name = "hmac(sha512)",
3696 .cra_driver_name = "hmac-sha512-chcr",
3697 .cra_blocksize = SHA512_BLOCK_SIZE,
3698 }
3699 }
3700 },
3701 /* Add AEAD Algorithms */
3702 {
3703 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3704 .is_registered = 0,
3705 .alg.aead = {
3706 .base = {
3707 .cra_name = "gcm(aes)",
3708 .cra_driver_name = "gcm-aes-chcr",
3709 .cra_blocksize = 1,
3710 .cra_priority = CHCR_AEAD_PRIORITY,
3711 .cra_ctxsize = sizeof(struct chcr_context) +
3712 sizeof(struct chcr_aead_ctx) +
3713 sizeof(struct chcr_gcm_ctx),
3714 },
3715 .ivsize = GCM_AES_IV_SIZE,
3716 .maxauthsize = GHASH_DIGEST_SIZE,
3717 .setkey = chcr_gcm_setkey,
3718 .setauthsize = chcr_gcm_setauthsize,
3719 }
3720 },
3721 {
3722 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3723 .is_registered = 0,
3724 .alg.aead = {
3725 .base = {
3726 .cra_name = "rfc4106(gcm(aes))",
3727 .cra_driver_name = "rfc4106-gcm-aes-chcr",
3728 .cra_blocksize = 1,
3729 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3730 .cra_ctxsize = sizeof(struct chcr_context) +
3731 sizeof(struct chcr_aead_ctx) +
3732 sizeof(struct chcr_gcm_ctx),
3733
3734 },
3735 .ivsize = GCM_RFC4106_IV_SIZE,
3736 .maxauthsize = GHASH_DIGEST_SIZE,
3737 .setkey = chcr_gcm_setkey,
3738 .setauthsize = chcr_4106_4309_setauthsize,
3739 }
3740 },
3741 {
3742 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3743 .is_registered = 0,
3744 .alg.aead = {
3745 .base = {
3746 .cra_name = "ccm(aes)",
3747 .cra_driver_name = "ccm-aes-chcr",
3748 .cra_blocksize = 1,
3749 .cra_priority = CHCR_AEAD_PRIORITY,
3750 .cra_ctxsize = sizeof(struct chcr_context) +
3751 sizeof(struct chcr_aead_ctx),
3752
3753 },
3754 .ivsize = AES_BLOCK_SIZE,
3755 .maxauthsize = GHASH_DIGEST_SIZE,
3756 .setkey = chcr_aead_ccm_setkey,
3757 .setauthsize = chcr_ccm_setauthsize,
3758 }
3759 },
3760 {
3761 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3762 .is_registered = 0,
3763 .alg.aead = {
3764 .base = {
3765 .cra_name = "rfc4309(ccm(aes))",
3766 .cra_driver_name = "rfc4309-ccm-aes-chcr",
3767 .cra_blocksize = 1,
3768 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3769 .cra_ctxsize = sizeof(struct chcr_context) +
3770 sizeof(struct chcr_aead_ctx),
3771
3772 },
3773 .ivsize = 8,
3774 .maxauthsize = GHASH_DIGEST_SIZE,
3775 .setkey = chcr_aead_rfc4309_setkey,
3776 .setauthsize = chcr_4106_4309_setauthsize,
3777 }
3778 },
3779 {
3780 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3781 .is_registered = 0,
3782 .alg.aead = {
3783 .base = {
3784 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3785 .cra_driver_name =
3786 "authenc-hmac-sha1-cbc-aes-chcr",
3787 .cra_blocksize = AES_BLOCK_SIZE,
3788 .cra_priority = CHCR_AEAD_PRIORITY,
3789 .cra_ctxsize = sizeof(struct chcr_context) +
3790 sizeof(struct chcr_aead_ctx) +
3791 sizeof(struct chcr_authenc_ctx),
3792
3793 },
3794 .ivsize = AES_BLOCK_SIZE,
3795 .maxauthsize = SHA1_DIGEST_SIZE,
3796 .setkey = chcr_authenc_setkey,
3797 .setauthsize = chcr_authenc_setauthsize,
3798 }
3799 },
3800 {
3801 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3802 .is_registered = 0,
3803 .alg.aead = {
3804 .base = {
3805
3806 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3807 .cra_driver_name =
3808 "authenc-hmac-sha256-cbc-aes-chcr",
3809 .cra_blocksize = AES_BLOCK_SIZE,
3810 .cra_priority = CHCR_AEAD_PRIORITY,
3811 .cra_ctxsize = sizeof(struct chcr_context) +
3812 sizeof(struct chcr_aead_ctx) +
3813 sizeof(struct chcr_authenc_ctx),
3814
3815 },
3816 .ivsize = AES_BLOCK_SIZE,
3817 .maxauthsize = SHA256_DIGEST_SIZE,
3818 .setkey = chcr_authenc_setkey,
3819 .setauthsize = chcr_authenc_setauthsize,
3820 }
3821 },
3822 {
3823 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3824 .is_registered = 0,
3825 .alg.aead = {
3826 .base = {
3827 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3828 .cra_driver_name =
3829 "authenc-hmac-sha224-cbc-aes-chcr",
3830 .cra_blocksize = AES_BLOCK_SIZE,
3831 .cra_priority = CHCR_AEAD_PRIORITY,
3832 .cra_ctxsize = sizeof(struct chcr_context) +
3833 sizeof(struct chcr_aead_ctx) +
3834 sizeof(struct chcr_authenc_ctx),
3835 },
3836 .ivsize = AES_BLOCK_SIZE,
3837 .maxauthsize = SHA224_DIGEST_SIZE,
3838 .setkey = chcr_authenc_setkey,
3839 .setauthsize = chcr_authenc_setauthsize,
3840 }
3841 },
3842 {
3843 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3844 .is_registered = 0,
3845 .alg.aead = {
3846 .base = {
3847 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3848 .cra_driver_name =
3849 "authenc-hmac-sha384-cbc-aes-chcr",
3850 .cra_blocksize = AES_BLOCK_SIZE,
3851 .cra_priority = CHCR_AEAD_PRIORITY,
3852 .cra_ctxsize = sizeof(struct chcr_context) +
3853 sizeof(struct chcr_aead_ctx) +
3854 sizeof(struct chcr_authenc_ctx),
3855
3856 },
3857 .ivsize = AES_BLOCK_SIZE,
3858 .maxauthsize = SHA384_DIGEST_SIZE,
3859 .setkey = chcr_authenc_setkey,
3860 .setauthsize = chcr_authenc_setauthsize,
3861 }
3862 },
3863 {
3864 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
3865 .is_registered = 0,
3866 .alg.aead = {
3867 .base = {
3868 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3869 .cra_driver_name =
3870 "authenc-hmac-sha512-cbc-aes-chcr",
3871 .cra_blocksize = AES_BLOCK_SIZE,
3872 .cra_priority = CHCR_AEAD_PRIORITY,
3873 .cra_ctxsize = sizeof(struct chcr_context) +
3874 sizeof(struct chcr_aead_ctx) +
3875 sizeof(struct chcr_authenc_ctx),
3876
3877 },
3878 .ivsize = AES_BLOCK_SIZE,
3879 .maxauthsize = SHA512_DIGEST_SIZE,
3880 .setkey = chcr_authenc_setkey,
3881 .setauthsize = chcr_authenc_setauthsize,
3882 }
3883 },
3884 {
3885 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
3886 .is_registered = 0,
3887 .alg.aead = {
3888 .base = {
3889 .cra_name = "authenc(digest_null,cbc(aes))",
3890 .cra_driver_name =
3891 "authenc-digest_null-cbc-aes-chcr",
3892 .cra_blocksize = AES_BLOCK_SIZE,
3893 .cra_priority = CHCR_AEAD_PRIORITY,
3894 .cra_ctxsize = sizeof(struct chcr_context) +
3895 sizeof(struct chcr_aead_ctx) +
3896 sizeof(struct chcr_authenc_ctx),
3897
3898 },
3899 .ivsize = AES_BLOCK_SIZE,
3900 .maxauthsize = 0,
3901 .setkey = chcr_aead_digest_null_setkey,
3902 .setauthsize = chcr_authenc_null_setauthsize,
3903 }
3904 },
3905 {
3906 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
3907 .is_registered = 0,
3908 .alg.aead = {
3909 .base = {
3910 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
3911 .cra_driver_name =
3912 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
3913 .cra_blocksize = 1,
3914 .cra_priority = CHCR_AEAD_PRIORITY,
3915 .cra_ctxsize = sizeof(struct chcr_context) +
3916 sizeof(struct chcr_aead_ctx) +
3917 sizeof(struct chcr_authenc_ctx),
3918
3919 },
3920 .ivsize = CTR_RFC3686_IV_SIZE,
3921 .maxauthsize = SHA1_DIGEST_SIZE,
3922 .setkey = chcr_authenc_setkey,
3923 .setauthsize = chcr_authenc_setauthsize,
3924 }
3925 },
3926 {
3927 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
3928 .is_registered = 0,
3929 .alg.aead = {
3930 .base = {
3931
3932 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
3933 .cra_driver_name =
3934 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
3935 .cra_blocksize = 1,
3936 .cra_priority = CHCR_AEAD_PRIORITY,
3937 .cra_ctxsize = sizeof(struct chcr_context) +
3938 sizeof(struct chcr_aead_ctx) +
3939 sizeof(struct chcr_authenc_ctx),
3940
3941 },
3942 .ivsize = CTR_RFC3686_IV_SIZE,
3943 .maxauthsize = SHA256_DIGEST_SIZE,
3944 .setkey = chcr_authenc_setkey,
3945 .setauthsize = chcr_authenc_setauthsize,
3946 }
3947 },
3948 {
3949 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
3950 .is_registered = 0,
3951 .alg.aead = {
3952 .base = {
3953 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
3954 .cra_driver_name =
3955 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
3956 .cra_blocksize = 1,
3957 .cra_priority = CHCR_AEAD_PRIORITY,
3958 .cra_ctxsize = sizeof(struct chcr_context) +
3959 sizeof(struct chcr_aead_ctx) +
3960 sizeof(struct chcr_authenc_ctx),
3961 },
3962 .ivsize = CTR_RFC3686_IV_SIZE,
3963 .maxauthsize = SHA224_DIGEST_SIZE,
3964 .setkey = chcr_authenc_setkey,
3965 .setauthsize = chcr_authenc_setauthsize,
3966 }
3967 },
3968 {
3969 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
3970 .is_registered = 0,
3971 .alg.aead = {
3972 .base = {
3973 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
3974 .cra_driver_name =
3975 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
3976 .cra_blocksize = 1,
3977 .cra_priority = CHCR_AEAD_PRIORITY,
3978 .cra_ctxsize = sizeof(struct chcr_context) +
3979 sizeof(struct chcr_aead_ctx) +
3980 sizeof(struct chcr_authenc_ctx),
3981
3982 },
3983 .ivsize = CTR_RFC3686_IV_SIZE,
3984 .maxauthsize = SHA384_DIGEST_SIZE,
3985 .setkey = chcr_authenc_setkey,
3986 .setauthsize = chcr_authenc_setauthsize,
3987 }
3988 },
3989 {
3990 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
3991 .is_registered = 0,
3992 .alg.aead = {
3993 .base = {
3994 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
3995 .cra_driver_name =
3996 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
3997 .cra_blocksize = 1,
3998 .cra_priority = CHCR_AEAD_PRIORITY,
3999 .cra_ctxsize = sizeof(struct chcr_context) +
4000 sizeof(struct chcr_aead_ctx) +
4001 sizeof(struct chcr_authenc_ctx),
4002
4003 },
4004 .ivsize = CTR_RFC3686_IV_SIZE,
4005 .maxauthsize = SHA512_DIGEST_SIZE,
4006 .setkey = chcr_authenc_setkey,
4007 .setauthsize = chcr_authenc_setauthsize,
4008 }
4009 },
4010 {
4011 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4012 .is_registered = 0,
4013 .alg.aead = {
4014 .base = {
4015 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4016 .cra_driver_name =
4017 "authenc-digest_null-rfc3686-ctr-aes-chcr",
4018 .cra_blocksize = 1,
4019 .cra_priority = CHCR_AEAD_PRIORITY,
4020 .cra_ctxsize = sizeof(struct chcr_context) +
4021 sizeof(struct chcr_aead_ctx) +
4022 sizeof(struct chcr_authenc_ctx),
4023
4024 },
4025 .ivsize = CTR_RFC3686_IV_SIZE,
4026 .maxauthsize = 0,
4027 .setkey = chcr_aead_digest_null_setkey,
4028 .setauthsize = chcr_authenc_null_setauthsize,
4029 }
4030 },
4031
4032 };
4033
4034 /*
4035 * chcr_unregister_alg - Deregister crypto algorithms with
4036 * kernel framework.
4037 */
4038 static int chcr_unregister_alg(void)
4039 {
4040 int i;
4041
4042 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4043 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4044 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4045 if (driver_algs[i].is_registered)
4046 crypto_unregister_alg(
4047 &driver_algs[i].alg.crypto);
4048 break;
4049 case CRYPTO_ALG_TYPE_AEAD:
4050 if (driver_algs[i].is_registered)
4051 crypto_unregister_aead(
4052 &driver_algs[i].alg.aead);
4053 break;
4054 case CRYPTO_ALG_TYPE_AHASH:
4055 if (driver_algs[i].is_registered)
4056 crypto_unregister_ahash(
4057 &driver_algs[i].alg.hash);
4058 break;
4059 }
4060 driver_algs[i].is_registered = 0;
4061 }
4062 return 0;
4063 }
4064
4065 #define SZ_AHASH_CTX sizeof(struct chcr_context)
4066 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4067 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
4068 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
4069
4070 /*
4071 * chcr_register_alg - Register crypto algorithms with kernel framework.
4072 */
4073 static int chcr_register_alg(void)
4074 {
4075 struct crypto_alg ai;
4076 struct ahash_alg *a_hash;
4077 int err = 0, i;
4078 char *name = NULL;
4079
4080 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4081 if (driver_algs[i].is_registered)
4082 continue;
4083 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4084 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4085 driver_algs[i].alg.crypto.cra_priority =
4086 CHCR_CRA_PRIORITY;
4087 driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
4088 driver_algs[i].alg.crypto.cra_flags =
4089 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
4090 CRYPTO_ALG_NEED_FALLBACK;
4091 driver_algs[i].alg.crypto.cra_ctxsize =
4092 sizeof(struct chcr_context) +
4093 sizeof(struct ablk_ctx);
4094 driver_algs[i].alg.crypto.cra_alignmask = 0;
4095 driver_algs[i].alg.crypto.cra_type =
4096 &crypto_ablkcipher_type;
4097 err = crypto_register_alg(&driver_algs[i].alg.crypto);
4098 name = driver_algs[i].alg.crypto.cra_driver_name;
4099 break;
4100 case CRYPTO_ALG_TYPE_AEAD:
4101 driver_algs[i].alg.aead.base.cra_flags =
4102 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
4103 CRYPTO_ALG_NEED_FALLBACK;
4104 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4105 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4106 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4107 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4108 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4109 err = crypto_register_aead(&driver_algs[i].alg.aead);
4110 name = driver_algs[i].alg.aead.base.cra_driver_name;
4111 break;
4112 case CRYPTO_ALG_TYPE_AHASH:
4113 a_hash = &driver_algs[i].alg.hash;
4114 a_hash->update = chcr_ahash_update;
4115 a_hash->final = chcr_ahash_final;
4116 a_hash->finup = chcr_ahash_finup;
4117 a_hash->digest = chcr_ahash_digest;
4118 a_hash->export = chcr_ahash_export;
4119 a_hash->import = chcr_ahash_import;
4120 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4121 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4122 a_hash->halg.base.cra_module = THIS_MODULE;
4123 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
4124 a_hash->halg.base.cra_alignmask = 0;
4125 a_hash->halg.base.cra_exit = NULL;
4126 a_hash->halg.base.cra_type = &crypto_ahash_type;
4127
4128 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4129 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4130 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
4131 a_hash->init = chcr_hmac_init;
4132 a_hash->setkey = chcr_ahash_setkey;
4133 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4134 } else {
4135 a_hash->init = chcr_sha_init;
4136 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4137 a_hash->halg.base.cra_init = chcr_sha_cra_init;
4138 }
4139 err = crypto_register_ahash(&driver_algs[i].alg.hash);
4140 ai = driver_algs[i].alg.hash.halg.base;
4141 name = ai.cra_driver_name;
4142 break;
4143 }
4144 if (err) {
4145 pr_err("chcr : %s : Algorithm registration failed\n",
4146 name);
4147 goto register_err;
4148 } else {
4149 driver_algs[i].is_registered = 1;
4150 }
4151 }
4152 return 0;
4153
4154 register_err:
4155 chcr_unregister_alg();
4156 return err;
4157 }
4158
4159 /*
4160 * start_crypto - Register the crypto algorithms.
4161 * This should called once when the first device comesup. After this
4162 * kernel will start calling driver APIs for crypto operations.
4163 */
4164 int start_crypto(void)
4165 {
4166 return chcr_register_alg();
4167 }
4168
4169 /*
4170 * stop_crypto - Deregister all the crypto algorithms with kernel.
4171 * This should be called once when the last device goes down. After this
4172 * kernel will not call the driver API for crypto operations.
4173 */
4174 int stop_crypto(void)
4175 {
4176 chcr_unregister_alg();
4177 return 0;
4178 }
CRIS linux kernel tree