?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'ceph-for-4.13-rc8' of git://github.com/ceph/ceph-client
[linux/fpc-iii.git] / drivers / crypto / chelsio / chcr_algo.c
blob0e81607018331a2fe0823463b05efa8a0357fd81
1 /*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42 #define pr_fmt(fmt) "chcr:" fmt
43
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/cryptohash.h>
48 #include <linux/skbuff.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/highmem.h>
51 #include <linux/scatterlist.h>
52
53 #include <crypto/aes.h>
54 #include <crypto/algapi.h>
55 #include <crypto/hash.h>
56 #include <crypto/sha.h>
57 #include <crypto/authenc.h>
58 #include <crypto/ctr.h>
59 #include <crypto/gf128mul.h>
60 #include <crypto/internal/aead.h>
61 #include <crypto/null.h>
62 #include <crypto/internal/skcipher.h>
63 #include <crypto/aead.h>
64 #include <crypto/scatterwalk.h>
65 #include <crypto/internal/hash.h>
66
67 #include "t4fw_api.h"
68 #include "t4_msg.h"
69 #include "chcr_core.h"
70 #include "chcr_algo.h"
71 #include "chcr_crypto.h"
72
73 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
74 {
75 return ctx->crypto_ctx->aeadctx;
76 }
77
78 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
79 {
80 return ctx->crypto_ctx->ablkctx;
81 }
82
83 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
84 {
85 return ctx->crypto_ctx->hmacctx;
86 }
87
88 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
89 {
90 return gctx->ctx->gcm;
91 }
92
93 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
94 {
95 return gctx->ctx->authenc;
96 }
97
98 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
99 {
100 return ctx->dev->u_ctx;
101 }
102
103 static inline int is_ofld_imm(const struct sk_buff *skb)
104 {
105 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
106 }
107
108 /*
109 * sgl_len - calculates the size of an SGL of the given capacity
110 * @n: the number of SGL entries
111 * Calculates the number of flits needed for a scatter/gather list that
112 * can hold the given number of entries.
113 */
114 static inline unsigned int sgl_len(unsigned int n)
115 {
116 n--;
117 return (3 * n) / 2 + (n & 1) + 2;
118 }
119
120 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
121 {
122 u8 temp[SHA512_DIGEST_SIZE];
123 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
124 int authsize = crypto_aead_authsize(tfm);
125 struct cpl_fw6_pld *fw6_pld;
126 int cmp = 0;
127
128 fw6_pld = (struct cpl_fw6_pld *)input;
129 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
130 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
131 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
132 } else {
133
134 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
135 authsize, req->assoclen +
136 req->cryptlen - authsize);
137 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
138 }
139 if (cmp)
140 *err = -EBADMSG;
141 else
142 *err = 0;
143 }
144
145 /*
146 * chcr_handle_resp - Unmap the DMA buffers associated with the request
147 * @req: crypto request
148 */
149 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
150 int err)
151 {
152 struct crypto_tfm *tfm = req->tfm;
153 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
154 struct uld_ctx *u_ctx = ULD_CTX(ctx);
155 struct chcr_req_ctx ctx_req;
156 unsigned int digestsize, updated_digestsize;
157 struct adapter *adap = padap(ctx->dev);
158
159 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
160 case CRYPTO_ALG_TYPE_AEAD:
161 ctx_req.req.aead_req = aead_request_cast(req);
162 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
163 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
164 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
165 if (ctx_req.ctx.reqctx->skb) {
166 kfree_skb(ctx_req.ctx.reqctx->skb);
167 ctx_req.ctx.reqctx->skb = NULL;
168 }
169 free_new_sg(ctx_req.ctx.reqctx->newdstsg);
170 ctx_req.ctx.reqctx->newdstsg = NULL;
171 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
172 chcr_verify_tag(ctx_req.req.aead_req, input,
173 &err);
174 ctx_req.ctx.reqctx->verify = VERIFY_HW;
175 }
176 ctx_req.req.aead_req->base.complete(req, err);
177 break;
178
179 case CRYPTO_ALG_TYPE_ABLKCIPHER:
180 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
181 input, err);
182 break;
183
184 case CRYPTO_ALG_TYPE_AHASH:
185 ctx_req.req.ahash_req = ahash_request_cast(req);
186 ctx_req.ctx.ahash_ctx =
187 ahash_request_ctx(ctx_req.req.ahash_req);
188 digestsize =
189 crypto_ahash_digestsize(crypto_ahash_reqtfm(
190 ctx_req.req.ahash_req));
191 updated_digestsize = digestsize;
192 if (digestsize == SHA224_DIGEST_SIZE)
193 updated_digestsize = SHA256_DIGEST_SIZE;
194 else if (digestsize == SHA384_DIGEST_SIZE)
195 updated_digestsize = SHA512_DIGEST_SIZE;
196 if (ctx_req.ctx.ahash_ctx->skb) {
197 kfree_skb(ctx_req.ctx.ahash_ctx->skb);
198 ctx_req.ctx.ahash_ctx->skb = NULL;
199 }
200 if (ctx_req.ctx.ahash_ctx->result == 1) {
201 ctx_req.ctx.ahash_ctx->result = 0;
202 memcpy(ctx_req.req.ahash_req->result, input +
203 sizeof(struct cpl_fw6_pld),
204 digestsize);
205 } else {
206 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
207 sizeof(struct cpl_fw6_pld),
208 updated_digestsize);
209 }
210 ctx_req.req.ahash_req->base.complete(req, err);
211 break;
212 }
213 atomic_inc(&adap->chcr_stats.complete);
214 return err;
215 }
216
217 /*
218 * calc_tx_flits_ofld - calculate # of flits for an offload packet
219 * @skb: the packet
220 * Returns the number of flits needed for the given offload packet.
221 * These packets are already fully constructed and no additional headers
222 * will be added.
223 */
224 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
225 {
226 unsigned int flits, cnt;
227
228 if (is_ofld_imm(skb))
229 return DIV_ROUND_UP(skb->len, 8);
230
231 flits = skb_transport_offset(skb) / 8; /* headers */
232 cnt = skb_shinfo(skb)->nr_frags;
233 if (skb_tail_pointer(skb) != skb_transport_header(skb))
234 cnt++;
235 return flits + sgl_len(cnt);
236 }
237
238 static inline void get_aes_decrypt_key(unsigned char *dec_key,
239 const unsigned char *key,
240 unsigned int keylength)
241 {
242 u32 temp;
243 u32 w_ring[MAX_NK];
244 int i, j, k;
245 u8 nr, nk;
246
247 switch (keylength) {
248 case AES_KEYLENGTH_128BIT:
249 nk = KEYLENGTH_4BYTES;
250 nr = NUMBER_OF_ROUNDS_10;
251 break;
252 case AES_KEYLENGTH_192BIT:
253 nk = KEYLENGTH_6BYTES;
254 nr = NUMBER_OF_ROUNDS_12;
255 break;
256 case AES_KEYLENGTH_256BIT:
257 nk = KEYLENGTH_8BYTES;
258 nr = NUMBER_OF_ROUNDS_14;
259 break;
260 default:
261 return;
262 }
263 for (i = 0; i < nk; i++)
264 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
265
266 i = 0;
267 temp = w_ring[nk - 1];
268 while (i + nk < (nr + 1) * 4) {
269 if (!(i % nk)) {
270 /* RotWord(temp) */
271 temp = (temp << 8) | (temp >> 24);
272 temp = aes_ks_subword(temp);
273 temp ^= round_constant[i / nk];
274 } else if (nk == 8 && (i % 4 == 0)) {
275 temp = aes_ks_subword(temp);
276 }
277 w_ring[i % nk] ^= temp;
278 temp = w_ring[i % nk];
279 i++;
280 }
281 i--;
282 for (k = 0, j = i % nk; k < nk; k++) {
283 *((u32 *)dec_key + k) = htonl(w_ring[j]);
284 j--;
285 if (j < 0)
286 j += nk;
287 }
288 }
289
290 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
291 {
292 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
293
294 switch (ds) {
295 case SHA1_DIGEST_SIZE:
296 base_hash = crypto_alloc_shash("sha1", 0, 0);
297 break;
298 case SHA224_DIGEST_SIZE:
299 base_hash = crypto_alloc_shash("sha224", 0, 0);
300 break;
301 case SHA256_DIGEST_SIZE:
302 base_hash = crypto_alloc_shash("sha256", 0, 0);
303 break;
304 case SHA384_DIGEST_SIZE:
305 base_hash = crypto_alloc_shash("sha384", 0, 0);
306 break;
307 case SHA512_DIGEST_SIZE:
308 base_hash = crypto_alloc_shash("sha512", 0, 0);
309 break;
310 }
311
312 return base_hash;
313 }
314
315 static int chcr_compute_partial_hash(struct shash_desc *desc,
316 char *iopad, char *result_hash,
317 int digest_size)
318 {
319 struct sha1_state sha1_st;
320 struct sha256_state sha256_st;
321 struct sha512_state sha512_st;
322 int error;
323
324 if (digest_size == SHA1_DIGEST_SIZE) {
325 error = crypto_shash_init(desc) ?:
326 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
327 crypto_shash_export(desc, (void *)&sha1_st);
328 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
329 } else if (digest_size == SHA224_DIGEST_SIZE) {
330 error = crypto_shash_init(desc) ?:
331 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
332 crypto_shash_export(desc, (void *)&sha256_st);
333 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
334
335 } else if (digest_size == SHA256_DIGEST_SIZE) {
336 error = crypto_shash_init(desc) ?:
337 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
338 crypto_shash_export(desc, (void *)&sha256_st);
339 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
340
341 } else if (digest_size == SHA384_DIGEST_SIZE) {
342 error = crypto_shash_init(desc) ?:
343 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
344 crypto_shash_export(desc, (void *)&sha512_st);
345 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
346
347 } else if (digest_size == SHA512_DIGEST_SIZE) {
348 error = crypto_shash_init(desc) ?:
349 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
350 crypto_shash_export(desc, (void *)&sha512_st);
351 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
352 } else {
353 error = -EINVAL;
354 pr_err("Unknown digest size %d\n", digest_size);
355 }
356 return error;
357 }
358
359 static void chcr_change_order(char *buf, int ds)
360 {
361 int i;
362
363 if (ds == SHA512_DIGEST_SIZE) {
364 for (i = 0; i < (ds / sizeof(u64)); i++)
365 *((__be64 *)buf + i) =
366 cpu_to_be64(*((u64 *)buf + i));
367 } else {
368 for (i = 0; i < (ds / sizeof(u32)); i++)
369 *((__be32 *)buf + i) =
370 cpu_to_be32(*((u32 *)buf + i));
371 }
372 }
373
374 static inline int is_hmac(struct crypto_tfm *tfm)
375 {
376 struct crypto_alg *alg = tfm->__crt_alg;
377 struct chcr_alg_template *chcr_crypto_alg =
378 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
379 alg.hash);
380 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
381 return 1;
382 return 0;
383 }
384
385 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
386 struct scatterlist *sg,
387 struct phys_sge_parm *sg_param)
388 {
389 struct phys_sge_pairs *to;
390 unsigned int len = 0, left_size = sg_param->obsize;
391 unsigned int nents = sg_param->nents, i, j = 0;
392
393 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
394 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
395 phys_cpl->pcirlxorder_to_noofsgentr =
396 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
397 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
398 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
399 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
400 CPL_RX_PHYS_DSGL_DCAID_V(0) |
401 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
402 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
403 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
404 phys_cpl->rss_hdr_int.hash_val = 0;
405 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
406 sizeof(struct cpl_rx_phys_dsgl));
407 for (i = 0; nents && left_size; to++) {
408 for (j = 0; j < 8 && nents && left_size; j++, nents--) {
409 len = min(left_size, sg_dma_len(sg));
410 to->len[j] = htons(len);
411 to->addr[j] = cpu_to_be64(sg_dma_address(sg));
412 left_size -= len;
413 sg = sg_next(sg);
414 }
415 }
416 }
417
418 static inline int map_writesg_phys_cpl(struct device *dev,
419 struct cpl_rx_phys_dsgl *phys_cpl,
420 struct scatterlist *sg,
421 struct phys_sge_parm *sg_param)
422 {
423 if (!sg || !sg_param->nents)
424 return -EINVAL;
425
426 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
427 if (sg_param->nents == 0) {
428 pr_err("CHCR : DMA mapping failed\n");
429 return -EINVAL;
430 }
431 write_phys_cpl(phys_cpl, sg, sg_param);
432 return 0;
433 }
434
435 static inline int get_aead_subtype(struct crypto_aead *aead)
436 {
437 struct aead_alg *alg = crypto_aead_alg(aead);
438 struct chcr_alg_template *chcr_crypto_alg =
439 container_of(alg, struct chcr_alg_template, alg.aead);
440 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
441 }
442
443 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
444 {
445 struct crypto_alg *alg = tfm->__crt_alg;
446 struct chcr_alg_template *chcr_crypto_alg =
447 container_of(alg, struct chcr_alg_template, alg.crypto);
448
449 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
450 }
451
452 static inline void write_buffer_to_skb(struct sk_buff *skb,
453 unsigned int *frags,
454 char *bfr,
455 u8 bfr_len)
456 {
457 skb->len += bfr_len;
458 skb->data_len += bfr_len;
459 skb->truesize += bfr_len;
460 get_page(virt_to_page(bfr));
461 skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
462 offset_in_page(bfr), bfr_len);
463 (*frags)++;
464 }
465
466
467 static inline void
468 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
469 struct scatterlist *sg, unsigned int count)
470 {
471 struct page *spage;
472 unsigned int page_len;
473
474 skb->len += count;
475 skb->data_len += count;
476 skb->truesize += count;
477
478 while (count > 0) {
479 if (!sg || (!(sg->length)))
480 break;
481 spage = sg_page(sg);
482 get_page(spage);
483 page_len = min(sg->length, count);
484 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
485 (*frags)++;
486 count -= page_len;
487 sg = sg_next(sg);
488 }
489 }
490
491 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
492 {
493 struct adapter *adap = netdev2adap(dev);
494 struct sge_uld_txq_info *txq_info =
495 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
496 struct sge_uld_txq *txq;
497 int ret = 0;
498
499 local_bh_disable();
500 txq = &txq_info->uldtxq[idx];
501 spin_lock(&txq->sendq.lock);
502 if (txq->full)
503 ret = -1;
504 spin_unlock(&txq->sendq.lock);
505 local_bh_enable();
506 return ret;
507 }
508
509 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
510 struct _key_ctx *key_ctx)
511 {
512 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
513 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
514 } else {
515 memcpy(key_ctx->key,
516 ablkctx->key + (ablkctx->enckey_len >> 1),
517 ablkctx->enckey_len >> 1);
518 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
519 ablkctx->rrkey, ablkctx->enckey_len >> 1);
520 }
521 return 0;
522 }
523 static int chcr_sg_ent_in_wr(struct scatterlist *src,
524 struct scatterlist *dst,
525 unsigned int minsg,
526 unsigned int space,
527 short int *sent,
528 short int *dent)
529 {
530 int srclen = 0, dstlen = 0;
531 int srcsg = minsg, dstsg = 0;
532
533 *sent = 0;
534 *dent = 0;
535 while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) &&
536 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
537 srclen += src->length;
538 srcsg++;
539 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
540 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
541 if (srclen <= dstlen)
542 break;
543 dstlen += dst->length;
544 dst = sg_next(dst);
545 dstsg++;
546 }
547 src = sg_next(src);
548 }
549 *sent = srcsg - minsg;
550 *dent = dstsg;
551 return min(srclen, dstlen);
552 }
553
554 static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
555 u32 flags,
556 struct scatterlist *src,
557 struct scatterlist *dst,
558 unsigned int nbytes,
559 u8 *iv,
560 unsigned short op_type)
561 {
562 int err;
563
564 SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
565 skcipher_request_set_tfm(subreq, cipher);
566 skcipher_request_set_callback(subreq, flags, NULL, NULL);
567 skcipher_request_set_crypt(subreq, src, dst,
568 nbytes, iv);
569
570 err = op_type ? crypto_skcipher_decrypt(subreq) :
571 crypto_skcipher_encrypt(subreq);
572 skcipher_request_zero(subreq);
573
574 return err;
575
576 }
577 static inline void create_wreq(struct chcr_context *ctx,
578 struct chcr_wr *chcr_req,
579 void *req, struct sk_buff *skb,
580 int kctx_len, int hash_sz,
581 int is_iv,
582 unsigned int sc_len,
583 unsigned int lcb)
584 {
585 struct uld_ctx *u_ctx = ULD_CTX(ctx);
586 int iv_loc = IV_DSGL;
587 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
588 unsigned int immdatalen = 0, nr_frags = 0;
589
590 if (is_ofld_imm(skb)) {
591 immdatalen = skb->data_len;
592 iv_loc = IV_IMMEDIATE;
593 } else {
594 nr_frags = skb_shinfo(skb)->nr_frags;
595 }
596
597 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
598 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
599 chcr_req->wreq.pld_size_hash_size =
600 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
601 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
602 chcr_req->wreq.len16_pkd =
603 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
604 (calc_tx_flits_ofld(skb) * 8), 16)));
605 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
606 chcr_req->wreq.rx_chid_to_rx_q_id =
607 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
608 is_iv ? iv_loc : IV_NOP, !!lcb,
609 ctx->tx_qidx);
610
611 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
612 qid);
613 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
614 16) - ((sizeof(chcr_req->wreq)) >> 4)));
615
616 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
617 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
618 sizeof(chcr_req->key_ctx) +
619 kctx_len + sc_len + immdatalen);
620 }
621
622 /**
623 * create_cipher_wr - form the WR for cipher operations
624 * @req: cipher req.
625 * @ctx: crypto driver context of the request.
626 * @qid: ingress qid where response of this WR should be received.
627 * @op_type: encryption or decryption
628 */
629 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
630 {
631 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
632 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
633 struct uld_ctx *u_ctx = ULD_CTX(ctx);
634 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
635 struct sk_buff *skb = NULL;
636 struct chcr_wr *chcr_req;
637 struct cpl_rx_phys_dsgl *phys_cpl;
638 struct chcr_blkcipher_req_ctx *reqctx =
639 ablkcipher_request_ctx(wrparam->req);
640 struct phys_sge_parm sg_param;
641 unsigned int frags = 0, transhdr_len, phys_dsgl;
642 int error;
643 unsigned int ivsize = AES_BLOCK_SIZE, kctx_len;
644 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
645 GFP_KERNEL : GFP_ATOMIC;
646 struct adapter *adap = padap(ctx->dev);
647
648 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
649
650 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
651 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
652 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
653 if (!skb) {
654 error = -ENOMEM;
655 goto err;
656 }
657 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
658 chcr_req = __skb_put_zero(skb, transhdr_len);
659 chcr_req->sec_cpl.op_ivinsrtofst =
660 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
661
662 chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes);
663 chcr_req->sec_cpl.aadstart_cipherstop_hi =
664 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
665
666 chcr_req->sec_cpl.cipherstop_lo_authinsert =
667 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
668 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
669 ablkctx->ciph_mode,
670 0, 0, ivsize >> 1);
671 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
672 0, 1, phys_dsgl);
673
674 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
675 if ((reqctx->op == CHCR_DECRYPT_OP) &&
676 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
677 CRYPTO_ALG_SUB_TYPE_CTR)) &&
678 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
679 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
680 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
681 } else {
682 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
683 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
684 memcpy(chcr_req->key_ctx.key, ablkctx->key,
685 ablkctx->enckey_len);
686 } else {
687 memcpy(chcr_req->key_ctx.key, ablkctx->key +
688 (ablkctx->enckey_len >> 1),
689 ablkctx->enckey_len >> 1);
690 memcpy(chcr_req->key_ctx.key +
691 (ablkctx->enckey_len >> 1),
692 ablkctx->key,
693 ablkctx->enckey_len >> 1);
694 }
695 }
696 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
697 sg_param.nents = reqctx->dst_nents;
698 sg_param.obsize = wrparam->bytes;
699 sg_param.qid = wrparam->qid;
700 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
701 reqctx->dst, &sg_param);
702 if (error)
703 goto map_fail1;
704
705 skb_set_transport_header(skb, transhdr_len);
706 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
707 write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes);
708 atomic_inc(&adap->chcr_stats.cipher_rqst);
709 create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1,
710 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl,
711 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
712 reqctx->skb = skb;
713 skb_get(skb);
714 return skb;
715 map_fail1:
716 kfree_skb(skb);
717 err:
718 return ERR_PTR(error);
719 }
720
721 static inline int chcr_keyctx_ck_size(unsigned int keylen)
722 {
723 int ck_size = 0;
724
725 if (keylen == AES_KEYSIZE_128)
726 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
727 else if (keylen == AES_KEYSIZE_192)
728 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
729 else if (keylen == AES_KEYSIZE_256)
730 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
731 else
732 ck_size = 0;
733
734 return ck_size;
735 }
736 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
737 const u8 *key,
738 unsigned int keylen)
739 {
740 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
741 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
742 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
743 int err = 0;
744
745 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
746 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
747 CRYPTO_TFM_REQ_MASK);
748 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
749 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
750 tfm->crt_flags |=
751 crypto_skcipher_get_flags(ablkctx->sw_cipher) &
752 CRYPTO_TFM_RES_MASK;
753 return err;
754 }
755
756 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
757 const u8 *key,
758 unsigned int keylen)
759 {
760 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
761 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
762 unsigned int ck_size, context_size;
763 u16 alignment = 0;
764 int err;
765
766 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
767 if (err)
768 goto badkey_err;
769
770 ck_size = chcr_keyctx_ck_size(keylen);
771 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
772 memcpy(ablkctx->key, key, keylen);
773 ablkctx->enckey_len = keylen;
774 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
775 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
776 keylen + alignment) >> 4;
777
778 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
779 0, 0, context_size);
780 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
781 return 0;
782 badkey_err:
783 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
784 ablkctx->enckey_len = 0;
785
786 return err;
787 }
788
789 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
790 const u8 *key,
791 unsigned int keylen)
792 {
793 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
794 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
795 unsigned int ck_size, context_size;
796 u16 alignment = 0;
797 int err;
798
799 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
800 if (err)
801 goto badkey_err;
802 ck_size = chcr_keyctx_ck_size(keylen);
803 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
804 memcpy(ablkctx->key, key, keylen);
805 ablkctx->enckey_len = keylen;
806 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
807 keylen + alignment) >> 4;
808
809 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
810 0, 0, context_size);
811 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
812
813 return 0;
814 badkey_err:
815 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
816 ablkctx->enckey_len = 0;
817
818 return err;
819 }
820
821 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
822 const u8 *key,
823 unsigned int keylen)
824 {
825 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
826 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
827 unsigned int ck_size, context_size;
828 u16 alignment = 0;
829 int err;
830
831 if (keylen < CTR_RFC3686_NONCE_SIZE)
832 return -EINVAL;
833 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
834 CTR_RFC3686_NONCE_SIZE);
835
836 keylen -= CTR_RFC3686_NONCE_SIZE;
837 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
838 if (err)
839 goto badkey_err;
840
841 ck_size = chcr_keyctx_ck_size(keylen);
842 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
843 memcpy(ablkctx->key, key, keylen);
844 ablkctx->enckey_len = keylen;
845 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
846 keylen + alignment) >> 4;
847
848 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
849 0, 0, context_size);
850 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
851
852 return 0;
853 badkey_err:
854 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
855 ablkctx->enckey_len = 0;
856
857 return err;
858 }
859 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
860 {
861 unsigned int size = AES_BLOCK_SIZE;
862 __be32 *b = (__be32 *)(dstiv + size);
863 u32 c, prev;
864
865 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
866 for (; size >= 4; size -= 4) {
867 prev = be32_to_cpu(*--b);
868 c = prev + add;
869 *b = cpu_to_be32(c);
870 if (prev < c)
871 break;
872 add = 1;
873 }
874
875 }
876
877 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
878 {
879 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
880 u64 c;
881 u32 temp = be32_to_cpu(*--b);
882
883 temp = ~temp;
884 c = (u64)temp + 1; // No of block can processed withou overflow
885 if ((bytes / AES_BLOCK_SIZE) > c)
886 bytes = c * AES_BLOCK_SIZE;
887 return bytes;
888 }
889
890 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
891 {
892 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
893 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
894 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
895 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
896 struct crypto_cipher *cipher;
897 int ret, i;
898 u8 *key;
899 unsigned int keylen;
900
901 cipher = ablkctx->aes_generic;
902 memcpy(iv, req->info, AES_BLOCK_SIZE);
903
904 keylen = ablkctx->enckey_len / 2;
905 key = ablkctx->key + keylen;
906 ret = crypto_cipher_setkey(cipher, key, keylen);
907 if (ret)
908 goto out;
909
910 crypto_cipher_encrypt_one(cipher, iv, iv);
911 for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++)
912 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
913
914 crypto_cipher_decrypt_one(cipher, iv, iv);
915 out:
916 return ret;
917 }
918
919 static int chcr_update_cipher_iv(struct ablkcipher_request *req,
920 struct cpl_fw6_pld *fw6_pld, u8 *iv)
921 {
922 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
923 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
924 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
925 int ret = 0;
926
927 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
928 ctr_add_iv(iv, req->info, (reqctx->processed /
929 AES_BLOCK_SIZE));
930 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
931 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
932 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
933 AES_BLOCK_SIZE) + 1);
934 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
935 ret = chcr_update_tweak(req, iv);
936 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
937 if (reqctx->op)
938 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
939 16,
940 reqctx->processed - AES_BLOCK_SIZE);
941 else
942 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
943 }
944
945 return ret;
946
947 }
948
949 /* We need separate function for final iv because in rfc3686 Initial counter
950 * starts from 1 and buffer size of iv is 8 byte only which remains constant
951 * for subsequent update requests
952 */
953
954 static int chcr_final_cipher_iv(struct ablkcipher_request *req,
955 struct cpl_fw6_pld *fw6_pld, u8 *iv)
956 {
957 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
958 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
959 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
960 int ret = 0;
961
962 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
963 ctr_add_iv(iv, req->info, (reqctx->processed /
964 AES_BLOCK_SIZE));
965 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
966 ret = chcr_update_tweak(req, iv);
967 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
968 if (reqctx->op)
969 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
970 16,
971 reqctx->processed - AES_BLOCK_SIZE);
972 else
973 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
974
975 }
976 return ret;
977
978 }
979
980
981 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
982 unsigned char *input, int err)
983 {
984 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
985 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
986 struct uld_ctx *u_ctx = ULD_CTX(ctx);
987 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
988 struct sk_buff *skb;
989 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
990 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
991 struct cipher_wr_param wrparam;
992 int bytes;
993
994 dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents,
995 DMA_FROM_DEVICE);
996
997 if (reqctx->skb) {
998 kfree_skb(reqctx->skb);
999 reqctx->skb = NULL;
1000 }
1001 if (err)
1002 goto complete;
1003
1004 if (req->nbytes == reqctx->processed) {
1005 err = chcr_final_cipher_iv(req, fw6_pld, req->info);
1006 goto complete;
1007 }
1008
1009 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1010 ctx->tx_qidx))) {
1011 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1012 err = -EBUSY;
1013 goto complete;
1014 }
1015
1016 }
1017 wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src,
1018 reqctx->processed);
1019 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg,
1020 reqctx->processed);
1021 if (!wrparam.srcsg || !reqctx->dst) {
1022 pr_err("Input sg list length less that nbytes\n");
1023 err = -EINVAL;
1024 goto complete;
1025 }
1026 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1,
1027 SPACE_LEFT(ablkctx->enckey_len),
1028 &wrparam.snent, &reqctx->dst_nents);
1029 if ((bytes + reqctx->processed) >= req->nbytes)
1030 bytes = req->nbytes - reqctx->processed;
1031 else
1032 bytes = ROUND_16(bytes);
1033 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1034 if (err)
1035 goto complete;
1036
1037 if (unlikely(bytes == 0)) {
1038 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1039 req->base.flags,
1040 wrparam.srcsg,
1041 reqctx->dst,
1042 req->nbytes - reqctx->processed,
1043 reqctx->iv,
1044 reqctx->op);
1045 goto complete;
1046 }
1047
1048 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1049 CRYPTO_ALG_SUB_TYPE_CTR)
1050 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1051 reqctx->processed += bytes;
1052 wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
1053 wrparam.req = req;
1054 wrparam.bytes = bytes;
1055 skb = create_cipher_wr(&wrparam);
1056 if (IS_ERR(skb)) {
1057 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1058 err = PTR_ERR(skb);
1059 goto complete;
1060 }
1061 skb->dev = u_ctx->lldi.ports[0];
1062 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1063 chcr_send_wr(skb);
1064 return 0;
1065 complete:
1066 free_new_sg(reqctx->newdstsg);
1067 reqctx->newdstsg = NULL;
1068 req->base.complete(&req->base, err);
1069 return err;
1070 }
1071
1072 static int process_cipher(struct ablkcipher_request *req,
1073 unsigned short qid,
1074 struct sk_buff **skb,
1075 unsigned short op_type)
1076 {
1077 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1078 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
1079 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1080 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1081 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1082 struct cipher_wr_param wrparam;
1083 int bytes, nents, err = -EINVAL;
1084
1085 reqctx->newdstsg = NULL;
1086 reqctx->processed = 0;
1087 if (!req->info)
1088 goto error;
1089 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1090 (req->nbytes == 0) ||
1091 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
1092 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1093 ablkctx->enckey_len, req->nbytes, ivsize);
1094 goto error;
1095 }
1096 wrparam.srcsg = req->src;
1097 if (is_newsg(req->dst, &nents)) {
1098 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
1099 if (IS_ERR(reqctx->newdstsg))
1100 return PTR_ERR(reqctx->newdstsg);
1101 reqctx->dstsg = reqctx->newdstsg;
1102 } else {
1103 reqctx->dstsg = req->dst;
1104 }
1105 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG,
1106 SPACE_LEFT(ablkctx->enckey_len),
1107 &wrparam.snent,
1108 &reqctx->dst_nents);
1109 if ((bytes + reqctx->processed) >= req->nbytes)
1110 bytes = req->nbytes - reqctx->processed;
1111 else
1112 bytes = ROUND_16(bytes);
1113 if (unlikely(bytes > req->nbytes))
1114 bytes = req->nbytes;
1115 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1116 CRYPTO_ALG_SUB_TYPE_CTR) {
1117 bytes = adjust_ctr_overflow(req->info, bytes);
1118 }
1119 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1120 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1121 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1122 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
1123 CTR_RFC3686_IV_SIZE);
1124
1125 /* initialize counter portion of counter block */
1126 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1127 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1128
1129 } else {
1130
1131 memcpy(reqctx->iv, req->info, ivsize);
1132 }
1133 if (unlikely(bytes == 0)) {
1134 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1135 req->base.flags,
1136 req->src,
1137 req->dst,
1138 req->nbytes,
1139 req->info,
1140 op_type);
1141 goto error;
1142 }
1143 reqctx->processed = bytes;
1144 reqctx->dst = reqctx->dstsg;
1145 reqctx->op = op_type;
1146 wrparam.qid = qid;
1147 wrparam.req = req;
1148 wrparam.bytes = bytes;
1149 *skb = create_cipher_wr(&wrparam);
1150 if (IS_ERR(*skb)) {
1151 err = PTR_ERR(*skb);
1152 goto error;
1153 }
1154
1155 return 0;
1156 error:
1157 free_new_sg(reqctx->newdstsg);
1158 reqctx->newdstsg = NULL;
1159 return err;
1160 }
1161
1162 static int chcr_aes_encrypt(struct ablkcipher_request *req)
1163 {
1164 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1165 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1166 struct sk_buff *skb = NULL;
1167 int err;
1168 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1169
1170 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1171 ctx->tx_qidx))) {
1172 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1173 return -EBUSY;
1174 }
1175
1176 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
1177 CHCR_ENCRYPT_OP);
1178 if (err || !skb)
1179 return err;
1180 skb->dev = u_ctx->lldi.ports[0];
1181 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1182 chcr_send_wr(skb);
1183 return -EINPROGRESS;
1184 }
1185
1186 static int chcr_aes_decrypt(struct ablkcipher_request *req)
1187 {
1188 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1189 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1190 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1191 struct sk_buff *skb = NULL;
1192 int err;
1193
1194 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1195 ctx->tx_qidx))) {
1196 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1197 return -EBUSY;
1198 }
1199
1200 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
1201 CHCR_DECRYPT_OP);
1202 if (err || !skb)
1203 return err;
1204 skb->dev = u_ctx->lldi.ports[0];
1205 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1206 chcr_send_wr(skb);
1207 return -EINPROGRESS;
1208 }
1209
1210 static int chcr_device_init(struct chcr_context *ctx)
1211 {
1212 struct uld_ctx *u_ctx = NULL;
1213 struct adapter *adap;
1214 unsigned int id;
1215 int txq_perchan, txq_idx, ntxq;
1216 int err = 0, rxq_perchan, rxq_idx;
1217
1218 id = smp_processor_id();
1219 if (!ctx->dev) {
1220 u_ctx = assign_chcr_device();
1221 if (!u_ctx) {
1222 pr_err("chcr device assignment fails\n");
1223 goto out;
1224 }
1225 ctx->dev = u_ctx->dev;
1226 adap = padap(ctx->dev);
1227 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
1228 adap->vres.ncrypto_fc);
1229 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1230 txq_perchan = ntxq / u_ctx->lldi.nchan;
1231 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
1232 rxq_idx += id % rxq_perchan;
1233 txq_idx = ctx->dev->tx_channel_id * txq_perchan;
1234 txq_idx += id % txq_perchan;
1235 spin_lock(&ctx->dev->lock_chcr_dev);
1236 ctx->rx_qidx = rxq_idx;
1237 ctx->tx_qidx = txq_idx;
1238 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
1239 ctx->dev->rx_channel_id = 0;
1240 spin_unlock(&ctx->dev->lock_chcr_dev);
1241 }
1242 out:
1243 return err;
1244 }
1245
1246 static int chcr_cra_init(struct crypto_tfm *tfm)
1247 {
1248 struct crypto_alg *alg = tfm->__crt_alg;
1249 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1250 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1251
1252 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
1253 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1254 if (IS_ERR(ablkctx->sw_cipher)) {
1255 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1256 return PTR_ERR(ablkctx->sw_cipher);
1257 }
1258
1259 if (get_cryptoalg_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_XTS) {
1260 /* To update tweak*/
1261 ablkctx->aes_generic = crypto_alloc_cipher("aes-generic", 0, 0);
1262 if (IS_ERR(ablkctx->aes_generic)) {
1263 pr_err("failed to allocate aes cipher for tweak\n");
1264 return PTR_ERR(ablkctx->aes_generic);
1265 }
1266 } else
1267 ablkctx->aes_generic = NULL;
1268
1269 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1270 return chcr_device_init(crypto_tfm_ctx(tfm));
1271 }
1272
1273 static int chcr_rfc3686_init(struct crypto_tfm *tfm)
1274 {
1275 struct crypto_alg *alg = tfm->__crt_alg;
1276 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1277 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1278
1279 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1280 * cannot be used as fallback in chcr_handle_cipher_response
1281 */
1282 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1283 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1284 if (IS_ERR(ablkctx->sw_cipher)) {
1285 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1286 return PTR_ERR(ablkctx->sw_cipher);
1287 }
1288 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1289 return chcr_device_init(crypto_tfm_ctx(tfm));
1290 }
1291
1292
1293 static void chcr_cra_exit(struct crypto_tfm *tfm)
1294 {
1295 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1296 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1297
1298 crypto_free_skcipher(ablkctx->sw_cipher);
1299 if (ablkctx->aes_generic)
1300 crypto_free_cipher(ablkctx->aes_generic);
1301 }
1302
1303 static int get_alg_config(struct algo_param *params,
1304 unsigned int auth_size)
1305 {
1306 switch (auth_size) {
1307 case SHA1_DIGEST_SIZE:
1308 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1309 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1310 params->result_size = SHA1_DIGEST_SIZE;
1311 break;
1312 case SHA224_DIGEST_SIZE:
1313 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1314 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1315 params->result_size = SHA256_DIGEST_SIZE;
1316 break;
1317 case SHA256_DIGEST_SIZE:
1318 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1319 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1320 params->result_size = SHA256_DIGEST_SIZE;
1321 break;
1322 case SHA384_DIGEST_SIZE:
1323 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1324 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1325 params->result_size = SHA512_DIGEST_SIZE;
1326 break;
1327 case SHA512_DIGEST_SIZE:
1328 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1329 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1330 params->result_size = SHA512_DIGEST_SIZE;
1331 break;
1332 default:
1333 pr_err("chcr : ERROR, unsupported digest size\n");
1334 return -EINVAL;
1335 }
1336 return 0;
1337 }
1338
1339 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1340 {
1341 crypto_free_shash(base_hash);
1342 }
1343
1344 /**
1345 * create_hash_wr - Create hash work request
1346 * @req - Cipher req base
1347 */
1348 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1349 struct hash_wr_param *param)
1350 {
1351 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1352 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1353 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1354 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1355 struct sk_buff *skb = NULL;
1356 struct chcr_wr *chcr_req;
1357 unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
1358 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1359 unsigned int kctx_len = 0;
1360 u8 hash_size_in_response = 0;
1361 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1362 GFP_ATOMIC;
1363 struct adapter *adap = padap(ctx->dev);
1364
1365 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
1366 kctx_len = param->alg_prm.result_size + iopad_alignment;
1367 if (param->opad_needed)
1368 kctx_len += param->alg_prm.result_size + iopad_alignment;
1369
1370 if (req_ctx->result)
1371 hash_size_in_response = digestsize;
1372 else
1373 hash_size_in_response = param->alg_prm.result_size;
1374 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
1375 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1376 if (!skb)
1377 return skb;
1378
1379 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1380 chcr_req = __skb_put_zero(skb, transhdr_len);
1381
1382 chcr_req->sec_cpl.op_ivinsrtofst =
1383 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
1384 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1385
1386 chcr_req->sec_cpl.aadstart_cipherstop_hi =
1387 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1388 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1389 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1390 chcr_req->sec_cpl.seqno_numivs =
1391 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1392 param->opad_needed, 0);
1393
1394 chcr_req->sec_cpl.ivgen_hdrlen =
1395 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1396
1397 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1398 param->alg_prm.result_size);
1399
1400 if (param->opad_needed)
1401 memcpy(chcr_req->key_ctx.key +
1402 ((param->alg_prm.result_size <= 32) ? 32 :
1403 CHCR_HASH_MAX_DIGEST_SIZE),
1404 hmacctx->opad, param->alg_prm.result_size);
1405
1406 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1407 param->alg_prm.mk_size, 0,
1408 param->opad_needed,
1409 ((kctx_len +
1410 sizeof(chcr_req->key_ctx)) >> 4));
1411 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1412
1413 skb_set_transport_header(skb, transhdr_len);
1414 if (param->bfr_len != 0)
1415 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
1416 param->bfr_len);
1417 if (param->sg_len != 0)
1418 write_sg_to_skb(skb, &frags, req->src, param->sg_len);
1419 atomic_inc(&adap->chcr_stats.digest_rqst);
1420 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len,
1421 hash_size_in_response, 0, DUMMY_BYTES, 0);
1422 req_ctx->skb = skb;
1423 skb_get(skb);
1424 return skb;
1425 }
1426
1427 static int chcr_ahash_update(struct ahash_request *req)
1428 {
1429 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1430 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1431 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1432 struct uld_ctx *u_ctx = NULL;
1433 struct sk_buff *skb;
1434 u8 remainder = 0, bs;
1435 unsigned int nbytes = req->nbytes;
1436 struct hash_wr_param params;
1437
1438 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1439
1440 u_ctx = ULD_CTX(ctx);
1441 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1442 ctx->tx_qidx))) {
1443 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1444 return -EBUSY;
1445 }
1446
1447 if (nbytes + req_ctx->reqlen >= bs) {
1448 remainder = (nbytes + req_ctx->reqlen) % bs;
1449 nbytes = nbytes + req_ctx->reqlen - remainder;
1450 } else {
1451 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1452 + req_ctx->reqlen, nbytes, 0);
1453 req_ctx->reqlen += nbytes;
1454 return 0;
1455 }
1456
1457 params.opad_needed = 0;
1458 params.more = 1;
1459 params.last = 0;
1460 params.sg_len = nbytes - req_ctx->reqlen;
1461 params.bfr_len = req_ctx->reqlen;
1462 params.scmd1 = 0;
1463 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1464 req_ctx->result = 0;
1465 req_ctx->data_len += params.sg_len + params.bfr_len;
1466 skb = create_hash_wr(req, &params);
1467 if (!skb)
1468 return -ENOMEM;
1469
1470 if (remainder) {
1471 u8 *temp;
1472 /* Swap buffers */
1473 temp = req_ctx->reqbfr;
1474 req_ctx->reqbfr = req_ctx->skbfr;
1475 req_ctx->skbfr = temp;
1476 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1477 req_ctx->reqbfr, remainder, req->nbytes -
1478 remainder);
1479 }
1480 req_ctx->reqlen = remainder;
1481 skb->dev = u_ctx->lldi.ports[0];
1482 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1483 chcr_send_wr(skb);
1484
1485 return -EINPROGRESS;
1486 }
1487
1488 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1489 {
1490 memset(bfr_ptr, 0, bs);
1491 *bfr_ptr = 0x80;
1492 if (bs == 64)
1493 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1494 else
1495 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1496 }
1497
1498 static int chcr_ahash_final(struct ahash_request *req)
1499 {
1500 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1501 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1502 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1503 struct hash_wr_param params;
1504 struct sk_buff *skb;
1505 struct uld_ctx *u_ctx = NULL;
1506 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1507
1508 u_ctx = ULD_CTX(ctx);
1509 if (is_hmac(crypto_ahash_tfm(rtfm)))
1510 params.opad_needed = 1;
1511 else
1512 params.opad_needed = 0;
1513 params.sg_len = 0;
1514 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1515 req_ctx->result = 1;
1516 params.bfr_len = req_ctx->reqlen;
1517 req_ctx->data_len += params.bfr_len + params.sg_len;
1518 if (req_ctx->reqlen == 0) {
1519 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1520 params.last = 0;
1521 params.more = 1;
1522 params.scmd1 = 0;
1523 params.bfr_len = bs;
1524
1525 } else {
1526 params.scmd1 = req_ctx->data_len;
1527 params.last = 1;
1528 params.more = 0;
1529 }
1530 skb = create_hash_wr(req, &params);
1531 if (!skb)
1532 return -ENOMEM;
1533
1534 skb->dev = u_ctx->lldi.ports[0];
1535 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1536 chcr_send_wr(skb);
1537 return -EINPROGRESS;
1538 }
1539
1540 static int chcr_ahash_finup(struct ahash_request *req)
1541 {
1542 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1543 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1544 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1545 struct uld_ctx *u_ctx = NULL;
1546 struct sk_buff *skb;
1547 struct hash_wr_param params;
1548 u8 bs;
1549
1550 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1551 u_ctx = ULD_CTX(ctx);
1552
1553 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1554 ctx->tx_qidx))) {
1555 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1556 return -EBUSY;
1557 }
1558
1559 if (is_hmac(crypto_ahash_tfm(rtfm)))
1560 params.opad_needed = 1;
1561 else
1562 params.opad_needed = 0;
1563
1564 params.sg_len = req->nbytes;
1565 params.bfr_len = req_ctx->reqlen;
1566 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1567 req_ctx->data_len += params.bfr_len + params.sg_len;
1568 req_ctx->result = 1;
1569 if ((req_ctx->reqlen + req->nbytes) == 0) {
1570 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1571 params.last = 0;
1572 params.more = 1;
1573 params.scmd1 = 0;
1574 params.bfr_len = bs;
1575 } else {
1576 params.scmd1 = req_ctx->data_len;
1577 params.last = 1;
1578 params.more = 0;
1579 }
1580
1581 skb = create_hash_wr(req, &params);
1582 if (!skb)
1583 return -ENOMEM;
1584
1585 skb->dev = u_ctx->lldi.ports[0];
1586 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1587 chcr_send_wr(skb);
1588
1589 return -EINPROGRESS;
1590 }
1591
1592 static int chcr_ahash_digest(struct ahash_request *req)
1593 {
1594 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1595 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1596 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1597 struct uld_ctx *u_ctx = NULL;
1598 struct sk_buff *skb;
1599 struct hash_wr_param params;
1600 u8 bs;
1601
1602 rtfm->init(req);
1603 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1604
1605 u_ctx = ULD_CTX(ctx);
1606 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1607 ctx->tx_qidx))) {
1608 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1609 return -EBUSY;
1610 }
1611
1612 if (is_hmac(crypto_ahash_tfm(rtfm)))
1613 params.opad_needed = 1;
1614 else
1615 params.opad_needed = 0;
1616
1617 params.last = 0;
1618 params.more = 0;
1619 params.sg_len = req->nbytes;
1620 params.bfr_len = 0;
1621 params.scmd1 = 0;
1622 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1623 req_ctx->result = 1;
1624 req_ctx->data_len += params.bfr_len + params.sg_len;
1625
1626 if (req->nbytes == 0) {
1627 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1628 params.more = 1;
1629 params.bfr_len = bs;
1630 }
1631
1632 skb = create_hash_wr(req, &params);
1633 if (!skb)
1634 return -ENOMEM;
1635
1636 skb->dev = u_ctx->lldi.ports[0];
1637 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1638 chcr_send_wr(skb);
1639 return -EINPROGRESS;
1640 }
1641
1642 static int chcr_ahash_export(struct ahash_request *areq, void *out)
1643 {
1644 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1645 struct chcr_ahash_req_ctx *state = out;
1646
1647 state->reqlen = req_ctx->reqlen;
1648 state->data_len = req_ctx->data_len;
1649 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
1650 memcpy(state->partial_hash, req_ctx->partial_hash,
1651 CHCR_HASH_MAX_DIGEST_SIZE);
1652 return 0;
1653 }
1654
1655 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1656 {
1657 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1658 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1659
1660 req_ctx->reqlen = state->reqlen;
1661 req_ctx->data_len = state->data_len;
1662 req_ctx->reqbfr = req_ctx->bfr1;
1663 req_ctx->skbfr = req_ctx->bfr2;
1664 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
1665 memcpy(req_ctx->partial_hash, state->partial_hash,
1666 CHCR_HASH_MAX_DIGEST_SIZE);
1667 return 0;
1668 }
1669
1670 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1671 unsigned int keylen)
1672 {
1673 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1674 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1675 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1676 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1677 unsigned int i, err = 0, updated_digestsize;
1678
1679 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1680
1681 /* use the key to calculate the ipad and opad. ipad will sent with the
1682 * first request's data. opad will be sent with the final hash result
1683 * ipad in hmacctx->ipad and opad in hmacctx->opad location
1684 */
1685 shash->tfm = hmacctx->base_hash;
1686 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
1687 if (keylen > bs) {
1688 err = crypto_shash_digest(shash, key, keylen,
1689 hmacctx->ipad);
1690 if (err)
1691 goto out;
1692 keylen = digestsize;
1693 } else {
1694 memcpy(hmacctx->ipad, key, keylen);
1695 }
1696 memset(hmacctx->ipad + keylen, 0, bs - keylen);
1697 memcpy(hmacctx->opad, hmacctx->ipad, bs);
1698
1699 for (i = 0; i < bs / sizeof(int); i++) {
1700 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1701 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1702 }
1703
1704 updated_digestsize = digestsize;
1705 if (digestsize == SHA224_DIGEST_SIZE)
1706 updated_digestsize = SHA256_DIGEST_SIZE;
1707 else if (digestsize == SHA384_DIGEST_SIZE)
1708 updated_digestsize = SHA512_DIGEST_SIZE;
1709 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
1710 hmacctx->ipad, digestsize);
1711 if (err)
1712 goto out;
1713 chcr_change_order(hmacctx->ipad, updated_digestsize);
1714
1715 err = chcr_compute_partial_hash(shash, hmacctx->opad,
1716 hmacctx->opad, digestsize);
1717 if (err)
1718 goto out;
1719 chcr_change_order(hmacctx->opad, updated_digestsize);
1720 out:
1721 return err;
1722 }
1723
1724 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1725 unsigned int key_len)
1726 {
1727 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
1728 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1729 unsigned short context_size = 0;
1730 int err;
1731
1732 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
1733 if (err)
1734 goto badkey_err;
1735
1736 memcpy(ablkctx->key, key, key_len);
1737 ablkctx->enckey_len = key_len;
1738 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1739 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1740 ablkctx->key_ctx_hdr =
1741 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1742 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1743 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1744 CHCR_KEYCTX_NO_KEY, 1,
1745 0, context_size);
1746 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1747 return 0;
1748 badkey_err:
1749 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1750 ablkctx->enckey_len = 0;
1751
1752 return err;
1753 }
1754
1755 static int chcr_sha_init(struct ahash_request *areq)
1756 {
1757 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1758 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1759 int digestsize = crypto_ahash_digestsize(tfm);
1760
1761 req_ctx->data_len = 0;
1762 req_ctx->reqlen = 0;
1763 req_ctx->reqbfr = req_ctx->bfr1;
1764 req_ctx->skbfr = req_ctx->bfr2;
1765 req_ctx->skb = NULL;
1766 req_ctx->result = 0;
1767 copy_hash_init_values(req_ctx->partial_hash, digestsize);
1768 return 0;
1769 }
1770
1771 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1772 {
1773 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1774 sizeof(struct chcr_ahash_req_ctx));
1775 return chcr_device_init(crypto_tfm_ctx(tfm));
1776 }
1777
1778 static int chcr_hmac_init(struct ahash_request *areq)
1779 {
1780 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1781 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1782 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1783 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1784 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1785 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1786
1787 chcr_sha_init(areq);
1788 req_ctx->data_len = bs;
1789 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1790 if (digestsize == SHA224_DIGEST_SIZE)
1791 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1792 SHA256_DIGEST_SIZE);
1793 else if (digestsize == SHA384_DIGEST_SIZE)
1794 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1795 SHA512_DIGEST_SIZE);
1796 else
1797 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1798 digestsize);
1799 }
1800 return 0;
1801 }
1802
1803 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1804 {
1805 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1806 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1807 unsigned int digestsize =
1808 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
1809
1810 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1811 sizeof(struct chcr_ahash_req_ctx));
1812 hmacctx->base_hash = chcr_alloc_shash(digestsize);
1813 if (IS_ERR(hmacctx->base_hash))
1814 return PTR_ERR(hmacctx->base_hash);
1815 return chcr_device_init(crypto_tfm_ctx(tfm));
1816 }
1817
1818 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
1819 {
1820 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1821 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1822
1823 if (hmacctx->base_hash) {
1824 chcr_free_shash(hmacctx->base_hash);
1825 hmacctx->base_hash = NULL;
1826 }
1827 }
1828
1829 static int is_newsg(struct scatterlist *sgl, unsigned int *newents)
1830 {
1831 int nents = 0;
1832 int ret = 0;
1833
1834 while (sgl) {
1835 if (sgl->length > CHCR_SG_SIZE)
1836 ret = 1;
1837 nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE);
1838 sgl = sg_next(sgl);
1839 }
1840 *newents = nents;
1841 return ret;
1842 }
1843
1844 static inline void free_new_sg(struct scatterlist *sgl)
1845 {
1846 kfree(sgl);
1847 }
1848
1849 static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
1850 unsigned int nents)
1851 {
1852 struct scatterlist *newsg, *sg;
1853 int i, len, processed = 0;
1854 struct page *spage;
1855 int offset;
1856
1857 newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL);
1858 if (!newsg)
1859 return ERR_PTR(-ENOMEM);
1860 sg = newsg;
1861 sg_init_table(sg, nents);
1862 offset = sgl->offset;
1863 spage = sg_page(sgl);
1864 for (i = 0; i < nents; i++) {
1865 len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE);
1866 sg_set_page(sg, spage, len, offset);
1867 processed += len;
1868 offset += len;
1869 if (offset >= PAGE_SIZE) {
1870 offset = offset % PAGE_SIZE;
1871 spage++;
1872 }
1873 if (processed == sgl->length) {
1874 processed = 0;
1875 sgl = sg_next(sgl);
1876 if (!sgl)
1877 break;
1878 spage = sg_page(sgl);
1879 offset = sgl->offset;
1880 }
1881 sg = sg_next(sg);
1882 }
1883 return newsg;
1884 }
1885
1886 static int chcr_copy_assoc(struct aead_request *req,
1887 struct chcr_aead_ctx *ctx)
1888 {
1889 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
1890
1891 skcipher_request_set_tfm(skreq, ctx->null);
1892 skcipher_request_set_callback(skreq, aead_request_flags(req),
1893 NULL, NULL);
1894 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
1895 NULL);
1896
1897 return crypto_skcipher_encrypt(skreq);
1898 }
1899 static int chcr_aead_need_fallback(struct aead_request *req, int src_nent,
1900 int aadmax, int wrlen,
1901 unsigned short op_type)
1902 {
1903 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
1904
1905 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
1906 (req->assoclen > aadmax) ||
1907 (src_nent > MAX_SKB_FRAGS) ||
1908 (wrlen > MAX_WR_SIZE))
1909 return 1;
1910 return 0;
1911 }
1912
1913 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
1914 {
1915 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1916 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1917 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1918 struct aead_request *subreq = aead_request_ctx(req);
1919
1920 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
1921 aead_request_set_callback(subreq, req->base.flags,
1922 req->base.complete, req->base.data);
1923 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
1924 req->iv);
1925 aead_request_set_ad(subreq, req->assoclen);
1926 return op_type ? crypto_aead_decrypt(subreq) :
1927 crypto_aead_encrypt(subreq);
1928 }
1929
1930 static struct sk_buff *create_authenc_wr(struct aead_request *req,
1931 unsigned short qid,
1932 int size,
1933 unsigned short op_type)
1934 {
1935 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1936 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1937 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1938 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1939 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
1940 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1941 struct sk_buff *skb = NULL;
1942 struct chcr_wr *chcr_req;
1943 struct cpl_rx_phys_dsgl *phys_cpl;
1944 struct phys_sge_parm sg_param;
1945 struct scatterlist *src;
1946 unsigned int frags = 0, transhdr_len;
1947 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
1948 unsigned int kctx_len = 0, nents;
1949 unsigned short stop_offset = 0;
1950 unsigned int assoclen = req->assoclen;
1951 unsigned int authsize = crypto_aead_authsize(tfm);
1952 int error = -EINVAL, src_nent;
1953 int null = 0;
1954 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1955 GFP_ATOMIC;
1956 struct adapter *adap = padap(ctx->dev);
1957
1958 reqctx->newdstsg = NULL;
1959 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
1960 authsize);
1961 if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0))
1962 goto err;
1963
1964 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1965 goto err;
1966 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
1967 if (src_nent < 0)
1968 goto err;
1969 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1970
1971 if (req->src != req->dst) {
1972 error = chcr_copy_assoc(req, aeadctx);
1973 if (error)
1974 return ERR_PTR(error);
1975 }
1976 if (dst_size && is_newsg(req->dst, &nents)) {
1977 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
1978 if (IS_ERR(reqctx->newdstsg))
1979 return ERR_CAST(reqctx->newdstsg);
1980 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
1981 reqctx->newdstsg, req->assoclen);
1982 } else {
1983 if (req->src == req->dst)
1984 reqctx->dst = src;
1985 else
1986 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
1987 req->dst, req->assoclen);
1988 }
1989 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
1990 null = 1;
1991 assoclen = 0;
1992 }
1993 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
1994 (op_type ? -authsize : authsize));
1995 if (reqctx->dst_nents < 0) {
1996 pr_err("AUTHENC:Invalid Destination sg entries\n");
1997 error = -EINVAL;
1998 goto err;
1999 }
2000 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2001 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
2002 - sizeof(chcr_req->key_ctx);
2003 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2004 if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG,
2005 T6_MAX_AAD_SIZE,
2006 transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
2007 op_type)) {
2008 atomic_inc(&adap->chcr_stats.fallback);
2009 free_new_sg(reqctx->newdstsg);
2010 reqctx->newdstsg = NULL;
2011 return ERR_PTR(chcr_aead_fallback(req, op_type));
2012 }
2013 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2014 if (!skb) {
2015 error = -ENOMEM;
2016 goto err;
2017 }
2018
2019 /* LLD is going to write the sge hdr. */
2020 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2021
2022 /* Write WR */
2023 chcr_req = __skb_put_zero(skb, transhdr_len);
2024
2025 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2026
2027 /*
2028 * Input order is AAD,IV and Payload. where IV should be included as
2029 * the part of authdata. All other fields should be filled according
2030 * to the hardware spec
2031 */
2032 chcr_req->sec_cpl.op_ivinsrtofst =
2033 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
2034 (ivsize ? (assoclen + 1) : 0));
2035 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
2036 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2037 assoclen ? 1 : 0, assoclen,
2038 assoclen + ivsize + 1,
2039 (stop_offset & 0x1F0) >> 4);
2040 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2041 stop_offset & 0xF,
2042 null ? 0 : assoclen + ivsize + 1,
2043 stop_offset, stop_offset);
2044 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2045 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
2046 CHCR_SCMD_CIPHER_MODE_AES_CBC,
2047 actx->auth_mode, aeadctx->hmac_ctrl,
2048 ivsize >> 1);
2049 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2050 0, 1, dst_size);
2051
2052 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2053 if (op_type == CHCR_ENCRYPT_OP)
2054 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2055 aeadctx->enckey_len);
2056 else
2057 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2058 aeadctx->enckey_len);
2059
2060 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
2061 4), actx->h_iopad, kctx_len -
2062 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
2063
2064 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2065 sg_param.nents = reqctx->dst_nents;
2066 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2067 sg_param.qid = qid;
2068 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2069 reqctx->dst, &sg_param);
2070 if (error)
2071 goto dstmap_fail;
2072
2073 skb_set_transport_header(skb, transhdr_len);
2074
2075 if (assoclen) {
2076 /* AAD buffer in */
2077 write_sg_to_skb(skb, &frags, req->src, assoclen);
2078
2079 }
2080 write_buffer_to_skb(skb, &frags, req->iv, ivsize);
2081 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2082 atomic_inc(&adap->chcr_stats.cipher_rqst);
2083 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
2084 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2085 reqctx->skb = skb;
2086 skb_get(skb);
2087
2088 return skb;
2089 dstmap_fail:
2090 /* ivmap_fail: */
2091 kfree_skb(skb);
2092 err:
2093 free_new_sg(reqctx->newdstsg);
2094 reqctx->newdstsg = NULL;
2095 return ERR_PTR(error);
2096 }
2097
2098 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2099 {
2100 __be32 data;
2101
2102 memset(block, 0, csize);
2103 block += csize;
2104
2105 if (csize >= 4)
2106 csize = 4;
2107 else if (msglen > (unsigned int)(1 << (8 * csize)))
2108 return -EOVERFLOW;
2109
2110 data = cpu_to_be32(msglen);
2111 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2112
2113 return 0;
2114 }
2115
2116 static void generate_b0(struct aead_request *req,
2117 struct chcr_aead_ctx *aeadctx,
2118 unsigned short op_type)
2119 {
2120 unsigned int l, lp, m;
2121 int rc;
2122 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2123 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2124 u8 *b0 = reqctx->scratch_pad;
2125
2126 m = crypto_aead_authsize(aead);
2127
2128 memcpy(b0, reqctx->iv, 16);
2129
2130 lp = b0[0];
2131 l = lp + 1;
2132
2133 /* set m, bits 3-5 */
2134 *b0 |= (8 * ((m - 2) / 2));
2135
2136 /* set adata, bit 6, if associated data is used */
2137 if (req->assoclen)
2138 *b0 |= 64;
2139 rc = set_msg_len(b0 + 16 - l,
2140 (op_type == CHCR_DECRYPT_OP) ?
2141 req->cryptlen - m : req->cryptlen, l);
2142 }
2143
2144 static inline int crypto_ccm_check_iv(const u8 *iv)
2145 {
2146 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2147 if (iv[0] < 1 || iv[0] > 7)
2148 return -EINVAL;
2149
2150 return 0;
2151 }
2152
2153 static int ccm_format_packet(struct aead_request *req,
2154 struct chcr_aead_ctx *aeadctx,
2155 unsigned int sub_type,
2156 unsigned short op_type)
2157 {
2158 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2159 int rc = 0;
2160
2161 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2162 reqctx->iv[0] = 3;
2163 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
2164 memcpy(reqctx->iv + 4, req->iv, 8);
2165 memset(reqctx->iv + 12, 0, 4);
2166 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2167 htons(req->assoclen - 8);
2168 } else {
2169 memcpy(reqctx->iv, req->iv, 16);
2170 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2171 htons(req->assoclen);
2172 }
2173 generate_b0(req, aeadctx, op_type);
2174 /* zero the ctr value */
2175 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
2176 return rc;
2177 }
2178
2179 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2180 unsigned int dst_size,
2181 struct aead_request *req,
2182 unsigned short op_type,
2183 struct chcr_context *chcrctx)
2184 {
2185 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2186 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2187 unsigned int ivsize = AES_BLOCK_SIZE;
2188 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2189 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2190 unsigned int c_id = chcrctx->dev->rx_channel_id;
2191 unsigned int ccm_xtra;
2192 unsigned char tag_offset = 0, auth_offset = 0;
2193 unsigned int assoclen;
2194
2195 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2196 assoclen = req->assoclen - 8;
2197 else
2198 assoclen = req->assoclen;
2199 ccm_xtra = CCM_B0_SIZE +
2200 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2201
2202 auth_offset = req->cryptlen ?
2203 (assoclen + ivsize + 1 + ccm_xtra) : 0;
2204 if (op_type == CHCR_DECRYPT_OP) {
2205 if (crypto_aead_authsize(tfm) != req->cryptlen)
2206 tag_offset = crypto_aead_authsize(tfm);
2207 else
2208 auth_offset = 0;
2209 }
2210
2211
2212 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
2213 2, (ivsize ? (assoclen + 1) : 0) +
2214 ccm_xtra);
2215 sec_cpl->pldlen =
2216 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
2217 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2218 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2219 1, assoclen + ccm_xtra, assoclen
2220 + ivsize + 1 + ccm_xtra, 0);
2221
2222 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2223 auth_offset, tag_offset,
2224 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2225 crypto_aead_authsize(tfm));
2226 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2227 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2228 cipher_mode, mac_mode,
2229 aeadctx->hmac_ctrl, ivsize >> 1);
2230
2231 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2232 1, dst_size);
2233 }
2234
2235 int aead_ccm_validate_input(unsigned short op_type,
2236 struct aead_request *req,
2237 struct chcr_aead_ctx *aeadctx,
2238 unsigned int sub_type)
2239 {
2240 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2241 if (crypto_ccm_check_iv(req->iv)) {
2242 pr_err("CCM: IV check fails\n");
2243 return -EINVAL;
2244 }
2245 } else {
2246 if (req->assoclen != 16 && req->assoclen != 20) {
2247 pr_err("RFC4309: Invalid AAD length %d\n",
2248 req->assoclen);
2249 return -EINVAL;
2250 }
2251 }
2252 if (aeadctx->enckey_len == 0) {
2253 pr_err("CCM: Encryption key not set\n");
2254 return -EINVAL;
2255 }
2256 return 0;
2257 }
2258
2259 unsigned int fill_aead_req_fields(struct sk_buff *skb,
2260 struct aead_request *req,
2261 struct scatterlist *src,
2262 unsigned int ivsize,
2263 struct chcr_aead_ctx *aeadctx)
2264 {
2265 unsigned int frags = 0;
2266 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2267 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2268 /* b0 and aad length(if available) */
2269
2270 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
2271 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0));
2272 if (req->assoclen) {
2273 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2274 write_sg_to_skb(skb, &frags, req->src,
2275 req->assoclen - 8);
2276 else
2277 write_sg_to_skb(skb, &frags, req->src, req->assoclen);
2278 }
2279 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
2280 if (req->cryptlen)
2281 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2282
2283 return frags;
2284 }
2285
2286 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2287 unsigned short qid,
2288 int size,
2289 unsigned short op_type)
2290 {
2291 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2292 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2293 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2294 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2295 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2296 struct sk_buff *skb = NULL;
2297 struct chcr_wr *chcr_req;
2298 struct cpl_rx_phys_dsgl *phys_cpl;
2299 struct phys_sge_parm sg_param;
2300 struct scatterlist *src;
2301 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
2302 unsigned int dst_size = 0, kctx_len, nents;
2303 unsigned int sub_type;
2304 unsigned int authsize = crypto_aead_authsize(tfm);
2305 int error = -EINVAL, src_nent;
2306 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2307 GFP_ATOMIC;
2308 struct adapter *adap = padap(ctx->dev);
2309
2310 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
2311 authsize);
2312 reqctx->newdstsg = NULL;
2313 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
2314 goto err;
2315 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
2316 if (src_nent < 0)
2317 goto err;
2318
2319 sub_type = get_aead_subtype(tfm);
2320 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
2321 if (req->src != req->dst) {
2322 error = chcr_copy_assoc(req, aeadctx);
2323 if (error) {
2324 pr_err("AAD copy to destination buffer fails\n");
2325 return ERR_PTR(error);
2326 }
2327 }
2328 if (dst_size && is_newsg(req->dst, &nents)) {
2329 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
2330 if (IS_ERR(reqctx->newdstsg))
2331 return ERR_CAST(reqctx->newdstsg);
2332 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2333 reqctx->newdstsg, req->assoclen);
2334 } else {
2335 if (req->src == req->dst)
2336 reqctx->dst = src;
2337 else
2338 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2339 req->dst, req->assoclen);
2340 }
2341 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2342 (op_type ? -authsize : authsize));
2343 if (reqctx->dst_nents < 0) {
2344 pr_err("CCM:Invalid Destination sg entries\n");
2345 error = -EINVAL;
2346 goto err;
2347 }
2348 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
2349 if (error)
2350 goto err;
2351
2352 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2353 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
2354 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2355 if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG,
2356 T6_MAX_AAD_SIZE - 18,
2357 transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
2358 op_type)) {
2359 atomic_inc(&adap->chcr_stats.fallback);
2360 free_new_sg(reqctx->newdstsg);
2361 reqctx->newdstsg = NULL;
2362 return ERR_PTR(chcr_aead_fallback(req, op_type));
2363 }
2364
2365 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2366
2367 if (!skb) {
2368 error = -ENOMEM;
2369 goto err;
2370 }
2371
2372 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2373
2374 chcr_req = __skb_put_zero(skb, transhdr_len);
2375
2376 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
2377
2378 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2379 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2380 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2381 16), aeadctx->key, aeadctx->enckey_len);
2382
2383 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2384 error = ccm_format_packet(req, aeadctx, sub_type, op_type);
2385 if (error)
2386 goto dstmap_fail;
2387
2388 sg_param.nents = reqctx->dst_nents;
2389 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2390 sg_param.qid = qid;
2391 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2392 reqctx->dst, &sg_param);
2393 if (error)
2394 goto dstmap_fail;
2395
2396 skb_set_transport_header(skb, transhdr_len);
2397 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
2398 atomic_inc(&adap->chcr_stats.aead_rqst);
2399 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1,
2400 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2401 reqctx->skb = skb;
2402 skb_get(skb);
2403 return skb;
2404 dstmap_fail:
2405 kfree_skb(skb);
2406 err:
2407 free_new_sg(reqctx->newdstsg);
2408 reqctx->newdstsg = NULL;
2409 return ERR_PTR(error);
2410 }
2411
2412 static struct sk_buff *create_gcm_wr(struct aead_request *req,
2413 unsigned short qid,
2414 int size,
2415 unsigned short op_type)
2416 {
2417 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2418 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2419 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2420 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2421 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2422 struct sk_buff *skb = NULL;
2423 struct chcr_wr *chcr_req;
2424 struct cpl_rx_phys_dsgl *phys_cpl;
2425 struct phys_sge_parm sg_param;
2426 struct scatterlist *src;
2427 unsigned int frags = 0, transhdr_len;
2428 unsigned int ivsize = AES_BLOCK_SIZE;
2429 unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen;
2430 unsigned char tag_offset = 0;
2431 unsigned int authsize = crypto_aead_authsize(tfm);
2432 int error = -EINVAL, src_nent;
2433 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2434 GFP_ATOMIC;
2435 struct adapter *adap = padap(ctx->dev);
2436
2437 reqctx->newdstsg = NULL;
2438 dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
2439 authsize);
2440 /* validate key size */
2441 if (aeadctx->enckey_len == 0)
2442 goto err;
2443
2444 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
2445 goto err;
2446 src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen);
2447 if (src_nent < 0)
2448 goto err;
2449
2450 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen);
2451 if (req->src != req->dst) {
2452 error = chcr_copy_assoc(req, aeadctx);
2453 if (error)
2454 return ERR_PTR(error);
2455 }
2456
2457 if (dst_size && is_newsg(req->dst, &nents)) {
2458 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
2459 if (IS_ERR(reqctx->newdstsg))
2460 return ERR_CAST(reqctx->newdstsg);
2461 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2462 reqctx->newdstsg, assoclen);
2463 } else {
2464 if (req->src == req->dst)
2465 reqctx->dst = src;
2466 else
2467 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2468 req->dst, assoclen);
2469 }
2470
2471 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2472 (op_type ? -authsize : authsize));
2473 if (reqctx->dst_nents < 0) {
2474 pr_err("GCM:Invalid Destination sg entries\n");
2475 error = -EINVAL;
2476 goto err;
2477 }
2478
2479
2480 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2481 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
2482 AEAD_H_SIZE;
2483 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2484 if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG,
2485 T6_MAX_AAD_SIZE,
2486 transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
2487 op_type)) {
2488 atomic_inc(&adap->chcr_stats.fallback);
2489 free_new_sg(reqctx->newdstsg);
2490 reqctx->newdstsg = NULL;
2491 return ERR_PTR(chcr_aead_fallback(req, op_type));
2492 }
2493 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2494 if (!skb) {
2495 error = -ENOMEM;
2496 goto err;
2497 }
2498
2499 /* NIC driver is going to write the sge hdr. */
2500 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2501
2502 chcr_req = __skb_put_zero(skb, transhdr_len);
2503
2504 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
2505 assoclen = req->assoclen - 8;
2506
2507 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2508 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
2509 ctx->dev->rx_channel_id, 2, (ivsize ?
2510 (assoclen + 1) : 0));
2511 chcr_req->sec_cpl.pldlen =
2512 htonl(assoclen + ivsize + req->cryptlen);
2513 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2514 assoclen ? 1 : 0, assoclen,
2515 assoclen + ivsize + 1, 0);
2516 chcr_req->sec_cpl.cipherstop_lo_authinsert =
2517 FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1,
2518 tag_offset, tag_offset);
2519 chcr_req->sec_cpl.seqno_numivs =
2520 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
2521 CHCR_ENCRYPT_OP) ? 1 : 0,
2522 CHCR_SCMD_CIPHER_MODE_AES_GCM,
2523 CHCR_SCMD_AUTH_MODE_GHASH,
2524 aeadctx->hmac_ctrl, ivsize >> 1);
2525 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2526 0, 1, dst_size);
2527 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2528 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2529 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2530 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2531
2532 /* prepare a 16 byte iv */
2533 /* S A L T | IV | 0x00000001 */
2534 if (get_aead_subtype(tfm) ==
2535 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
2536 memcpy(reqctx->iv, aeadctx->salt, 4);
2537 memcpy(reqctx->iv + 4, req->iv, 8);
2538 } else {
2539 memcpy(reqctx->iv, req->iv, 12);
2540 }
2541 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
2542
2543 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2544 sg_param.nents = reqctx->dst_nents;
2545 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2546 sg_param.qid = qid;
2547 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2548 reqctx->dst, &sg_param);
2549 if (error)
2550 goto dstmap_fail;
2551
2552 skb_set_transport_header(skb, transhdr_len);
2553 write_sg_to_skb(skb, &frags, req->src, assoclen);
2554 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
2555 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2556 atomic_inc(&adap->chcr_stats.aead_rqst);
2557 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
2558 sizeof(struct cpl_rx_phys_dsgl) + dst_size,
2559 reqctx->verify);
2560 reqctx->skb = skb;
2561 skb_get(skb);
2562 return skb;
2563
2564 dstmap_fail:
2565 /* ivmap_fail: */
2566 kfree_skb(skb);
2567 err:
2568 free_new_sg(reqctx->newdstsg);
2569 reqctx->newdstsg = NULL;
2570 return ERR_PTR(error);
2571 }
2572
2573
2574
2575 static int chcr_aead_cra_init(struct crypto_aead *tfm)
2576 {
2577 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2578 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2579 struct aead_alg *alg = crypto_aead_alg(tfm);
2580
2581 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
2582 CRYPTO_ALG_NEED_FALLBACK |
2583 CRYPTO_ALG_ASYNC);
2584 if (IS_ERR(aeadctx->sw_cipher))
2585 return PTR_ERR(aeadctx->sw_cipher);
2586 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
2587 sizeof(struct aead_request) +
2588 crypto_aead_reqsize(aeadctx->sw_cipher)));
2589 aeadctx->null = crypto_get_default_null_skcipher();
2590 if (IS_ERR(aeadctx->null))
2591 return PTR_ERR(aeadctx->null);
2592 return chcr_device_init(ctx);
2593 }
2594
2595 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
2596 {
2597 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2598 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2599
2600 crypto_put_default_null_skcipher();
2601 crypto_free_aead(aeadctx->sw_cipher);
2602 }
2603
2604 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
2605 unsigned int authsize)
2606 {
2607 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2608
2609 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
2610 aeadctx->mayverify = VERIFY_HW;
2611 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2612 }
2613 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
2614 unsigned int authsize)
2615 {
2616 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2617 u32 maxauth = crypto_aead_maxauthsize(tfm);
2618
2619 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
2620 * true for sha1. authsize == 12 condition should be before
2621 * authsize == (maxauth >> 1)
2622 */
2623 if (authsize == ICV_4) {
2624 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2625 aeadctx->mayverify = VERIFY_HW;
2626 } else if (authsize == ICV_6) {
2627 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2628 aeadctx->mayverify = VERIFY_HW;
2629 } else if (authsize == ICV_10) {
2630 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2631 aeadctx->mayverify = VERIFY_HW;
2632 } else if (authsize == ICV_12) {
2633 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2634 aeadctx->mayverify = VERIFY_HW;
2635 } else if (authsize == ICV_14) {
2636 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2637 aeadctx->mayverify = VERIFY_HW;
2638 } else if (authsize == (maxauth >> 1)) {
2639 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2640 aeadctx->mayverify = VERIFY_HW;
2641 } else if (authsize == maxauth) {
2642 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2643 aeadctx->mayverify = VERIFY_HW;
2644 } else {
2645 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2646 aeadctx->mayverify = VERIFY_SW;
2647 }
2648 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2649 }
2650
2651
2652 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2653 {
2654 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2655
2656 switch (authsize) {
2657 case ICV_4:
2658 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2659 aeadctx->mayverify = VERIFY_HW;
2660 break;
2661 case ICV_8:
2662 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2663 aeadctx->mayverify = VERIFY_HW;
2664 break;
2665 case ICV_12:
2666 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2667 aeadctx->mayverify = VERIFY_HW;
2668 break;
2669 case ICV_14:
2670 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2671 aeadctx->mayverify = VERIFY_HW;
2672 break;
2673 case ICV_16:
2674 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2675 aeadctx->mayverify = VERIFY_HW;
2676 break;
2677 case ICV_13:
2678 case ICV_15:
2679 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2680 aeadctx->mayverify = VERIFY_SW;
2681 break;
2682 default:
2683
2684 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2685 CRYPTO_TFM_RES_BAD_KEY_LEN);
2686 return -EINVAL;
2687 }
2688 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2689 }
2690
2691 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2692 unsigned int authsize)
2693 {
2694 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2695
2696 switch (authsize) {
2697 case ICV_8:
2698 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2699 aeadctx->mayverify = VERIFY_HW;
2700 break;
2701 case ICV_12:
2702 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2703 aeadctx->mayverify = VERIFY_HW;
2704 break;
2705 case ICV_16:
2706 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2707 aeadctx->mayverify = VERIFY_HW;
2708 break;
2709 default:
2710 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2711 CRYPTO_TFM_RES_BAD_KEY_LEN);
2712 return -EINVAL;
2713 }
2714 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2715 }
2716
2717 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
2718 unsigned int authsize)
2719 {
2720 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2721
2722 switch (authsize) {
2723 case ICV_4:
2724 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2725 aeadctx->mayverify = VERIFY_HW;
2726 break;
2727 case ICV_6:
2728 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2729 aeadctx->mayverify = VERIFY_HW;
2730 break;
2731 case ICV_8:
2732 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2733 aeadctx->mayverify = VERIFY_HW;
2734 break;
2735 case ICV_10:
2736 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2737 aeadctx->mayverify = VERIFY_HW;
2738 break;
2739 case ICV_12:
2740 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2741 aeadctx->mayverify = VERIFY_HW;
2742 break;
2743 case ICV_14:
2744 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2745 aeadctx->mayverify = VERIFY_HW;
2746 break;
2747 case ICV_16:
2748 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2749 aeadctx->mayverify = VERIFY_HW;
2750 break;
2751 default:
2752 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2753 CRYPTO_TFM_RES_BAD_KEY_LEN);
2754 return -EINVAL;
2755 }
2756 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2757 }
2758
2759 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
2760 const u8 *key,
2761 unsigned int keylen)
2762 {
2763 struct chcr_context *ctx = crypto_aead_ctx(aead);
2764 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2765 unsigned char ck_size, mk_size;
2766 int key_ctx_size = 0;
2767
2768 key_ctx_size = sizeof(struct _key_ctx) +
2769 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2;
2770 if (keylen == AES_KEYSIZE_128) {
2771 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2772 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2773 } else if (keylen == AES_KEYSIZE_192) {
2774 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2775 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
2776 } else if (keylen == AES_KEYSIZE_256) {
2777 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2778 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2779 } else {
2780 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2781 CRYPTO_TFM_RES_BAD_KEY_LEN);
2782 aeadctx->enckey_len = 0;
2783 return -EINVAL;
2784 }
2785 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
2786 key_ctx_size >> 4);
2787 memcpy(aeadctx->key, key, keylen);
2788 aeadctx->enckey_len = keylen;
2789
2790 return 0;
2791 }
2792
2793 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
2794 const u8 *key,
2795 unsigned int keylen)
2796 {
2797 struct chcr_context *ctx = crypto_aead_ctx(aead);
2798 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2799 int error;
2800
2801 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2802 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2803 CRYPTO_TFM_REQ_MASK);
2804 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2805 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2806 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2807 CRYPTO_TFM_RES_MASK);
2808 if (error)
2809 return error;
2810 return chcr_ccm_common_setkey(aead, key, keylen);
2811 }
2812
2813 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
2814 unsigned int keylen)
2815 {
2816 struct chcr_context *ctx = crypto_aead_ctx(aead);
2817 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2818 int error;
2819
2820 if (keylen < 3) {
2821 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2822 CRYPTO_TFM_RES_BAD_KEY_LEN);
2823 aeadctx->enckey_len = 0;
2824 return -EINVAL;
2825 }
2826 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2827 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2828 CRYPTO_TFM_REQ_MASK);
2829 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2830 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2831 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2832 CRYPTO_TFM_RES_MASK);
2833 if (error)
2834 return error;
2835 keylen -= 3;
2836 memcpy(aeadctx->salt, key + keylen, 3);
2837 return chcr_ccm_common_setkey(aead, key, keylen);
2838 }
2839
2840 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2841 unsigned int keylen)
2842 {
2843 struct chcr_context *ctx = crypto_aead_ctx(aead);
2844 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2845 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
2846 struct crypto_cipher *cipher;
2847 unsigned int ck_size;
2848 int ret = 0, key_ctx_size = 0;
2849
2850 aeadctx->enckey_len = 0;
2851 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2852 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
2853 & CRYPTO_TFM_REQ_MASK);
2854 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2855 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2856 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2857 CRYPTO_TFM_RES_MASK);
2858 if (ret)
2859 goto out;
2860
2861 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
2862 keylen > 3) {
2863 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
2864 memcpy(aeadctx->salt, key + keylen, 4);
2865 }
2866 if (keylen == AES_KEYSIZE_128) {
2867 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2868 } else if (keylen == AES_KEYSIZE_192) {
2869 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2870 } else if (keylen == AES_KEYSIZE_256) {
2871 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2872 } else {
2873 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2874 CRYPTO_TFM_RES_BAD_KEY_LEN);
2875 pr_err("GCM: Invalid key length %d\n", keylen);
2876 ret = -EINVAL;
2877 goto out;
2878 }
2879
2880 memcpy(aeadctx->key, key, keylen);
2881 aeadctx->enckey_len = keylen;
2882 key_ctx_size = sizeof(struct _key_ctx) +
2883 ((DIV_ROUND_UP(keylen, 16)) << 4) +
2884 AEAD_H_SIZE;
2885 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2886 CHCR_KEYCTX_MAC_KEY_SIZE_128,
2887 0, 0,
2888 key_ctx_size >> 4);
2889 /* Calculate the H = CIPH(K, 0 repeated 16 times).
2890 * It will go in key context
2891 */
2892 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
2893 if (IS_ERR(cipher)) {
2894 aeadctx->enckey_len = 0;
2895 ret = -ENOMEM;
2896 goto out;
2897 }
2898
2899 ret = crypto_cipher_setkey(cipher, key, keylen);
2900 if (ret) {
2901 aeadctx->enckey_len = 0;
2902 goto out1;
2903 }
2904 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
2905 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2906
2907 out1:
2908 crypto_free_cipher(cipher);
2909 out:
2910 return ret;
2911 }
2912
2913 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
2914 unsigned int keylen)
2915 {
2916 struct chcr_context *ctx = crypto_aead_ctx(authenc);
2917 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2918 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2919 /* it contains auth and cipher key both*/
2920 struct crypto_authenc_keys keys;
2921 unsigned int bs;
2922 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
2923 int err = 0, i, key_ctx_len = 0;
2924 unsigned char ck_size = 0;
2925 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
2926 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
2927 struct algo_param param;
2928 int align;
2929 u8 *o_ptr = NULL;
2930
2931 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2932 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
2933 & CRYPTO_TFM_REQ_MASK);
2934 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2935 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
2936 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
2937 & CRYPTO_TFM_RES_MASK);
2938 if (err)
2939 goto out;
2940
2941 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2942 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2943 goto out;
2944 }
2945
2946 if (get_alg_config(&param, max_authsize)) {
2947 pr_err("chcr : Unsupported digest size\n");
2948 goto out;
2949 }
2950 if (keys.enckeylen == AES_KEYSIZE_128) {
2951 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2952 } else if (keys.enckeylen == AES_KEYSIZE_192) {
2953 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2954 } else if (keys.enckeylen == AES_KEYSIZE_256) {
2955 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2956 } else {
2957 pr_err("chcr : Unsupported cipher key\n");
2958 goto out;
2959 }
2960
2961 /* Copy only encryption key. We use authkey to generate h(ipad) and
2962 * h(opad) so authkey is not needed again. authkeylen size have the
2963 * size of the hash digest size.
2964 */
2965 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2966 aeadctx->enckey_len = keys.enckeylen;
2967 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2968 aeadctx->enckey_len << 3);
2969
2970 base_hash = chcr_alloc_shash(max_authsize);
2971 if (IS_ERR(base_hash)) {
2972 pr_err("chcr : Base driver cannot be loaded\n");
2973 aeadctx->enckey_len = 0;
2974 return -EINVAL;
2975 }
2976 {
2977 SHASH_DESC_ON_STACK(shash, base_hash);
2978 shash->tfm = base_hash;
2979 shash->flags = crypto_shash_get_flags(base_hash);
2980 bs = crypto_shash_blocksize(base_hash);
2981 align = KEYCTX_ALIGN_PAD(max_authsize);
2982 o_ptr = actx->h_iopad + param.result_size + align;
2983
2984 if (keys.authkeylen > bs) {
2985 err = crypto_shash_digest(shash, keys.authkey,
2986 keys.authkeylen,
2987 o_ptr);
2988 if (err) {
2989 pr_err("chcr : Base driver cannot be loaded\n");
2990 goto out;
2991 }
2992 keys.authkeylen = max_authsize;
2993 } else
2994 memcpy(o_ptr, keys.authkey, keys.authkeylen);
2995
2996 /* Compute the ipad-digest*/
2997 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2998 memcpy(pad, o_ptr, keys.authkeylen);
2999 for (i = 0; i < bs >> 2; i++)
3000 *((unsigned int *)pad + i) ^= IPAD_DATA;
3001
3002 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3003 max_authsize))
3004 goto out;
3005 /* Compute the opad-digest */
3006 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3007 memcpy(pad, o_ptr, keys.authkeylen);
3008 for (i = 0; i < bs >> 2; i++)
3009 *((unsigned int *)pad + i) ^= OPAD_DATA;
3010
3011 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3012 goto out;
3013
3014 /* convert the ipad and opad digest to network order */
3015 chcr_change_order(actx->h_iopad, param.result_size);
3016 chcr_change_order(o_ptr, param.result_size);
3017 key_ctx_len = sizeof(struct _key_ctx) +
3018 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
3019 (param.result_size + align) * 2;
3020 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3021 0, 1, key_ctx_len >> 4);
3022 actx->auth_mode = param.auth_mode;
3023 chcr_free_shash(base_hash);
3024
3025 return 0;
3026 }
3027 out:
3028 aeadctx->enckey_len = 0;
3029 if (!IS_ERR(base_hash))
3030 chcr_free_shash(base_hash);
3031 return -EINVAL;
3032 }
3033
3034 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3035 const u8 *key, unsigned int keylen)
3036 {
3037 struct chcr_context *ctx = crypto_aead_ctx(authenc);
3038 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3039 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3040 struct crypto_authenc_keys keys;
3041 int err;
3042 /* it contains auth and cipher key both*/
3043 int key_ctx_len = 0;
3044 unsigned char ck_size = 0;
3045
3046 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3047 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3048 & CRYPTO_TFM_REQ_MASK);
3049 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3050 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3051 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3052 & CRYPTO_TFM_RES_MASK);
3053 if (err)
3054 goto out;
3055
3056 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3057 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3058 goto out;
3059 }
3060 if (keys.enckeylen == AES_KEYSIZE_128) {
3061 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3062 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3063 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3064 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3065 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3066 } else {
3067 pr_err("chcr : Unsupported cipher key\n");
3068 goto out;
3069 }
3070 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3071 aeadctx->enckey_len = keys.enckeylen;
3072 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3073 aeadctx->enckey_len << 3);
3074 key_ctx_len = sizeof(struct _key_ctx)
3075 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
3076
3077 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3078 0, key_ctx_len >> 4);
3079 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3080 return 0;
3081 out:
3082 aeadctx->enckey_len = 0;
3083 return -EINVAL;
3084 }
3085 static int chcr_aead_encrypt(struct aead_request *req)
3086 {
3087 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3088 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3089
3090 reqctx->verify = VERIFY_HW;
3091
3092 switch (get_aead_subtype(tfm)) {
3093 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3094 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3095 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3096 create_authenc_wr);
3097 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3098 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3099 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3100 create_aead_ccm_wr);
3101 default:
3102 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3103 create_gcm_wr);
3104 }
3105 }
3106
3107 static int chcr_aead_decrypt(struct aead_request *req)
3108 {
3109 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3110 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
3111 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3112 int size;
3113
3114 if (aeadctx->mayverify == VERIFY_SW) {
3115 size = crypto_aead_maxauthsize(tfm);
3116 reqctx->verify = VERIFY_SW;
3117 } else {
3118 size = 0;
3119 reqctx->verify = VERIFY_HW;
3120 }
3121
3122 switch (get_aead_subtype(tfm)) {
3123 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3124 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3125 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3126 create_authenc_wr);
3127 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3128 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3129 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3130 create_aead_ccm_wr);
3131 default:
3132 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3133 create_gcm_wr);
3134 }
3135 }
3136
3137 static int chcr_aead_op(struct aead_request *req,
3138 unsigned short op_type,
3139 int size,
3140 create_wr_t create_wr_fn)
3141 {
3142 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3143 struct chcr_context *ctx = crypto_aead_ctx(tfm);
3144 struct uld_ctx *u_ctx;
3145 struct sk_buff *skb;
3146
3147 if (!ctx->dev) {
3148 pr_err("chcr : %s : No crypto device.\n", __func__);
3149 return -ENXIO;
3150 }
3151 u_ctx = ULD_CTX(ctx);
3152 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3153 ctx->tx_qidx)) {
3154 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3155 return -EBUSY;
3156 }
3157
3158 /* Form a WR from req */
3159 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size,
3160 op_type);
3161
3162 if (IS_ERR(skb) || !skb)
3163 return PTR_ERR(skb);
3164
3165 skb->dev = u_ctx->lldi.ports[0];
3166 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
3167 chcr_send_wr(skb);
3168 return -EINPROGRESS;
3169 }
3170 static struct chcr_alg_template driver_algs[] = {
3171 /* AES-CBC */
3172 {
3173 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3174 .is_registered = 0,
3175 .alg.crypto = {
3176 .cra_name = "cbc(aes)",
3177 .cra_driver_name = "cbc-aes-chcr",
3178 .cra_blocksize = AES_BLOCK_SIZE,
3179 .cra_init = chcr_cra_init,
3180 .cra_exit = chcr_cra_exit,
3181 .cra_u.ablkcipher = {
3182 .min_keysize = AES_MIN_KEY_SIZE,
3183 .max_keysize = AES_MAX_KEY_SIZE,
3184 .ivsize = AES_BLOCK_SIZE,
3185 .setkey = chcr_aes_cbc_setkey,
3186 .encrypt = chcr_aes_encrypt,
3187 .decrypt = chcr_aes_decrypt,
3188 }
3189 }
3190 },
3191 {
3192 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3193 .is_registered = 0,
3194 .alg.crypto = {
3195 .cra_name = "xts(aes)",
3196 .cra_driver_name = "xts-aes-chcr",
3197 .cra_blocksize = AES_BLOCK_SIZE,
3198 .cra_init = chcr_cra_init,
3199 .cra_exit = NULL,
3200 .cra_u .ablkcipher = {
3201 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3202 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3203 .ivsize = AES_BLOCK_SIZE,
3204 .setkey = chcr_aes_xts_setkey,
3205 .encrypt = chcr_aes_encrypt,
3206 .decrypt = chcr_aes_decrypt,
3207 }
3208 }
3209 },
3210 {
3211 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3212 .is_registered = 0,
3213 .alg.crypto = {
3214 .cra_name = "ctr(aes)",
3215 .cra_driver_name = "ctr-aes-chcr",
3216 .cra_blocksize = 1,
3217 .cra_init = chcr_cra_init,
3218 .cra_exit = chcr_cra_exit,
3219 .cra_u.ablkcipher = {
3220 .min_keysize = AES_MIN_KEY_SIZE,
3221 .max_keysize = AES_MAX_KEY_SIZE,
3222 .ivsize = AES_BLOCK_SIZE,
3223 .setkey = chcr_aes_ctr_setkey,
3224 .encrypt = chcr_aes_encrypt,
3225 .decrypt = chcr_aes_decrypt,
3226 }
3227 }
3228 },
3229 {
3230 .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
3231 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3232 .is_registered = 0,
3233 .alg.crypto = {
3234 .cra_name = "rfc3686(ctr(aes))",
3235 .cra_driver_name = "rfc3686-ctr-aes-chcr",
3236 .cra_blocksize = 1,
3237 .cra_init = chcr_rfc3686_init,
3238 .cra_exit = chcr_cra_exit,
3239 .cra_u.ablkcipher = {
3240 .min_keysize = AES_MIN_KEY_SIZE +
3241 CTR_RFC3686_NONCE_SIZE,
3242 .max_keysize = AES_MAX_KEY_SIZE +
3243 CTR_RFC3686_NONCE_SIZE,
3244 .ivsize = CTR_RFC3686_IV_SIZE,
3245 .setkey = chcr_aes_rfc3686_setkey,
3246 .encrypt = chcr_aes_encrypt,
3247 .decrypt = chcr_aes_decrypt,
3248 .geniv = "seqiv",
3249 }
3250 }
3251 },
3252 /* SHA */
3253 {
3254 .type = CRYPTO_ALG_TYPE_AHASH,
3255 .is_registered = 0,
3256 .alg.hash = {
3257 .halg.digestsize = SHA1_DIGEST_SIZE,
3258 .halg.base = {
3259 .cra_name = "sha1",
3260 .cra_driver_name = "sha1-chcr",
3261 .cra_blocksize = SHA1_BLOCK_SIZE,
3262 }
3263 }
3264 },
3265 {
3266 .type = CRYPTO_ALG_TYPE_AHASH,
3267 .is_registered = 0,
3268 .alg.hash = {
3269 .halg.digestsize = SHA256_DIGEST_SIZE,
3270 .halg.base = {
3271 .cra_name = "sha256",
3272 .cra_driver_name = "sha256-chcr",
3273 .cra_blocksize = SHA256_BLOCK_SIZE,
3274 }
3275 }
3276 },
3277 {
3278 .type = CRYPTO_ALG_TYPE_AHASH,
3279 .is_registered = 0,
3280 .alg.hash = {
3281 .halg.digestsize = SHA224_DIGEST_SIZE,
3282 .halg.base = {
3283 .cra_name = "sha224",
3284 .cra_driver_name = "sha224-chcr",
3285 .cra_blocksize = SHA224_BLOCK_SIZE,
3286 }
3287 }
3288 },
3289 {
3290 .type = CRYPTO_ALG_TYPE_AHASH,
3291 .is_registered = 0,
3292 .alg.hash = {
3293 .halg.digestsize = SHA384_DIGEST_SIZE,
3294 .halg.base = {
3295 .cra_name = "sha384",
3296 .cra_driver_name = "sha384-chcr",
3297 .cra_blocksize = SHA384_BLOCK_SIZE,
3298 }
3299 }
3300 },
3301 {
3302 .type = CRYPTO_ALG_TYPE_AHASH,
3303 .is_registered = 0,
3304 .alg.hash = {
3305 .halg.digestsize = SHA512_DIGEST_SIZE,
3306 .halg.base = {
3307 .cra_name = "sha512",
3308 .cra_driver_name = "sha512-chcr",
3309 .cra_blocksize = SHA512_BLOCK_SIZE,
3310 }
3311 }
3312 },
3313 /* HMAC */
3314 {
3315 .type = CRYPTO_ALG_TYPE_HMAC,
3316 .is_registered = 0,
3317 .alg.hash = {
3318 .halg.digestsize = SHA1_DIGEST_SIZE,
3319 .halg.base = {
3320 .cra_name = "hmac(sha1)",
3321 .cra_driver_name = "hmac-sha1-chcr",
3322 .cra_blocksize = SHA1_BLOCK_SIZE,
3323 }
3324 }
3325 },
3326 {
3327 .type = CRYPTO_ALG_TYPE_HMAC,
3328 .is_registered = 0,
3329 .alg.hash = {
3330 .halg.digestsize = SHA224_DIGEST_SIZE,
3331 .halg.base = {
3332 .cra_name = "hmac(sha224)",
3333 .cra_driver_name = "hmac-sha224-chcr",
3334 .cra_blocksize = SHA224_BLOCK_SIZE,
3335 }
3336 }
3337 },
3338 {
3339 .type = CRYPTO_ALG_TYPE_HMAC,
3340 .is_registered = 0,
3341 .alg.hash = {
3342 .halg.digestsize = SHA256_DIGEST_SIZE,
3343 .halg.base = {
3344 .cra_name = "hmac(sha256)",
3345 .cra_driver_name = "hmac-sha256-chcr",
3346 .cra_blocksize = SHA256_BLOCK_SIZE,
3347 }
3348 }
3349 },
3350 {
3351 .type = CRYPTO_ALG_TYPE_HMAC,
3352 .is_registered = 0,
3353 .alg.hash = {
3354 .halg.digestsize = SHA384_DIGEST_SIZE,
3355 .halg.base = {
3356 .cra_name = "hmac(sha384)",
3357 .cra_driver_name = "hmac-sha384-chcr",
3358 .cra_blocksize = SHA384_BLOCK_SIZE,
3359 }
3360 }
3361 },
3362 {
3363 .type = CRYPTO_ALG_TYPE_HMAC,
3364 .is_registered = 0,
3365 .alg.hash = {
3366 .halg.digestsize = SHA512_DIGEST_SIZE,
3367 .halg.base = {
3368 .cra_name = "hmac(sha512)",
3369 .cra_driver_name = "hmac-sha512-chcr",
3370 .cra_blocksize = SHA512_BLOCK_SIZE,
3371 }
3372 }
3373 },
3374 /* Add AEAD Algorithms */
3375 {
3376 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3377 .is_registered = 0,
3378 .alg.aead = {
3379 .base = {
3380 .cra_name = "gcm(aes)",
3381 .cra_driver_name = "gcm-aes-chcr",
3382 .cra_blocksize = 1,
3383 .cra_priority = CHCR_AEAD_PRIORITY,
3384 .cra_ctxsize = sizeof(struct chcr_context) +
3385 sizeof(struct chcr_aead_ctx) +
3386 sizeof(struct chcr_gcm_ctx),
3387 },
3388 .ivsize = 12,
3389 .maxauthsize = GHASH_DIGEST_SIZE,
3390 .setkey = chcr_gcm_setkey,
3391 .setauthsize = chcr_gcm_setauthsize,
3392 }
3393 },
3394 {
3395 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3396 .is_registered = 0,
3397 .alg.aead = {
3398 .base = {
3399 .cra_name = "rfc4106(gcm(aes))",
3400 .cra_driver_name = "rfc4106-gcm-aes-chcr",
3401 .cra_blocksize = 1,
3402 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3403 .cra_ctxsize = sizeof(struct chcr_context) +
3404 sizeof(struct chcr_aead_ctx) +
3405 sizeof(struct chcr_gcm_ctx),
3406
3407 },
3408 .ivsize = 8,
3409 .maxauthsize = GHASH_DIGEST_SIZE,
3410 .setkey = chcr_gcm_setkey,
3411 .setauthsize = chcr_4106_4309_setauthsize,
3412 }
3413 },
3414 {
3415 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3416 .is_registered = 0,
3417 .alg.aead = {
3418 .base = {
3419 .cra_name = "ccm(aes)",
3420 .cra_driver_name = "ccm-aes-chcr",
3421 .cra_blocksize = 1,
3422 .cra_priority = CHCR_AEAD_PRIORITY,
3423 .cra_ctxsize = sizeof(struct chcr_context) +
3424 sizeof(struct chcr_aead_ctx),
3425
3426 },
3427 .ivsize = AES_BLOCK_SIZE,
3428 .maxauthsize = GHASH_DIGEST_SIZE,
3429 .setkey = chcr_aead_ccm_setkey,
3430 .setauthsize = chcr_ccm_setauthsize,
3431 }
3432 },
3433 {
3434 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3435 .is_registered = 0,
3436 .alg.aead = {
3437 .base = {
3438 .cra_name = "rfc4309(ccm(aes))",
3439 .cra_driver_name = "rfc4309-ccm-aes-chcr",
3440 .cra_blocksize = 1,
3441 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3442 .cra_ctxsize = sizeof(struct chcr_context) +
3443 sizeof(struct chcr_aead_ctx),
3444
3445 },
3446 .ivsize = 8,
3447 .maxauthsize = GHASH_DIGEST_SIZE,
3448 .setkey = chcr_aead_rfc4309_setkey,
3449 .setauthsize = chcr_4106_4309_setauthsize,
3450 }
3451 },
3452 {
3453 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3454 .is_registered = 0,
3455 .alg.aead = {
3456 .base = {
3457 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3458 .cra_driver_name =
3459 "authenc-hmac-sha1-cbc-aes-chcr",
3460 .cra_blocksize = AES_BLOCK_SIZE,
3461 .cra_priority = CHCR_AEAD_PRIORITY,
3462 .cra_ctxsize = sizeof(struct chcr_context) +
3463 sizeof(struct chcr_aead_ctx) +
3464 sizeof(struct chcr_authenc_ctx),
3465
3466 },
3467 .ivsize = AES_BLOCK_SIZE,
3468 .maxauthsize = SHA1_DIGEST_SIZE,
3469 .setkey = chcr_authenc_setkey,
3470 .setauthsize = chcr_authenc_setauthsize,
3471 }
3472 },
3473 {
3474 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3475 .is_registered = 0,
3476 .alg.aead = {
3477 .base = {
3478
3479 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3480 .cra_driver_name =
3481 "authenc-hmac-sha256-cbc-aes-chcr",
3482 .cra_blocksize = AES_BLOCK_SIZE,
3483 .cra_priority = CHCR_AEAD_PRIORITY,
3484 .cra_ctxsize = sizeof(struct chcr_context) +
3485 sizeof(struct chcr_aead_ctx) +
3486 sizeof(struct chcr_authenc_ctx),
3487
3488 },
3489 .ivsize = AES_BLOCK_SIZE,
3490 .maxauthsize = SHA256_DIGEST_SIZE,
3491 .setkey = chcr_authenc_setkey,
3492 .setauthsize = chcr_authenc_setauthsize,
3493 }
3494 },
3495 {
3496 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3497 .is_registered = 0,
3498 .alg.aead = {
3499 .base = {
3500 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3501 .cra_driver_name =
3502 "authenc-hmac-sha224-cbc-aes-chcr",
3503 .cra_blocksize = AES_BLOCK_SIZE,
3504 .cra_priority = CHCR_AEAD_PRIORITY,
3505 .cra_ctxsize = sizeof(struct chcr_context) +
3506 sizeof(struct chcr_aead_ctx) +
3507 sizeof(struct chcr_authenc_ctx),
3508 },
3509 .ivsize = AES_BLOCK_SIZE,
3510 .maxauthsize = SHA224_DIGEST_SIZE,
3511 .setkey = chcr_authenc_setkey,
3512 .setauthsize = chcr_authenc_setauthsize,
3513 }
3514 },
3515 {
3516 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3517 .is_registered = 0,
3518 .alg.aead = {
3519 .base = {
3520 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3521 .cra_driver_name =
3522 "authenc-hmac-sha384-cbc-aes-chcr",
3523 .cra_blocksize = AES_BLOCK_SIZE,
3524 .cra_priority = CHCR_AEAD_PRIORITY,
3525 .cra_ctxsize = sizeof(struct chcr_context) +
3526 sizeof(struct chcr_aead_ctx) +
3527 sizeof(struct chcr_authenc_ctx),
3528
3529 },
3530 .ivsize = AES_BLOCK_SIZE,
3531 .maxauthsize = SHA384_DIGEST_SIZE,
3532 .setkey = chcr_authenc_setkey,
3533 .setauthsize = chcr_authenc_setauthsize,
3534 }
3535 },
3536 {
3537 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3538 .is_registered = 0,
3539 .alg.aead = {
3540 .base = {
3541 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3542 .cra_driver_name =
3543 "authenc-hmac-sha512-cbc-aes-chcr",
3544 .cra_blocksize = AES_BLOCK_SIZE,
3545 .cra_priority = CHCR_AEAD_PRIORITY,
3546 .cra_ctxsize = sizeof(struct chcr_context) +
3547 sizeof(struct chcr_aead_ctx) +
3548 sizeof(struct chcr_authenc_ctx),
3549
3550 },
3551 .ivsize = AES_BLOCK_SIZE,
3552 .maxauthsize = SHA512_DIGEST_SIZE,
3553 .setkey = chcr_authenc_setkey,
3554 .setauthsize = chcr_authenc_setauthsize,
3555 }
3556 },
3557 {
3558 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
3559 .is_registered = 0,
3560 .alg.aead = {
3561 .base = {
3562 .cra_name = "authenc(digest_null,cbc(aes))",
3563 .cra_driver_name =
3564 "authenc-digest_null-cbc-aes-chcr",
3565 .cra_blocksize = AES_BLOCK_SIZE,
3566 .cra_priority = CHCR_AEAD_PRIORITY,
3567 .cra_ctxsize = sizeof(struct chcr_context) +
3568 sizeof(struct chcr_aead_ctx) +
3569 sizeof(struct chcr_authenc_ctx),
3570
3571 },
3572 .ivsize = AES_BLOCK_SIZE,
3573 .maxauthsize = 0,
3574 .setkey = chcr_aead_digest_null_setkey,
3575 .setauthsize = chcr_authenc_null_setauthsize,
3576 }
3577 },
3578 };
3579
3580 /*
3581 * chcr_unregister_alg - Deregister crypto algorithms with
3582 * kernel framework.
3583 */
3584 static int chcr_unregister_alg(void)
3585 {
3586 int i;
3587
3588 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3589 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3590 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3591 if (driver_algs[i].is_registered)
3592 crypto_unregister_alg(
3593 &driver_algs[i].alg.crypto);
3594 break;
3595 case CRYPTO_ALG_TYPE_AEAD:
3596 if (driver_algs[i].is_registered)
3597 crypto_unregister_aead(
3598 &driver_algs[i].alg.aead);
3599 break;
3600 case CRYPTO_ALG_TYPE_AHASH:
3601 if (driver_algs[i].is_registered)
3602 crypto_unregister_ahash(
3603 &driver_algs[i].alg.hash);
3604 break;
3605 }
3606 driver_algs[i].is_registered = 0;
3607 }
3608 return 0;
3609 }
3610
3611 #define SZ_AHASH_CTX sizeof(struct chcr_context)
3612 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
3613 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
3614 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
3615
3616 /*
3617 * chcr_register_alg - Register crypto algorithms with kernel framework.
3618 */
3619 static int chcr_register_alg(void)
3620 {
3621 struct crypto_alg ai;
3622 struct ahash_alg *a_hash;
3623 int err = 0, i;
3624 char *name = NULL;
3625
3626 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3627 if (driver_algs[i].is_registered)
3628 continue;
3629 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3630 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3631 driver_algs[i].alg.crypto.cra_priority =
3632 CHCR_CRA_PRIORITY;
3633 driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
3634 driver_algs[i].alg.crypto.cra_flags =
3635 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
3636 CRYPTO_ALG_NEED_FALLBACK;
3637 driver_algs[i].alg.crypto.cra_ctxsize =
3638 sizeof(struct chcr_context) +
3639 sizeof(struct ablk_ctx);
3640 driver_algs[i].alg.crypto.cra_alignmask = 0;
3641 driver_algs[i].alg.crypto.cra_type =
3642 &crypto_ablkcipher_type;
3643 err = crypto_register_alg(&driver_algs[i].alg.crypto);
3644 name = driver_algs[i].alg.crypto.cra_driver_name;
3645 break;
3646 case CRYPTO_ALG_TYPE_AEAD:
3647 driver_algs[i].alg.aead.base.cra_flags =
3648 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
3649 CRYPTO_ALG_NEED_FALLBACK;
3650 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
3651 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
3652 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
3653 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
3654 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
3655 err = crypto_register_aead(&driver_algs[i].alg.aead);
3656 name = driver_algs[i].alg.aead.base.cra_driver_name;
3657 break;
3658 case CRYPTO_ALG_TYPE_AHASH:
3659 a_hash = &driver_algs[i].alg.hash;
3660 a_hash->update = chcr_ahash_update;
3661 a_hash->final = chcr_ahash_final;
3662 a_hash->finup = chcr_ahash_finup;
3663 a_hash->digest = chcr_ahash_digest;
3664 a_hash->export = chcr_ahash_export;
3665 a_hash->import = chcr_ahash_import;
3666 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
3667 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
3668 a_hash->halg.base.cra_module = THIS_MODULE;
3669 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
3670 a_hash->halg.base.cra_alignmask = 0;
3671 a_hash->halg.base.cra_exit = NULL;
3672 a_hash->halg.base.cra_type = &crypto_ahash_type;
3673
3674 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
3675 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
3676 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
3677 a_hash->init = chcr_hmac_init;
3678 a_hash->setkey = chcr_ahash_setkey;
3679 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
3680 } else {
3681 a_hash->init = chcr_sha_init;
3682 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
3683 a_hash->halg.base.cra_init = chcr_sha_cra_init;
3684 }
3685 err = crypto_register_ahash(&driver_algs[i].alg.hash);
3686 ai = driver_algs[i].alg.hash.halg.base;
3687 name = ai.cra_driver_name;
3688 break;
3689 }
3690 if (err) {
3691 pr_err("chcr : %s : Algorithm registration failed\n",
3692 name);
3693 goto register_err;
3694 } else {
3695 driver_algs[i].is_registered = 1;
3696 }
3697 }
3698 return 0;
3699
3700 register_err:
3701 chcr_unregister_alg();
3702 return err;
3703 }
3704
3705 /*
3706 * start_crypto - Register the crypto algorithms.
3707 * This should called once when the first device comesup. After this
3708 * kernel will start calling driver APIs for crypto operations.
3709 */
3710 int start_crypto(void)
3711 {
3712 return chcr_register_alg();
3713 }
3714
3715 /*
3716 * stop_crypto - Deregister all the crypto algorithms with kernel.
3717 * This should be called once when the last device goes down. After this
3718 * kernel will not call the driver API for crypto operations.
3719 */
3720 int stop_crypto(void)
3721 {
3722 chcr_unregister_alg();
3723 return 0;
3724 }
Linux with added FPC-III support