?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-v6.13-2' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / drivers / crypto / marvell / octeontx2 / otx2_cptvf_algs.c
blob7eb0bc13994d90e905726bc7610cee6a87e20dfa
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Marvell. */
3
4 #include <crypto/aes.h>
5 #include <crypto/authenc.h>
6 #include <crypto/cryptd.h>
7 #include <crypto/des.h>
8 #include <crypto/internal/aead.h>
9 #include <crypto/sha1.h>
10 #include <crypto/sha2.h>
11 #include <crypto/xts.h>
12 #include <crypto/gcm.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/sort.h>
15 #include <linux/module.h>
16 #include "otx2_cptvf.h"
17 #include "otx2_cptvf_algs.h"
18 #include "otx2_cpt_reqmgr.h"
19 #include "cn10k_cpt.h"
20
21 /* Size of salt in AES GCM mode */
22 #define AES_GCM_SALT_SIZE 4
23 /* Size of IV in AES GCM mode */
24 #define AES_GCM_IV_SIZE 8
25 /* Size of ICV (Integrity Check Value) in AES GCM mode */
26 #define AES_GCM_ICV_SIZE 16
27 /* Offset of IV in AES GCM mode */
28 #define AES_GCM_IV_OFFSET 8
29 #define CONTROL_WORD_LEN 8
30 #define KEY2_OFFSET 48
31 #define DMA_MODE_FLAG(dma_mode) \
32 (((dma_mode) == OTX2_CPT_DMA_MODE_SG) ? (1 << 7) : 0)
33
34 /* Truncated SHA digest size */
35 #define SHA1_TRUNC_DIGEST_SIZE 12
36 #define SHA256_TRUNC_DIGEST_SIZE 16
37 #define SHA384_TRUNC_DIGEST_SIZE 24
38 #define SHA512_TRUNC_DIGEST_SIZE 32
39
40 static DEFINE_MUTEX(mutex);
41 static int is_crypto_registered;
42
43 struct cpt_device_desc {
44 struct pci_dev *dev;
45 int num_queues;
46 };
47
48 struct cpt_device_table {
49 atomic_t count;
50 struct cpt_device_desc desc[OTX2_CPT_MAX_LFS_NUM];
51 };
52
53 static struct cpt_device_table se_devices = {
54 .count = ATOMIC_INIT(0)
55 };
56
57 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg);
58
59 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
60 {
61 int count;
62
63 count = atomic_read(&se_devices.count);
64 if (count < 1)
65 return -ENODEV;
66
67 *cpu_num = get_cpu();
68 /*
69 * On OcteonTX2 platform CPT instruction queue is bound to each
70 * local function LF, in turn LFs can be attached to PF
71 * or VF therefore we always use first device. We get maximum
72 * performance if one CPT queue is available for each cpu
73 * otherwise CPT queues need to be shared between cpus.
74 */
75 if (*cpu_num >= se_devices.desc[0].num_queues)
76 *cpu_num %= se_devices.desc[0].num_queues;
77 *pdev = se_devices.desc[0].dev;
78
79 put_cpu();
80
81 return 0;
82 }
83
84 static inline int validate_hmac_cipher_null(struct otx2_cpt_req_info *cpt_req)
85 {
86 struct otx2_cpt_req_ctx *rctx;
87 struct aead_request *req;
88 struct crypto_aead *tfm;
89
90 req = container_of(cpt_req->areq, struct aead_request, base);
91 tfm = crypto_aead_reqtfm(req);
92 rctx = aead_request_ctx_dma(req);
93 if (memcmp(rctx->fctx.hmac.s.hmac_calc,
94 rctx->fctx.hmac.s.hmac_recv,
95 crypto_aead_authsize(tfm)) != 0)
96 return -EBADMSG;
97
98 return 0;
99 }
100
101 static void otx2_cpt_aead_callback(int status, void *arg1, void *arg2)
102 {
103 struct otx2_cpt_inst_info *inst_info = arg2;
104 struct crypto_async_request *areq = arg1;
105 struct otx2_cpt_req_info *cpt_req;
106 struct pci_dev *pdev;
107
108 if (inst_info) {
109 cpt_req = inst_info->req;
110 if (!status) {
111 /*
112 * When selected cipher is NULL we need to manually
113 * verify whether calculated hmac value matches
114 * received hmac value
115 */
116 if (cpt_req->req_type ==
117 OTX2_CPT_AEAD_ENC_DEC_NULL_REQ &&
118 !cpt_req->is_enc)
119 status = validate_hmac_cipher_null(cpt_req);
120 }
121 pdev = inst_info->pdev;
122 otx2_cpt_info_destroy(pdev, inst_info);
123 }
124 if (areq)
125 crypto_request_complete(areq, status);
126 }
127
128 static void output_iv_copyback(struct crypto_async_request *areq)
129 {
130 struct otx2_cpt_req_info *req_info;
131 struct otx2_cpt_req_ctx *rctx;
132 struct skcipher_request *sreq;
133 struct crypto_skcipher *stfm;
134 struct otx2_cpt_enc_ctx *ctx;
135 u32 start, ivsize;
136
137 sreq = container_of(areq, struct skcipher_request, base);
138 stfm = crypto_skcipher_reqtfm(sreq);
139 ctx = crypto_skcipher_ctx(stfm);
140 if (ctx->cipher_type == OTX2_CPT_AES_CBC ||
141 ctx->cipher_type == OTX2_CPT_DES3_CBC) {
142 rctx = skcipher_request_ctx_dma(sreq);
143 req_info = &rctx->cpt_req;
144 ivsize = crypto_skcipher_ivsize(stfm);
145 start = sreq->cryptlen - ivsize;
146
147 if (req_info->is_enc) {
148 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
149 ivsize, 0);
150 } else {
151 if (sreq->src != sreq->dst) {
152 scatterwalk_map_and_copy(sreq->iv, sreq->src,
153 start, ivsize, 0);
154 } else {
155 memcpy(sreq->iv, req_info->iv_out, ivsize);
156 kfree(req_info->iv_out);
157 }
158 }
159 }
160 }
161
162 static void otx2_cpt_skcipher_callback(int status, void *arg1, void *arg2)
163 {
164 struct otx2_cpt_inst_info *inst_info = arg2;
165 struct crypto_async_request *areq = arg1;
166 struct pci_dev *pdev;
167
168 if (areq) {
169 if (!status)
170 output_iv_copyback(areq);
171 if (inst_info) {
172 pdev = inst_info->pdev;
173 otx2_cpt_info_destroy(pdev, inst_info);
174 }
175 crypto_request_complete(areq, status);
176 }
177 }
178
179 static inline void update_input_data(struct otx2_cpt_req_info *req_info,
180 struct scatterlist *inp_sg,
181 u32 nbytes, u32 *argcnt)
182 {
183 req_info->req.dlen += nbytes;
184
185 while (nbytes) {
186 u32 len = (nbytes < inp_sg->length) ? nbytes : inp_sg->length;
187 u8 *ptr = sg_virt(inp_sg);
188
189 req_info->in[*argcnt].vptr = (void *)ptr;
190 req_info->in[*argcnt].size = len;
191 nbytes -= len;
192 ++(*argcnt);
193 inp_sg = sg_next(inp_sg);
194 }
195 }
196
197 static inline void update_output_data(struct otx2_cpt_req_info *req_info,
198 struct scatterlist *outp_sg,
199 u32 offset, u32 nbytes, u32 *argcnt)
200 {
201 u32 len, sg_len;
202 u8 *ptr;
203
204 req_info->rlen += nbytes;
205
206 while (nbytes) {
207 sg_len = outp_sg->length - offset;
208 len = (nbytes < sg_len) ? nbytes : sg_len;
209 ptr = sg_virt(outp_sg);
210
211 req_info->out[*argcnt].vptr = (void *) (ptr + offset);
212 req_info->out[*argcnt].size = len;
213 nbytes -= len;
214 ++(*argcnt);
215 offset = 0;
216 outp_sg = sg_next(outp_sg);
217 }
218 }
219
220 static inline int create_ctx_hdr(struct skcipher_request *req, u32 enc,
221 u32 *argcnt)
222 {
223 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
224 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
225 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
226 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
227 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
228 int ivsize = crypto_skcipher_ivsize(stfm);
229 u32 start = req->cryptlen - ivsize;
230 gfp_t flags;
231
232 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
233 GFP_KERNEL : GFP_ATOMIC;
234 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
235 req_info->ctrl.s.se_req = 1;
236
237 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
238 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
239 if (enc) {
240 req_info->req.opcode.s.minor = 2;
241 } else {
242 req_info->req.opcode.s.minor = 3;
243 if ((ctx->cipher_type == OTX2_CPT_AES_CBC ||
244 ctx->cipher_type == OTX2_CPT_DES3_CBC) &&
245 req->src == req->dst) {
246 req_info->iv_out = kmalloc(ivsize, flags);
247 if (!req_info->iv_out)
248 return -ENOMEM;
249
250 scatterwalk_map_and_copy(req_info->iv_out, req->src,
251 start, ivsize, 0);
252 }
253 }
254 /* Encryption data length */
255 req_info->req.param1 = req->cryptlen;
256 /* Authentication data length */
257 req_info->req.param2 = 0;
258
259 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
260 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
261 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
262
263 if (ctx->cipher_type == OTX2_CPT_AES_XTS)
264 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
265 else
266 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
267
268 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
269
270 cpu_to_be64s(&fctx->enc.enc_ctrl.u);
271
272 /*
273 * Storing Packet Data Information in offset
274 * Control Word First 8 bytes
275 */
276 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
277 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
278 req_info->req.dlen += CONTROL_WORD_LEN;
279 ++(*argcnt);
280
281 req_info->in[*argcnt].vptr = (u8 *)fctx;
282 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
283 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
284
285 ++(*argcnt);
286
287 return 0;
288 }
289
290 static inline int create_input_list(struct skcipher_request *req, u32 enc,
291 u32 enc_iv_len)
292 {
293 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
294 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
295 u32 argcnt = 0;
296 int ret;
297
298 ret = create_ctx_hdr(req, enc, &argcnt);
299 if (ret)
300 return ret;
301
302 update_input_data(req_info, req->src, req->cryptlen, &argcnt);
303 req_info->in_cnt = argcnt;
304
305 return 0;
306 }
307
308 static inline void create_output_list(struct skcipher_request *req,
309 u32 enc_iv_len)
310 {
311 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
312 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
313 u32 argcnt = 0;
314
315 /*
316 * OUTPUT Buffer Processing
317 * AES encryption/decryption output would be
318 * received in the following format
319 *
320 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
321 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
322 */
323 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
324 req_info->out_cnt = argcnt;
325 }
326
327 static int skcipher_do_fallback(struct skcipher_request *req, bool is_enc)
328 {
329 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
330 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
331 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
332 int ret;
333
334 if (ctx->fbk_cipher) {
335 skcipher_request_set_tfm(&rctx->sk_fbk_req, ctx->fbk_cipher);
336 skcipher_request_set_callback(&rctx->sk_fbk_req,
337 req->base.flags,
338 req->base.complete,
339 req->base.data);
340 skcipher_request_set_crypt(&rctx->sk_fbk_req, req->src,
341 req->dst, req->cryptlen, req->iv);
342 ret = is_enc ? crypto_skcipher_encrypt(&rctx->sk_fbk_req) :
343 crypto_skcipher_decrypt(&rctx->sk_fbk_req);
344 } else {
345 ret = -EINVAL;
346 }
347 return ret;
348 }
349
350 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
351 {
352 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
353 struct otx2_cpt_req_ctx *rctx = skcipher_request_ctx_dma(req);
354 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
355 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
356 u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
357 struct pci_dev *pdev;
358 int status, cpu_num;
359
360 if (req->cryptlen == 0)
361 return 0;
362
363 if (!IS_ALIGNED(req->cryptlen, ctx->enc_align_len))
364 return -EINVAL;
365
366 if (req->cryptlen > OTX2_CPT_MAX_REQ_SIZE)
367 return skcipher_do_fallback(req, enc);
368
369 /* Clear control words */
370 rctx->ctrl_word.flags = 0;
371 rctx->fctx.enc.enc_ctrl.u = 0;
372
373 status = create_input_list(req, enc, enc_iv_len);
374 if (status)
375 return status;
376 create_output_list(req, enc_iv_len);
377
378 status = get_se_device(&pdev, &cpu_num);
379 if (status)
380 return status;
381
382 req_info->callback = otx2_cpt_skcipher_callback;
383 req_info->areq = &req->base;
384 req_info->req_type = OTX2_CPT_ENC_DEC_REQ;
385 req_info->is_enc = enc;
386 req_info->is_trunc_hmac = false;
387 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
388
389 req_info->req.cptr = ctx->er_ctx.hw_ctx;
390 req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
391
392 /*
393 * We perform an asynchronous send and once
394 * the request is completed the driver would
395 * intimate through registered call back functions
396 */
397 status = otx2_cpt_do_request(pdev, req_info, cpu_num);
398
399 return status;
400 }
401
402 static int otx2_cpt_skcipher_encrypt(struct skcipher_request *req)
403 {
404 return cpt_enc_dec(req, true);
405 }
406
407 static int otx2_cpt_skcipher_decrypt(struct skcipher_request *req)
408 {
409 return cpt_enc_dec(req, false);
410 }
411
412 static int otx2_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
413 const u8 *key, u32 keylen)
414 {
415 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
416 const u8 *key2 = key + (keylen / 2);
417 const u8 *key1 = key;
418 int ret;
419
420 ret = xts_verify_key(tfm, key, keylen);
421 if (ret)
422 return ret;
423 ctx->key_len = keylen;
424 ctx->enc_align_len = 1;
425 memcpy(ctx->enc_key, key1, keylen / 2);
426 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
427 ctx->cipher_type = OTX2_CPT_AES_XTS;
428 switch (ctx->key_len) {
429 case 2 * AES_KEYSIZE_128:
430 ctx->key_type = OTX2_CPT_AES_128_BIT;
431 break;
432 case 2 * AES_KEYSIZE_192:
433 ctx->key_type = OTX2_CPT_AES_192_BIT;
434 break;
435 case 2 * AES_KEYSIZE_256:
436 ctx->key_type = OTX2_CPT_AES_256_BIT;
437 break;
438 default:
439 return -EINVAL;
440 }
441 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
442 }
443
444 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
445 u32 keylen, u8 cipher_type)
446 {
447 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
448
449 if (keylen != DES3_EDE_KEY_SIZE)
450 return -EINVAL;
451
452 ctx->key_len = keylen;
453 ctx->cipher_type = cipher_type;
454 ctx->enc_align_len = 8;
455
456 memcpy(ctx->enc_key, key, keylen);
457
458 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
459 }
460
461 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
462 u32 keylen, u8 cipher_type)
463 {
464 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
465
466 switch (keylen) {
467 case AES_KEYSIZE_128:
468 ctx->key_type = OTX2_CPT_AES_128_BIT;
469 break;
470 case AES_KEYSIZE_192:
471 ctx->key_type = OTX2_CPT_AES_192_BIT;
472 break;
473 case AES_KEYSIZE_256:
474 ctx->key_type = OTX2_CPT_AES_256_BIT;
475 break;
476 default:
477 return -EINVAL;
478 }
479 if (cipher_type == OTX2_CPT_AES_CBC || cipher_type == OTX2_CPT_AES_ECB)
480 ctx->enc_align_len = 16;
481 else
482 ctx->enc_align_len = 1;
483
484 ctx->key_len = keylen;
485 ctx->cipher_type = cipher_type;
486
487 memcpy(ctx->enc_key, key, keylen);
488
489 return crypto_skcipher_setkey(ctx->fbk_cipher, key, keylen);
490 }
491
492 static int otx2_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
493 const u8 *key, u32 keylen)
494 {
495 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_CBC);
496 }
497
498 static int otx2_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
499 const u8 *key, u32 keylen)
500 {
501 return cpt_aes_setkey(tfm, key, keylen, OTX2_CPT_AES_ECB);
502 }
503
504 static int otx2_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
505 const u8 *key, u32 keylen)
506 {
507 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_CBC);
508 }
509
510 static int otx2_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
511 const u8 *key, u32 keylen)
512 {
513 return cpt_des_setkey(tfm, key, keylen, OTX2_CPT_DES3_ECB);
514 }
515
516 static int cpt_skcipher_fallback_init(struct otx2_cpt_enc_ctx *ctx,
517 struct crypto_alg *alg)
518 {
519 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
520 ctx->fbk_cipher =
521 crypto_alloc_skcipher(alg->cra_name, 0,
522 CRYPTO_ALG_ASYNC |
523 CRYPTO_ALG_NEED_FALLBACK);
524 if (IS_ERR(ctx->fbk_cipher)) {
525 pr_err("%s() failed to allocate fallback for %s\n",
526 __func__, alg->cra_name);
527 return PTR_ERR(ctx->fbk_cipher);
528 }
529 }
530 return 0;
531 }
532
533 static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
534 {
535 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
536 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
537 struct crypto_alg *alg = tfm->__crt_alg;
538 struct pci_dev *pdev;
539 int ret, cpu_num;
540
541 memset(ctx, 0, sizeof(*ctx));
542 /*
543 * Additional memory for skcipher_request is
544 * allocated since the cryptd daemon uses
545 * this memory for request_ctx information
546 */
547 crypto_skcipher_set_reqsize_dma(
548 stfm, sizeof(struct otx2_cpt_req_ctx) +
549 sizeof(struct skcipher_request));
550
551 ret = get_se_device(&pdev, &cpu_num);
552 if (ret)
553 return ret;
554
555 ctx->pdev = pdev;
556 ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);
557 if (ret)
558 return ret;
559
560 return cpt_skcipher_fallback_init(ctx, alg);
561 }
562
563 static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
564 {
565 struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
566
567 if (ctx->fbk_cipher) {
568 crypto_free_skcipher(ctx->fbk_cipher);
569 ctx->fbk_cipher = NULL;
570 }
571 cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);
572 }
573
574 static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
575 struct crypto_alg *alg)
576 {
577 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
578 ctx->fbk_cipher =
579 crypto_alloc_aead(alg->cra_name, 0,
580 CRYPTO_ALG_ASYNC |
581 CRYPTO_ALG_NEED_FALLBACK);
582 if (IS_ERR(ctx->fbk_cipher)) {
583 pr_err("%s() failed to allocate fallback for %s\n",
584 __func__, alg->cra_name);
585 return PTR_ERR(ctx->fbk_cipher);
586 }
587 }
588 return 0;
589 }
590
591 static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
592 {
593 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);
594 struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
595 struct crypto_alg *alg = tfm->__crt_alg;
596 struct pci_dev *pdev;
597 int ret, cpu_num;
598
599 ctx->cipher_type = cipher_type;
600 ctx->mac_type = mac_type;
601
602 switch (ctx->mac_type) {
603 case OTX2_CPT_SHA1:
604 ctx->hashalg = crypto_alloc_shash("sha1", 0, 0);
605 break;
606
607 case OTX2_CPT_SHA256:
608 ctx->hashalg = crypto_alloc_shash("sha256", 0, 0);
609 break;
610
611 case OTX2_CPT_SHA384:
612 ctx->hashalg = crypto_alloc_shash("sha384", 0, 0);
613 break;
614
615 case OTX2_CPT_SHA512:
616 ctx->hashalg = crypto_alloc_shash("sha512", 0, 0);
617 break;
618 }
619
620 if (IS_ERR(ctx->hashalg))
621 return PTR_ERR(ctx->hashalg);
622
623 if (ctx->hashalg) {
624 ctx->sdesc = alloc_sdesc(ctx->hashalg);
625 if (!ctx->sdesc) {
626 crypto_free_shash(ctx->hashalg);
627 return -ENOMEM;
628 }
629 }
630
631 /*
632 * When selected cipher is NULL we use HMAC opcode instead of
633 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
634 * for calculating ipad and opad
635 */
636 if (ctx->cipher_type != OTX2_CPT_CIPHER_NULL && ctx->hashalg) {
637 int ss = crypto_shash_statesize(ctx->hashalg);
638
639 ctx->ipad = kzalloc(ss, GFP_KERNEL);
640 if (!ctx->ipad) {
641 kfree(ctx->sdesc);
642 crypto_free_shash(ctx->hashalg);
643 return -ENOMEM;
644 }
645
646 ctx->opad = kzalloc(ss, GFP_KERNEL);
647 if (!ctx->opad) {
648 kfree(ctx->ipad);
649 kfree(ctx->sdesc);
650 crypto_free_shash(ctx->hashalg);
651 return -ENOMEM;
652 }
653 }
654 switch (ctx->cipher_type) {
655 case OTX2_CPT_AES_CBC:
656 case OTX2_CPT_AES_ECB:
657 ctx->enc_align_len = 16;
658 break;
659 case OTX2_CPT_DES3_CBC:
660 case OTX2_CPT_DES3_ECB:
661 ctx->enc_align_len = 8;
662 break;
663 case OTX2_CPT_AES_GCM:
664 case OTX2_CPT_CIPHER_NULL:
665 ctx->enc_align_len = 1;
666 break;
667 }
668 crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));
669
670 ret = get_se_device(&pdev, &cpu_num);
671 if (ret)
672 return ret;
673
674 ctx->pdev = pdev;
675 ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);
676 if (ret)
677 return ret;
678
679 return cpt_aead_fallback_init(ctx, alg);
680 }
681
682 static int otx2_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
683 {
684 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA1);
685 }
686
687 static int otx2_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
688 {
689 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA256);
690 }
691
692 static int otx2_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
693 {
694 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA384);
695 }
696
697 static int otx2_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
698 {
699 return cpt_aead_init(tfm, OTX2_CPT_AES_CBC, OTX2_CPT_SHA512);
700 }
701
702 static int otx2_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
703 {
704 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA1);
705 }
706
707 static int otx2_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
708 {
709 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA256);
710 }
711
712 static int otx2_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
713 {
714 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA384);
715 }
716
717 static int otx2_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
718 {
719 return cpt_aead_init(tfm, OTX2_CPT_CIPHER_NULL, OTX2_CPT_SHA512);
720 }
721
722 static int otx2_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
723 {
724 return cpt_aead_init(tfm, OTX2_CPT_AES_GCM, OTX2_CPT_MAC_NULL);
725 }
726
727 static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
728 {
729 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
730
731 kfree(ctx->ipad);
732 kfree(ctx->opad);
733 crypto_free_shash(ctx->hashalg);
734 kfree(ctx->sdesc);
735
736 if (ctx->fbk_cipher) {
737 crypto_free_aead(ctx->fbk_cipher);
738 ctx->fbk_cipher = NULL;
739 }
740 cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);
741 }
742
743 static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
744 unsigned int authsize)
745 {
746 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
747
748 if (crypto_rfc4106_check_authsize(authsize))
749 return -EINVAL;
750
751 tfm->authsize = authsize;
752 /* Set authsize for fallback case */
753 if (ctx->fbk_cipher)
754 ctx->fbk_cipher->authsize = authsize;
755
756 return 0;
757 }
758
759 static int otx2_cpt_aead_set_authsize(struct crypto_aead *tfm,
760 unsigned int authsize)
761 {
762 tfm->authsize = authsize;
763
764 return 0;
765 }
766
767 static int otx2_cpt_aead_null_set_authsize(struct crypto_aead *tfm,
768 unsigned int authsize)
769 {
770 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
771
772 ctx->is_trunc_hmac = true;
773 tfm->authsize = authsize;
774
775 return 0;
776 }
777
778 static struct otx2_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
779 {
780 struct otx2_cpt_sdesc *sdesc;
781 int size;
782
783 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
784 sdesc = kmalloc(size, GFP_KERNEL);
785 if (!sdesc)
786 return NULL;
787
788 sdesc->shash.tfm = alg;
789
790 return sdesc;
791 }
792
793 static inline void swap_data32(void *buf, u32 len)
794 {
795 cpu_to_be32_array(buf, buf, len / 4);
796 }
797
798 static inline void swap_data64(void *buf, u32 len)
799 {
800 u64 *src = buf;
801 int i = 0;
802
803 for (i = 0 ; i < len / 8; i++, src++)
804 cpu_to_be64s(src);
805 }
806
807 static int swap_pad(u8 mac_type, u8 *pad)
808 {
809 struct sha512_state *sha512;
810 struct sha256_state *sha256;
811 struct sha1_state *sha1;
812
813 switch (mac_type) {
814 case OTX2_CPT_SHA1:
815 sha1 = (struct sha1_state *)pad;
816 swap_data32(sha1->state, SHA1_DIGEST_SIZE);
817 break;
818
819 case OTX2_CPT_SHA256:
820 sha256 = (struct sha256_state *)pad;
821 swap_data32(sha256->state, SHA256_DIGEST_SIZE);
822 break;
823
824 case OTX2_CPT_SHA384:
825 case OTX2_CPT_SHA512:
826 sha512 = (struct sha512_state *)pad;
827 swap_data64(sha512->state, SHA512_DIGEST_SIZE);
828 break;
829
830 default:
831 return -EINVAL;
832 }
833
834 return 0;
835 }
836
837 static int aead_hmac_init(struct crypto_aead *cipher,
838 struct crypto_authenc_keys *keys)
839 {
840 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
841 int ds = crypto_shash_digestsize(ctx->hashalg);
842 int bs = crypto_shash_blocksize(ctx->hashalg);
843 int authkeylen = keys->authkeylen;
844 u8 *ipad = NULL, *opad = NULL;
845 int icount = 0;
846 int ret;
847
848 if (authkeylen > bs) {
849 ret = crypto_shash_digest(&ctx->sdesc->shash, keys->authkey,
850 authkeylen, ctx->key);
851 if (ret)
852 goto calc_fail;
853
854 authkeylen = ds;
855 } else
856 memcpy(ctx->key, keys->authkey, authkeylen);
857
858 ctx->enc_key_len = keys->enckeylen;
859 ctx->auth_key_len = authkeylen;
860
861 if (ctx->cipher_type == OTX2_CPT_CIPHER_NULL)
862 return keys->enckeylen ? -EINVAL : 0;
863
864 switch (keys->enckeylen) {
865 case AES_KEYSIZE_128:
866 ctx->key_type = OTX2_CPT_AES_128_BIT;
867 break;
868 case AES_KEYSIZE_192:
869 ctx->key_type = OTX2_CPT_AES_192_BIT;
870 break;
871 case AES_KEYSIZE_256:
872 ctx->key_type = OTX2_CPT_AES_256_BIT;
873 break;
874 default:
875 /* Invalid key length */
876 return -EINVAL;
877 }
878
879 memcpy(ctx->key + authkeylen, keys->enckey, keys->enckeylen);
880
881 ipad = ctx->ipad;
882 opad = ctx->opad;
883
884 memcpy(ipad, ctx->key, authkeylen);
885 memset(ipad + authkeylen, 0, bs - authkeylen);
886 memcpy(opad, ipad, bs);
887
888 for (icount = 0; icount < bs; icount++) {
889 ipad[icount] ^= 0x36;
890 opad[icount] ^= 0x5c;
891 }
892
893 /*
894 * Partial Hash calculated from the software
895 * algorithm is retrieved for IPAD & OPAD
896 */
897
898 /* IPAD Calculation */
899 crypto_shash_init(&ctx->sdesc->shash);
900 crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
901 crypto_shash_export(&ctx->sdesc->shash, ipad);
902 ret = swap_pad(ctx->mac_type, ipad);
903 if (ret)
904 goto calc_fail;
905
906 /* OPAD Calculation */
907 crypto_shash_init(&ctx->sdesc->shash);
908 crypto_shash_update(&ctx->sdesc->shash, opad, bs);
909 crypto_shash_export(&ctx->sdesc->shash, opad);
910 ret = swap_pad(ctx->mac_type, opad);
911
912 calc_fail:
913 return ret;
914 }
915
916 static int otx2_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
917 const unsigned char *key,
918 unsigned int keylen)
919 {
920 struct crypto_authenc_keys authenc_keys;
921
922 return crypto_authenc_extractkeys(&authenc_keys, key, keylen) ?:
923 aead_hmac_init(cipher, &authenc_keys);
924 }
925
926 static int otx2_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
927 const unsigned char *key,
928 unsigned int keylen)
929 {
930 return otx2_cpt_aead_cbc_aes_sha_setkey(cipher, key, keylen);
931 }
932
933 static int otx2_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
934 const unsigned char *key,
935 unsigned int keylen)
936 {
937 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(cipher);
938
939 /*
940 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
941 * and salt (4 bytes)
942 */
943 switch (keylen) {
944 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
945 ctx->key_type = OTX2_CPT_AES_128_BIT;
946 ctx->enc_key_len = AES_KEYSIZE_128;
947 break;
948 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
949 ctx->key_type = OTX2_CPT_AES_192_BIT;
950 ctx->enc_key_len = AES_KEYSIZE_192;
951 break;
952 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
953 ctx->key_type = OTX2_CPT_AES_256_BIT;
954 ctx->enc_key_len = AES_KEYSIZE_256;
955 break;
956 default:
957 /* Invalid key and salt length */
958 return -EINVAL;
959 }
960
961 /* Store encryption key and salt */
962 memcpy(ctx->key, key, keylen);
963
964 return crypto_aead_setkey(ctx->fbk_cipher, key, keylen);
965 }
966
967 static inline int create_aead_ctx_hdr(struct aead_request *req, u32 enc,
968 u32 *argcnt)
969 {
970 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
971 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
972 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
973 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
974 struct otx2_cpt_fc_ctx *fctx = &rctx->fctx;
975 int mac_len = crypto_aead_authsize(tfm);
976 int ds;
977
978 rctx->ctrl_word.e.enc_data_offset = req->assoclen;
979
980 switch (ctx->cipher_type) {
981 case OTX2_CPT_AES_CBC:
982 if (req->assoclen > 248 || !IS_ALIGNED(req->assoclen, 8))
983 return -EINVAL;
984
985 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_CPTR;
986 /* Copy encryption key to context */
987 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
988 ctx->enc_key_len);
989 /* Copy IV to context */
990 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
991
992 ds = crypto_shash_digestsize(ctx->hashalg);
993 if (ctx->mac_type == OTX2_CPT_SHA384)
994 ds = SHA512_DIGEST_SIZE;
995 if (ctx->ipad)
996 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
997 if (ctx->opad)
998 memcpy(fctx->hmac.e.opad, ctx->opad, ds);
999 break;
1000
1001 case OTX2_CPT_AES_GCM:
1002 if (crypto_ipsec_check_assoclen(req->assoclen))
1003 return -EINVAL;
1004
1005 fctx->enc.enc_ctrl.e.iv_source = OTX2_CPT_FROM_DPTR;
1006 /* Copy encryption key to context */
1007 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1008 /* Copy salt to context */
1009 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1010 AES_GCM_SALT_SIZE);
1011
1012 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1013 break;
1014
1015 default:
1016 /* Unknown cipher type */
1017 return -EINVAL;
1018 }
1019 cpu_to_be64s(&rctx->ctrl_word.flags);
1020
1021 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1022 req_info->ctrl.s.se_req = 1;
1023 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_FC |
1024 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1025 if (enc) {
1026 req_info->req.opcode.s.minor = 2;
1027 req_info->req.param1 = req->cryptlen;
1028 req_info->req.param2 = req->cryptlen + req->assoclen;
1029 } else {
1030 req_info->req.opcode.s.minor = 3;
1031 req_info->req.param1 = req->cryptlen - mac_len;
1032 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1033 }
1034
1035 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1036 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1037 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1038 fctx->enc.enc_ctrl.e.mac_len = mac_len;
1039 cpu_to_be64s(&fctx->enc.enc_ctrl.u);
1040
1041 /*
1042 * Storing Packet Data Information in offset
1043 * Control Word First 8 bytes
1044 */
1045 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1046 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1047 req_info->req.dlen += CONTROL_WORD_LEN;
1048 ++(*argcnt);
1049
1050 req_info->in[*argcnt].vptr = (u8 *)fctx;
1051 req_info->in[*argcnt].size = sizeof(struct otx2_cpt_fc_ctx);
1052 req_info->req.dlen += sizeof(struct otx2_cpt_fc_ctx);
1053 ++(*argcnt);
1054
1055 return 0;
1056 }
1057
1058 static inline void create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1059 u32 enc)
1060 {
1061 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1062 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1063 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1064 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1065
1066 req_info->ctrl.s.dma_mode = OTX2_CPT_DMA_MODE_SG;
1067 req_info->ctrl.s.se_req = 1;
1068 req_info->req.opcode.s.major = OTX2_CPT_MAJOR_OP_HMAC |
1069 DMA_MODE_FLAG(OTX2_CPT_DMA_MODE_SG);
1070 req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1071
1072 req_info->req.opcode.s.minor = 0;
1073 req_info->req.param1 = ctx->auth_key_len;
1074 req_info->req.param2 = ctx->mac_type << 8;
1075
1076 /* Add authentication key */
1077 req_info->in[*argcnt].vptr = ctx->key;
1078 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1079 req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1080 ++(*argcnt);
1081 }
1082
1083 static inline int create_aead_input_list(struct aead_request *req, u32 enc)
1084 {
1085 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1086 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1087 u32 inputlen = req->cryptlen + req->assoclen;
1088 u32 status, argcnt = 0;
1089
1090 status = create_aead_ctx_hdr(req, enc, &argcnt);
1091 if (status)
1092 return status;
1093 update_input_data(req_info, req->src, inputlen, &argcnt);
1094 req_info->in_cnt = argcnt;
1095
1096 return 0;
1097 }
1098
1099 static inline void create_aead_output_list(struct aead_request *req, u32 enc,
1100 u32 mac_len)
1101 {
1102 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1103 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1104 u32 argcnt = 0, outputlen = 0;
1105
1106 if (enc)
1107 outputlen = req->cryptlen + req->assoclen + mac_len;
1108 else
1109 outputlen = req->cryptlen + req->assoclen - mac_len;
1110
1111 update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1112 req_info->out_cnt = argcnt;
1113 }
1114
1115 static inline void create_aead_null_input_list(struct aead_request *req,
1116 u32 enc, u32 mac_len)
1117 {
1118 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1119 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1120 u32 inputlen, argcnt = 0;
1121
1122 if (enc)
1123 inputlen = req->cryptlen + req->assoclen;
1124 else
1125 inputlen = req->cryptlen + req->assoclen - mac_len;
1126
1127 create_hmac_ctx_hdr(req, &argcnt, enc);
1128 update_input_data(req_info, req->src, inputlen, &argcnt);
1129 req_info->in_cnt = argcnt;
1130 }
1131
1132 static inline int create_aead_null_output_list(struct aead_request *req,
1133 u32 enc, u32 mac_len)
1134 {
1135 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1136 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1137 struct scatterlist *dst;
1138 u8 *ptr = NULL;
1139 int argcnt = 0, status, offset;
1140 u32 inputlen;
1141
1142 if (enc)
1143 inputlen = req->cryptlen + req->assoclen;
1144 else
1145 inputlen = req->cryptlen + req->assoclen - mac_len;
1146
1147 /*
1148 * If source and destination are different
1149 * then copy payload to destination
1150 */
1151 if (req->src != req->dst) {
1152
1153 ptr = kmalloc(inputlen, (req_info->areq->flags &
1154 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1155 GFP_KERNEL : GFP_ATOMIC);
1156 if (!ptr)
1157 return -ENOMEM;
1158
1159 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1160 inputlen);
1161 if (status != inputlen) {
1162 status = -EINVAL;
1163 goto error_free;
1164 }
1165 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1166 inputlen);
1167 if (status != inputlen) {
1168 status = -EINVAL;
1169 goto error_free;
1170 }
1171 kfree(ptr);
1172 }
1173
1174 if (enc) {
1175 /*
1176 * In an encryption scenario hmac needs
1177 * to be appended after payload
1178 */
1179 dst = req->dst;
1180 offset = inputlen;
1181 while (offset >= dst->length) {
1182 offset -= dst->length;
1183 dst = sg_next(dst);
1184 if (!dst)
1185 return -ENOENT;
1186 }
1187
1188 update_output_data(req_info, dst, offset, mac_len, &argcnt);
1189 } else {
1190 /*
1191 * In a decryption scenario calculated hmac for received
1192 * payload needs to be compare with hmac received
1193 */
1194 status = sg_copy_buffer(req->src, sg_nents(req->src),
1195 rctx->fctx.hmac.s.hmac_recv, mac_len,
1196 inputlen, true);
1197 if (status != mac_len)
1198 return -EINVAL;
1199
1200 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1201 req_info->out[argcnt].size = mac_len;
1202 argcnt++;
1203 }
1204
1205 req_info->out_cnt = argcnt;
1206 return 0;
1207
1208 error_free:
1209 kfree(ptr);
1210 return status;
1211 }
1212
1213 static int aead_do_fallback(struct aead_request *req, bool is_enc)
1214 {
1215 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1216 struct crypto_aead *aead = crypto_aead_reqtfm(req);
1217 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(aead);
1218 int ret;
1219
1220 if (ctx->fbk_cipher) {
1221 /* Store the cipher tfm and then use the fallback tfm */
1222 aead_request_set_tfm(&rctx->fbk_req, ctx->fbk_cipher);
1223 aead_request_set_callback(&rctx->fbk_req, req->base.flags,
1224 req->base.complete, req->base.data);
1225 aead_request_set_crypt(&rctx->fbk_req, req->src,
1226 req->dst, req->cryptlen, req->iv);
1227 aead_request_set_ad(&rctx->fbk_req, req->assoclen);
1228 ret = is_enc ? crypto_aead_encrypt(&rctx->fbk_req) :
1229 crypto_aead_decrypt(&rctx->fbk_req);
1230 } else {
1231 ret = -EINVAL;
1232 }
1233
1234 return ret;
1235 }
1236
1237 static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1238 {
1239 struct otx2_cpt_req_ctx *rctx = aead_request_ctx_dma(req);
1240 struct otx2_cpt_req_info *req_info = &rctx->cpt_req;
1241 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1242 struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(tfm);
1243 struct pci_dev *pdev;
1244 int status, cpu_num;
1245
1246 /* Clear control words */
1247 rctx->ctrl_word.flags = 0;
1248 rctx->fctx.enc.enc_ctrl.u = 0;
1249
1250 req_info->callback = otx2_cpt_aead_callback;
1251 req_info->areq = &req->base;
1252 req_info->req_type = reg_type;
1253 req_info->is_enc = enc;
1254 req_info->is_trunc_hmac = false;
1255
1256 req_info->req.cptr = ctx->er_ctx.hw_ctx;
1257 req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
1258
1259 switch (reg_type) {
1260 case OTX2_CPT_AEAD_ENC_DEC_REQ:
1261 status = create_aead_input_list(req, enc);
1262 if (status)
1263 return status;
1264 create_aead_output_list(req, enc, crypto_aead_authsize(tfm));
1265 break;
1266
1267 case OTX2_CPT_AEAD_ENC_DEC_NULL_REQ:
1268 create_aead_null_input_list(req, enc,
1269 crypto_aead_authsize(tfm));
1270 status = create_aead_null_output_list(req, enc,
1271 crypto_aead_authsize(tfm));
1272 if (status)
1273 return status;
1274 break;
1275
1276 default:
1277 return -EINVAL;
1278 }
1279 if (!IS_ALIGNED(req_info->req.param1, ctx->enc_align_len))
1280 return -EINVAL;
1281
1282 if (!req_info->req.param2 ||
1283 (req_info->req.param1 > OTX2_CPT_MAX_REQ_SIZE) ||
1284 (req_info->req.param2 > OTX2_CPT_MAX_REQ_SIZE))
1285 return aead_do_fallback(req, enc);
1286
1287 status = get_se_device(&pdev, &cpu_num);
1288 if (status)
1289 return status;
1290
1291 req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
1292
1293 /*
1294 * We perform an asynchronous send and once
1295 * the request is completed the driver would
1296 * intimate through registered call back functions
1297 */
1298 return otx2_cpt_do_request(pdev, req_info, cpu_num);
1299 }
1300
1301 static int otx2_cpt_aead_encrypt(struct aead_request *req)
1302 {
1303 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, true);
1304 }
1305
1306 static int otx2_cpt_aead_decrypt(struct aead_request *req)
1307 {
1308 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_REQ, false);
1309 }
1310
1311 static int otx2_cpt_aead_null_encrypt(struct aead_request *req)
1312 {
1313 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1314 }
1315
1316 static int otx2_cpt_aead_null_decrypt(struct aead_request *req)
1317 {
1318 return cpt_aead_enc_dec(req, OTX2_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1319 }
1320
1321 static struct skcipher_alg otx2_cpt_skciphers[] = { {
1322 .base.cra_name = "xts(aes)",
1323 .base.cra_driver_name = "cpt_xts_aes",
1324 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1325 .base.cra_blocksize = AES_BLOCK_SIZE,
1326 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1327 .base.cra_alignmask = 7,
1328 .base.cra_priority = 4001,
1329 .base.cra_module = THIS_MODULE,
1330
1331 .init = otx2_cpt_enc_dec_init,
1332 .exit = otx2_cpt_skcipher_exit,
1333 .ivsize = AES_BLOCK_SIZE,
1334 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1335 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1336 .setkey = otx2_cpt_skcipher_xts_setkey,
1337 .encrypt = otx2_cpt_skcipher_encrypt,
1338 .decrypt = otx2_cpt_skcipher_decrypt,
1339 }, {
1340 .base.cra_name = "cbc(aes)",
1341 .base.cra_driver_name = "cpt_cbc_aes",
1342 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1343 .base.cra_blocksize = AES_BLOCK_SIZE,
1344 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1345 .base.cra_alignmask = 7,
1346 .base.cra_priority = 4001,
1347 .base.cra_module = THIS_MODULE,
1348
1349 .init = otx2_cpt_enc_dec_init,
1350 .exit = otx2_cpt_skcipher_exit,
1351 .ivsize = AES_BLOCK_SIZE,
1352 .min_keysize = AES_MIN_KEY_SIZE,
1353 .max_keysize = AES_MAX_KEY_SIZE,
1354 .setkey = otx2_cpt_skcipher_cbc_aes_setkey,
1355 .encrypt = otx2_cpt_skcipher_encrypt,
1356 .decrypt = otx2_cpt_skcipher_decrypt,
1357 }, {
1358 .base.cra_name = "ecb(aes)",
1359 .base.cra_driver_name = "cpt_ecb_aes",
1360 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1361 .base.cra_blocksize = AES_BLOCK_SIZE,
1362 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1363 .base.cra_alignmask = 7,
1364 .base.cra_priority = 4001,
1365 .base.cra_module = THIS_MODULE,
1366
1367 .init = otx2_cpt_enc_dec_init,
1368 .exit = otx2_cpt_skcipher_exit,
1369 .ivsize = 0,
1370 .min_keysize = AES_MIN_KEY_SIZE,
1371 .max_keysize = AES_MAX_KEY_SIZE,
1372 .setkey = otx2_cpt_skcipher_ecb_aes_setkey,
1373 .encrypt = otx2_cpt_skcipher_encrypt,
1374 .decrypt = otx2_cpt_skcipher_decrypt,
1375 }, {
1376 .base.cra_name = "cbc(des3_ede)",
1377 .base.cra_driver_name = "cpt_cbc_des3_ede",
1378 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1379 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1380 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1381 .base.cra_alignmask = 7,
1382 .base.cra_priority = 4001,
1383 .base.cra_module = THIS_MODULE,
1384
1385 .init = otx2_cpt_enc_dec_init,
1386 .exit = otx2_cpt_skcipher_exit,
1387 .min_keysize = DES3_EDE_KEY_SIZE,
1388 .max_keysize = DES3_EDE_KEY_SIZE,
1389 .ivsize = DES_BLOCK_SIZE,
1390 .setkey = otx2_cpt_skcipher_cbc_des3_setkey,
1391 .encrypt = otx2_cpt_skcipher_encrypt,
1392 .decrypt = otx2_cpt_skcipher_decrypt,
1393 }, {
1394 .base.cra_name = "ecb(des3_ede)",
1395 .base.cra_driver_name = "cpt_ecb_des3_ede",
1396 .base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1397 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1398 .base.cra_ctxsize = sizeof(struct otx2_cpt_enc_ctx),
1399 .base.cra_alignmask = 7,
1400 .base.cra_priority = 4001,
1401 .base.cra_module = THIS_MODULE,
1402
1403 .init = otx2_cpt_enc_dec_init,
1404 .exit = otx2_cpt_skcipher_exit,
1405 .min_keysize = DES3_EDE_KEY_SIZE,
1406 .max_keysize = DES3_EDE_KEY_SIZE,
1407 .ivsize = 0,
1408 .setkey = otx2_cpt_skcipher_ecb_des3_setkey,
1409 .encrypt = otx2_cpt_skcipher_encrypt,
1410 .decrypt = otx2_cpt_skcipher_decrypt,
1411 } };
1412
1413 static struct aead_alg otx2_cpt_aeads[] = { {
1414 .base = {
1415 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1416 .cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1417 .cra_blocksize = AES_BLOCK_SIZE,
1418 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1419 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1420 .cra_priority = 4001,
1421 .cra_alignmask = 0,
1422 .cra_module = THIS_MODULE,
1423 },
1424 .init = otx2_cpt_aead_cbc_aes_sha1_init,
1425 .exit = otx2_cpt_aead_exit,
1426 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1427 .setauthsize = otx2_cpt_aead_set_authsize,
1428 .encrypt = otx2_cpt_aead_encrypt,
1429 .decrypt = otx2_cpt_aead_decrypt,
1430 .ivsize = AES_BLOCK_SIZE,
1431 .maxauthsize = SHA1_DIGEST_SIZE,
1432 }, {
1433 .base = {
1434 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1435 .cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1436 .cra_blocksize = AES_BLOCK_SIZE,
1437 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1438 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1439 .cra_priority = 4001,
1440 .cra_alignmask = 0,
1441 .cra_module = THIS_MODULE,
1442 },
1443 .init = otx2_cpt_aead_cbc_aes_sha256_init,
1444 .exit = otx2_cpt_aead_exit,
1445 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1446 .setauthsize = otx2_cpt_aead_set_authsize,
1447 .encrypt = otx2_cpt_aead_encrypt,
1448 .decrypt = otx2_cpt_aead_decrypt,
1449 .ivsize = AES_BLOCK_SIZE,
1450 .maxauthsize = SHA256_DIGEST_SIZE,
1451 }, {
1452 .base = {
1453 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1454 .cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1455 .cra_blocksize = AES_BLOCK_SIZE,
1456 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1457 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1458 .cra_priority = 4001,
1459 .cra_alignmask = 0,
1460 .cra_module = THIS_MODULE,
1461 },
1462 .init = otx2_cpt_aead_cbc_aes_sha384_init,
1463 .exit = otx2_cpt_aead_exit,
1464 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1465 .setauthsize = otx2_cpt_aead_set_authsize,
1466 .encrypt = otx2_cpt_aead_encrypt,
1467 .decrypt = otx2_cpt_aead_decrypt,
1468 .ivsize = AES_BLOCK_SIZE,
1469 .maxauthsize = SHA384_DIGEST_SIZE,
1470 }, {
1471 .base = {
1472 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1473 .cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1474 .cra_blocksize = AES_BLOCK_SIZE,
1475 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1476 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1477 .cra_priority = 4001,
1478 .cra_alignmask = 0,
1479 .cra_module = THIS_MODULE,
1480 },
1481 .init = otx2_cpt_aead_cbc_aes_sha512_init,
1482 .exit = otx2_cpt_aead_exit,
1483 .setkey = otx2_cpt_aead_cbc_aes_sha_setkey,
1484 .setauthsize = otx2_cpt_aead_set_authsize,
1485 .encrypt = otx2_cpt_aead_encrypt,
1486 .decrypt = otx2_cpt_aead_decrypt,
1487 .ivsize = AES_BLOCK_SIZE,
1488 .maxauthsize = SHA512_DIGEST_SIZE,
1489 }, {
1490 .base = {
1491 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1492 .cra_driver_name = "cpt_hmac_sha1_ecb_null",
1493 .cra_blocksize = 1,
1494 .cra_flags = CRYPTO_ALG_ASYNC,
1495 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1496 .cra_priority = 4001,
1497 .cra_alignmask = 0,
1498 .cra_module = THIS_MODULE,
1499 },
1500 .init = otx2_cpt_aead_ecb_null_sha1_init,
1501 .exit = otx2_cpt_aead_exit,
1502 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1503 .setauthsize = otx2_cpt_aead_null_set_authsize,
1504 .encrypt = otx2_cpt_aead_null_encrypt,
1505 .decrypt = otx2_cpt_aead_null_decrypt,
1506 .ivsize = 0,
1507 .maxauthsize = SHA1_DIGEST_SIZE,
1508 }, {
1509 .base = {
1510 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1511 .cra_driver_name = "cpt_hmac_sha256_ecb_null",
1512 .cra_blocksize = 1,
1513 .cra_flags = CRYPTO_ALG_ASYNC,
1514 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1515 .cra_priority = 4001,
1516 .cra_alignmask = 0,
1517 .cra_module = THIS_MODULE,
1518 },
1519 .init = otx2_cpt_aead_ecb_null_sha256_init,
1520 .exit = otx2_cpt_aead_exit,
1521 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1522 .setauthsize = otx2_cpt_aead_null_set_authsize,
1523 .encrypt = otx2_cpt_aead_null_encrypt,
1524 .decrypt = otx2_cpt_aead_null_decrypt,
1525 .ivsize = 0,
1526 .maxauthsize = SHA256_DIGEST_SIZE,
1527 }, {
1528 .base = {
1529 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1530 .cra_driver_name = "cpt_hmac_sha384_ecb_null",
1531 .cra_blocksize = 1,
1532 .cra_flags = CRYPTO_ALG_ASYNC,
1533 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1534 .cra_priority = 4001,
1535 .cra_alignmask = 0,
1536 .cra_module = THIS_MODULE,
1537 },
1538 .init = otx2_cpt_aead_ecb_null_sha384_init,
1539 .exit = otx2_cpt_aead_exit,
1540 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1541 .setauthsize = otx2_cpt_aead_null_set_authsize,
1542 .encrypt = otx2_cpt_aead_null_encrypt,
1543 .decrypt = otx2_cpt_aead_null_decrypt,
1544 .ivsize = 0,
1545 .maxauthsize = SHA384_DIGEST_SIZE,
1546 }, {
1547 .base = {
1548 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1549 .cra_driver_name = "cpt_hmac_sha512_ecb_null",
1550 .cra_blocksize = 1,
1551 .cra_flags = CRYPTO_ALG_ASYNC,
1552 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1553 .cra_priority = 4001,
1554 .cra_alignmask = 0,
1555 .cra_module = THIS_MODULE,
1556 },
1557 .init = otx2_cpt_aead_ecb_null_sha512_init,
1558 .exit = otx2_cpt_aead_exit,
1559 .setkey = otx2_cpt_aead_ecb_null_sha_setkey,
1560 .setauthsize = otx2_cpt_aead_null_set_authsize,
1561 .encrypt = otx2_cpt_aead_null_encrypt,
1562 .decrypt = otx2_cpt_aead_null_decrypt,
1563 .ivsize = 0,
1564 .maxauthsize = SHA512_DIGEST_SIZE,
1565 }, {
1566 .base = {
1567 .cra_name = "rfc4106(gcm(aes))",
1568 .cra_driver_name = "cpt_rfc4106_gcm_aes",
1569 .cra_blocksize = 1,
1570 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1571 .cra_ctxsize = sizeof(struct otx2_cpt_aead_ctx) + CRYPTO_DMA_PADDING,
1572 .cra_priority = 4001,
1573 .cra_alignmask = 0,
1574 .cra_module = THIS_MODULE,
1575 },
1576 .init = otx2_cpt_aead_gcm_aes_init,
1577 .exit = otx2_cpt_aead_exit,
1578 .setkey = otx2_cpt_aead_gcm_aes_setkey,
1579 .setauthsize = otx2_cpt_aead_gcm_set_authsize,
1580 .encrypt = otx2_cpt_aead_encrypt,
1581 .decrypt = otx2_cpt_aead_decrypt,
1582 .ivsize = AES_GCM_IV_SIZE,
1583 .maxauthsize = AES_GCM_ICV_SIZE,
1584 } };
1585
1586 static inline int cpt_register_algs(void)
1587 {
1588 int i, err = 0;
1589
1590 for (i = 0; i < ARRAY_SIZE(otx2_cpt_skciphers); i++)
1591 otx2_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1592
1593 err = crypto_register_skciphers(otx2_cpt_skciphers,
1594 ARRAY_SIZE(otx2_cpt_skciphers));
1595 if (err)
1596 return err;
1597
1598 for (i = 0; i < ARRAY_SIZE(otx2_cpt_aeads); i++)
1599 otx2_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1600
1601 err = crypto_register_aeads(otx2_cpt_aeads,
1602 ARRAY_SIZE(otx2_cpt_aeads));
1603 if (err) {
1604 crypto_unregister_skciphers(otx2_cpt_skciphers,
1605 ARRAY_SIZE(otx2_cpt_skciphers));
1606 return err;
1607 }
1608
1609 return 0;
1610 }
1611
1612 static inline void cpt_unregister_algs(void)
1613 {
1614 crypto_unregister_skciphers(otx2_cpt_skciphers,
1615 ARRAY_SIZE(otx2_cpt_skciphers));
1616 crypto_unregister_aeads(otx2_cpt_aeads, ARRAY_SIZE(otx2_cpt_aeads));
1617 }
1618
1619 static int compare_func(const void *lptr, const void *rptr)
1620 {
1621 const struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1622 const struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1623
1624 if (ldesc->dev->devfn < rdesc->dev->devfn)
1625 return -1;
1626 if (ldesc->dev->devfn > rdesc->dev->devfn)
1627 return 1;
1628 return 0;
1629 }
1630
1631 int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1632 int num_queues, int num_devices)
1633 {
1634 int ret = 0;
1635 int count;
1636
1637 mutex_lock(&mutex);
1638 count = atomic_read(&se_devices.count);
1639 if (count >= OTX2_CPT_MAX_LFS_NUM) {
1640 dev_err(&pdev->dev, "No space to add a new device\n");
1641 ret = -ENOSPC;
1642 goto unlock;
1643 }
1644 se_devices.desc[count].num_queues = num_queues;
1645 se_devices.desc[count++].dev = pdev;
1646 atomic_inc(&se_devices.count);
1647
1648 if (atomic_read(&se_devices.count) == num_devices &&
1649 is_crypto_registered == false) {
1650 if (cpt_register_algs()) {
1651 dev_err(&pdev->dev,
1652 "Error in registering crypto algorithms\n");
1653 ret = -EINVAL;
1654 goto unlock;
1655 }
1656 try_module_get(mod);
1657 is_crypto_registered = true;
1658 }
1659 sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1660 compare_func, NULL);
1661
1662 unlock:
1663 mutex_unlock(&mutex);
1664 return ret;
1665 }
1666
1667 void otx2_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod)
1668 {
1669 struct cpt_device_table *dev_tbl;
1670 bool dev_found = false;
1671 int i, j, count;
1672
1673 mutex_lock(&mutex);
1674
1675 dev_tbl = &se_devices;
1676 count = atomic_read(&dev_tbl->count);
1677 for (i = 0; i < count; i++) {
1678 if (pdev == dev_tbl->desc[i].dev) {
1679 for (j = i; j < count-1; j++)
1680 dev_tbl->desc[j] = dev_tbl->desc[j+1];
1681 dev_found = true;
1682 break;
1683 }
1684 }
1685
1686 if (!dev_found) {
1687 dev_err(&pdev->dev, "%s device not found\n", __func__);
1688 goto unlock;
1689 }
1690 if (atomic_dec_and_test(&se_devices.count)) {
1691 cpt_unregister_algs();
1692 module_put(mod);
1693 is_crypto_registered = false;
1694 }
1695
1696 unlock:
1697 mutex_unlock(&mutex);
1698 }
Kernel DRM miscellaneous fixes and cross-tree changes