?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
gpio: rcar: Fix runtime PM imbalance on error
[linux/fpc-iii.git] / drivers / crypto / marvell / octeontx / otx_cptvf_algs.c
blob06202bcffb33fe7cd91a5311588726c3c73b00c4
1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell OcteonTX CPT driver
3 *
4 * Copyright (C) 2019 Marvell International Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <crypto/aes.h>
12 #include <crypto/authenc.h>
13 #include <crypto/cryptd.h>
14 #include <crypto/des.h>
15 #include <crypto/internal/aead.h>
16 #include <crypto/sha.h>
17 #include <crypto/xts.h>
18 #include <crypto/scatterwalk.h>
19 #include <linux/rtnetlink.h>
20 #include <linux/sort.h>
21 #include <linux/module.h>
22 #include "otx_cptvf.h"
23 #include "otx_cptvf_algs.h"
24 #include "otx_cptvf_reqmgr.h"
25
26 #define CPT_MAX_VF_NUM 64
27 /* Size of salt in AES GCM mode */
28 #define AES_GCM_SALT_SIZE 4
29 /* Size of IV in AES GCM mode */
30 #define AES_GCM_IV_SIZE 8
31 /* Size of ICV (Integrity Check Value) in AES GCM mode */
32 #define AES_GCM_ICV_SIZE 16
33 /* Offset of IV in AES GCM mode */
34 #define AES_GCM_IV_OFFSET 8
35 #define CONTROL_WORD_LEN 8
36 #define KEY2_OFFSET 48
37 #define DMA_MODE_FLAG(dma_mode) \
38 (((dma_mode) == OTX_CPT_DMA_GATHER_SCATTER) ? (1 << 7) : 0)
39
40 /* Truncated SHA digest size */
41 #define SHA1_TRUNC_DIGEST_SIZE 12
42 #define SHA256_TRUNC_DIGEST_SIZE 16
43 #define SHA384_TRUNC_DIGEST_SIZE 24
44 #define SHA512_TRUNC_DIGEST_SIZE 32
45
46 static DEFINE_MUTEX(mutex);
47 static int is_crypto_registered;
48
49 struct cpt_device_desc {
50 enum otx_cptpf_type pf_type;
51 struct pci_dev *dev;
52 int num_queues;
53 };
54
55 struct cpt_device_table {
56 atomic_t count;
57 struct cpt_device_desc desc[CPT_MAX_VF_NUM];
58 };
59
60 static struct cpt_device_table se_devices = {
61 .count = ATOMIC_INIT(0)
62 };
63
64 static struct cpt_device_table ae_devices = {
65 .count = ATOMIC_INIT(0)
66 };
67
68 static inline int get_se_device(struct pci_dev **pdev, int *cpu_num)
69 {
70 int count, ret = 0;
71
72 count = atomic_read(&se_devices.count);
73 if (count < 1)
74 return -ENODEV;
75
76 *cpu_num = get_cpu();
77
78 if (se_devices.desc[0].pf_type == OTX_CPT_SE) {
79 /*
80 * On OcteonTX platform there is one CPT instruction queue bound
81 * to each VF. We get maximum performance if one CPT queue
82 * is available for each cpu otherwise CPT queues need to be
83 * shared between cpus.
84 */
85 if (*cpu_num >= count)
86 *cpu_num %= count;
87 *pdev = se_devices.desc[*cpu_num].dev;
88 } else {
89 pr_err("Unknown PF type %d\n", se_devices.desc[0].pf_type);
90 ret = -EINVAL;
91 }
92 put_cpu();
93
94 return ret;
95 }
96
97 static inline int validate_hmac_cipher_null(struct otx_cpt_req_info *cpt_req)
98 {
99 struct otx_cpt_req_ctx *rctx;
100 struct aead_request *req;
101 struct crypto_aead *tfm;
102
103 req = container_of(cpt_req->areq, struct aead_request, base);
104 tfm = crypto_aead_reqtfm(req);
105 rctx = aead_request_ctx(req);
106 if (memcmp(rctx->fctx.hmac.s.hmac_calc,
107 rctx->fctx.hmac.s.hmac_recv,
108 crypto_aead_authsize(tfm)) != 0)
109 return -EBADMSG;
110
111 return 0;
112 }
113
114 static void otx_cpt_aead_callback(int status, void *arg1, void *arg2)
115 {
116 struct otx_cpt_info_buffer *cpt_info = arg2;
117 struct crypto_async_request *areq = arg1;
118 struct otx_cpt_req_info *cpt_req;
119 struct pci_dev *pdev;
120
121 cpt_req = cpt_info->req;
122 if (!status) {
123 /*
124 * When selected cipher is NULL we need to manually
125 * verify whether calculated hmac value matches
126 * received hmac value
127 */
128 if (cpt_req->req_type == OTX_CPT_AEAD_ENC_DEC_NULL_REQ &&
129 !cpt_req->is_enc)
130 status = validate_hmac_cipher_null(cpt_req);
131 }
132 if (cpt_info) {
133 pdev = cpt_info->pdev;
134 do_request_cleanup(pdev, cpt_info);
135 }
136 if (areq)
137 areq->complete(areq, status);
138 }
139
140 static void output_iv_copyback(struct crypto_async_request *areq)
141 {
142 struct otx_cpt_req_info *req_info;
143 struct skcipher_request *sreq;
144 struct crypto_skcipher *stfm;
145 struct otx_cpt_req_ctx *rctx;
146 struct otx_cpt_enc_ctx *ctx;
147 u32 start, ivsize;
148
149 sreq = container_of(areq, struct skcipher_request, base);
150 stfm = crypto_skcipher_reqtfm(sreq);
151 ctx = crypto_skcipher_ctx(stfm);
152 if (ctx->cipher_type == OTX_CPT_AES_CBC ||
153 ctx->cipher_type == OTX_CPT_DES3_CBC) {
154 rctx = skcipher_request_ctx(sreq);
155 req_info = &rctx->cpt_req;
156 ivsize = crypto_skcipher_ivsize(stfm);
157 start = sreq->cryptlen - ivsize;
158
159 if (req_info->is_enc) {
160 scatterwalk_map_and_copy(sreq->iv, sreq->dst, start,
161 ivsize, 0);
162 } else {
163 if (sreq->src != sreq->dst) {
164 scatterwalk_map_and_copy(sreq->iv, sreq->src,
165 start, ivsize, 0);
166 } else {
167 memcpy(sreq->iv, req_info->iv_out, ivsize);
168 kfree(req_info->iv_out);
169 }
170 }
171 }
172 }
173
174 static void otx_cpt_skcipher_callback(int status, void *arg1, void *arg2)
175 {
176 struct otx_cpt_info_buffer *cpt_info = arg2;
177 struct crypto_async_request *areq = arg1;
178 struct pci_dev *pdev;
179
180 if (areq) {
181 if (!status)
182 output_iv_copyback(areq);
183 if (cpt_info) {
184 pdev = cpt_info->pdev;
185 do_request_cleanup(pdev, cpt_info);
186 }
187 areq->complete(areq, status);
188 }
189 }
190
191 static inline void update_input_data(struct otx_cpt_req_info *req_info,
192 struct scatterlist *inp_sg,
193 u32 nbytes, u32 *argcnt)
194 {
195 req_info->req.dlen += nbytes;
196
197 while (nbytes) {
198 u32 len = min(nbytes, inp_sg->length);
199 u8 *ptr = sg_virt(inp_sg);
200
201 req_info->in[*argcnt].vptr = (void *)ptr;
202 req_info->in[*argcnt].size = len;
203 nbytes -= len;
204 ++(*argcnt);
205 inp_sg = sg_next(inp_sg);
206 }
207 }
208
209 static inline void update_output_data(struct otx_cpt_req_info *req_info,
210 struct scatterlist *outp_sg,
211 u32 offset, u32 nbytes, u32 *argcnt)
212 {
213 req_info->rlen += nbytes;
214
215 while (nbytes) {
216 u32 len = min(nbytes, outp_sg->length - offset);
217 u8 *ptr = sg_virt(outp_sg);
218
219 req_info->out[*argcnt].vptr = (void *) (ptr + offset);
220 req_info->out[*argcnt].size = len;
221 nbytes -= len;
222 ++(*argcnt);
223 offset = 0;
224 outp_sg = sg_next(outp_sg);
225 }
226 }
227
228 static inline u32 create_ctx_hdr(struct skcipher_request *req, u32 enc,
229 u32 *argcnt)
230 {
231 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
232 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
233 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
234 struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
235 struct otx_cpt_enc_ctx *ctx = crypto_tfm_ctx(tfm);
236 struct otx_cpt_fc_ctx *fctx = &rctx->fctx;
237 int ivsize = crypto_skcipher_ivsize(stfm);
238 u32 start = req->cryptlen - ivsize;
239 u64 *ctrl_flags = NULL;
240 gfp_t flags;
241
242 flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
243 GFP_KERNEL : GFP_ATOMIC;
244 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
245 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;
246
247 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC |
248 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);
249 if (enc) {
250 req_info->req.opcode.s.minor = 2;
251 } else {
252 req_info->req.opcode.s.minor = 3;
253 if ((ctx->cipher_type == OTX_CPT_AES_CBC ||
254 ctx->cipher_type == OTX_CPT_DES3_CBC) &&
255 req->src == req->dst) {
256 req_info->iv_out = kmalloc(ivsize, flags);
257 if (!req_info->iv_out)
258 return -ENOMEM;
259
260 scatterwalk_map_and_copy(req_info->iv_out, req->src,
261 start, ivsize, 0);
262 }
263 }
264 /* Encryption data length */
265 req_info->req.param1 = req->cryptlen;
266 /* Authentication data length */
267 req_info->req.param2 = 0;
268
269 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
270 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
271 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR;
272
273 if (ctx->cipher_type == OTX_CPT_AES_XTS)
274 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len * 2);
275 else
276 memcpy(fctx->enc.encr_key, ctx->enc_key, ctx->key_len);
277
278 memcpy(fctx->enc.encr_iv, req->iv, crypto_skcipher_ivsize(stfm));
279
280 ctrl_flags = (u64 *)&fctx->enc.enc_ctrl.flags;
281 *ctrl_flags = cpu_to_be64(*ctrl_flags);
282
283 /*
284 * Storing Packet Data Information in offset
285 * Control Word First 8 bytes
286 */
287 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
288 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
289 req_info->req.dlen += CONTROL_WORD_LEN;
290 ++(*argcnt);
291
292 req_info->in[*argcnt].vptr = (u8 *)fctx;
293 req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx);
294 req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx);
295
296 ++(*argcnt);
297
298 return 0;
299 }
300
301 static inline u32 create_input_list(struct skcipher_request *req, u32 enc,
302 u32 enc_iv_len)
303 {
304 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
305 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
306 u32 argcnt = 0;
307 int ret;
308
309 ret = create_ctx_hdr(req, enc, &argcnt);
310 if (ret)
311 return ret;
312
313 update_input_data(req_info, req->src, req->cryptlen, &argcnt);
314 req_info->incnt = argcnt;
315
316 return 0;
317 }
318
319 static inline void create_output_list(struct skcipher_request *req,
320 u32 enc_iv_len)
321 {
322 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
323 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
324 u32 argcnt = 0;
325
326 /*
327 * OUTPUT Buffer Processing
328 * AES encryption/decryption output would be
329 * received in the following format
330 *
331 * ------IV--------|------ENCRYPTED/DECRYPTED DATA-----|
332 * [ 16 Bytes/ [ Request Enc/Dec/ DATA Len AES CBC ]
333 */
334 update_output_data(req_info, req->dst, 0, req->cryptlen, &argcnt);
335 req_info->outcnt = argcnt;
336 }
337
338 static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
339 {
340 struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
341 struct otx_cpt_req_ctx *rctx = skcipher_request_ctx(req);
342 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
343 u32 enc_iv_len = crypto_skcipher_ivsize(stfm);
344 struct pci_dev *pdev;
345 int status, cpu_num;
346
347 /* Validate that request doesn't exceed maximum CPT supported size */
348 if (req->cryptlen > OTX_CPT_MAX_REQ_SIZE)
349 return -E2BIG;
350
351 /* Clear control words */
352 rctx->ctrl_word.flags = 0;
353 rctx->fctx.enc.enc_ctrl.flags = 0;
354
355 status = create_input_list(req, enc, enc_iv_len);
356 if (status)
357 return status;
358 create_output_list(req, enc_iv_len);
359
360 status = get_se_device(&pdev, &cpu_num);
361 if (status)
362 return status;
363
364 req_info->callback = (void *)otx_cpt_skcipher_callback;
365 req_info->areq = &req->base;
366 req_info->req_type = OTX_CPT_ENC_DEC_REQ;
367 req_info->is_enc = enc;
368 req_info->is_trunc_hmac = false;
369 req_info->ctrl.s.grp = 0;
370
371 /*
372 * We perform an asynchronous send and once
373 * the request is completed the driver would
374 * intimate through registered call back functions
375 */
376 status = otx_cpt_do_request(pdev, req_info, cpu_num);
377
378 return status;
379 }
380
381 static int otx_cpt_skcipher_encrypt(struct skcipher_request *req)
382 {
383 return cpt_enc_dec(req, true);
384 }
385
386 static int otx_cpt_skcipher_decrypt(struct skcipher_request *req)
387 {
388 return cpt_enc_dec(req, false);
389 }
390
391 static int otx_cpt_skcipher_xts_setkey(struct crypto_skcipher *tfm,
392 const u8 *key, u32 keylen)
393 {
394 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
395 const u8 *key2 = key + (keylen / 2);
396 const u8 *key1 = key;
397 int ret;
398
399 ret = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
400 if (ret)
401 return ret;
402 ctx->key_len = keylen;
403 memcpy(ctx->enc_key, key1, keylen / 2);
404 memcpy(ctx->enc_key + KEY2_OFFSET, key2, keylen / 2);
405 ctx->cipher_type = OTX_CPT_AES_XTS;
406 switch (ctx->key_len) {
407 case 2 * AES_KEYSIZE_128:
408 ctx->key_type = OTX_CPT_AES_128_BIT;
409 break;
410 case 2 * AES_KEYSIZE_256:
411 ctx->key_type = OTX_CPT_AES_256_BIT;
412 break;
413 default:
414 return -EINVAL;
415 }
416
417 return 0;
418 }
419
420 static int cpt_des_setkey(struct crypto_skcipher *tfm, const u8 *key,
421 u32 keylen, u8 cipher_type)
422 {
423 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
424
425 if (keylen != DES3_EDE_KEY_SIZE)
426 return -EINVAL;
427
428 ctx->key_len = keylen;
429 ctx->cipher_type = cipher_type;
430
431 memcpy(ctx->enc_key, key, keylen);
432
433 return 0;
434 }
435
436 static int cpt_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
437 u32 keylen, u8 cipher_type)
438 {
439 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
440
441 switch (keylen) {
442 case AES_KEYSIZE_128:
443 ctx->key_type = OTX_CPT_AES_128_BIT;
444 break;
445 case AES_KEYSIZE_192:
446 ctx->key_type = OTX_CPT_AES_192_BIT;
447 break;
448 case AES_KEYSIZE_256:
449 ctx->key_type = OTX_CPT_AES_256_BIT;
450 break;
451 default:
452 return -EINVAL;
453 }
454 ctx->key_len = keylen;
455 ctx->cipher_type = cipher_type;
456
457 memcpy(ctx->enc_key, key, keylen);
458
459 return 0;
460 }
461
462 static int otx_cpt_skcipher_cbc_aes_setkey(struct crypto_skcipher *tfm,
463 const u8 *key, u32 keylen)
464 {
465 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CBC);
466 }
467
468 static int otx_cpt_skcipher_ecb_aes_setkey(struct crypto_skcipher *tfm,
469 const u8 *key, u32 keylen)
470 {
471 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_ECB);
472 }
473
474 static int otx_cpt_skcipher_cfb_aes_setkey(struct crypto_skcipher *tfm,
475 const u8 *key, u32 keylen)
476 {
477 return cpt_aes_setkey(tfm, key, keylen, OTX_CPT_AES_CFB);
478 }
479
480 static int otx_cpt_skcipher_cbc_des3_setkey(struct crypto_skcipher *tfm,
481 const u8 *key, u32 keylen)
482 {
483 return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_CBC);
484 }
485
486 static int otx_cpt_skcipher_ecb_des3_setkey(struct crypto_skcipher *tfm,
487 const u8 *key, u32 keylen)
488 {
489 return cpt_des_setkey(tfm, key, keylen, OTX_CPT_DES3_ECB);
490 }
491
492 static int otx_cpt_enc_dec_init(struct crypto_skcipher *tfm)
493 {
494 struct otx_cpt_enc_ctx *ctx = crypto_skcipher_ctx(tfm);
495
496 memset(ctx, 0, sizeof(*ctx));
497 /*
498 * Additional memory for skcipher_request is
499 * allocated since the cryptd daemon uses
500 * this memory for request_ctx information
501 */
502 crypto_skcipher_set_reqsize(tfm, sizeof(struct otx_cpt_req_ctx) +
503 sizeof(struct skcipher_request));
504
505 return 0;
506 }
507
508 static int cpt_aead_init(struct crypto_aead *tfm, u8 cipher_type, u8 mac_type)
509 {
510 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
511
512 ctx->cipher_type = cipher_type;
513 ctx->mac_type = mac_type;
514
515 /*
516 * When selected cipher is NULL we use HMAC opcode instead of
517 * FLEXICRYPTO opcode therefore we don't need to use HASH algorithms
518 * for calculating ipad and opad
519 */
520 if (ctx->cipher_type != OTX_CPT_CIPHER_NULL) {
521 switch (ctx->mac_type) {
522 case OTX_CPT_SHA1:
523 ctx->hashalg = crypto_alloc_shash("sha1", 0,
524 CRYPTO_ALG_ASYNC);
525 if (IS_ERR(ctx->hashalg))
526 return PTR_ERR(ctx->hashalg);
527 break;
528
529 case OTX_CPT_SHA256:
530 ctx->hashalg = crypto_alloc_shash("sha256", 0,
531 CRYPTO_ALG_ASYNC);
532 if (IS_ERR(ctx->hashalg))
533 return PTR_ERR(ctx->hashalg);
534 break;
535
536 case OTX_CPT_SHA384:
537 ctx->hashalg = crypto_alloc_shash("sha384", 0,
538 CRYPTO_ALG_ASYNC);
539 if (IS_ERR(ctx->hashalg))
540 return PTR_ERR(ctx->hashalg);
541 break;
542
543 case OTX_CPT_SHA512:
544 ctx->hashalg = crypto_alloc_shash("sha512", 0,
545 CRYPTO_ALG_ASYNC);
546 if (IS_ERR(ctx->hashalg))
547 return PTR_ERR(ctx->hashalg);
548 break;
549 }
550 }
551
552 crypto_aead_set_reqsize(tfm, sizeof(struct otx_cpt_req_ctx));
553
554 return 0;
555 }
556
557 static int otx_cpt_aead_cbc_aes_sha1_init(struct crypto_aead *tfm)
558 {
559 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA1);
560 }
561
562 static int otx_cpt_aead_cbc_aes_sha256_init(struct crypto_aead *tfm)
563 {
564 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA256);
565 }
566
567 static int otx_cpt_aead_cbc_aes_sha384_init(struct crypto_aead *tfm)
568 {
569 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA384);
570 }
571
572 static int otx_cpt_aead_cbc_aes_sha512_init(struct crypto_aead *tfm)
573 {
574 return cpt_aead_init(tfm, OTX_CPT_AES_CBC, OTX_CPT_SHA512);
575 }
576
577 static int otx_cpt_aead_ecb_null_sha1_init(struct crypto_aead *tfm)
578 {
579 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA1);
580 }
581
582 static int otx_cpt_aead_ecb_null_sha256_init(struct crypto_aead *tfm)
583 {
584 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA256);
585 }
586
587 static int otx_cpt_aead_ecb_null_sha384_init(struct crypto_aead *tfm)
588 {
589 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA384);
590 }
591
592 static int otx_cpt_aead_ecb_null_sha512_init(struct crypto_aead *tfm)
593 {
594 return cpt_aead_init(tfm, OTX_CPT_CIPHER_NULL, OTX_CPT_SHA512);
595 }
596
597 static int otx_cpt_aead_gcm_aes_init(struct crypto_aead *tfm)
598 {
599 return cpt_aead_init(tfm, OTX_CPT_AES_GCM, OTX_CPT_MAC_NULL);
600 }
601
602 static void otx_cpt_aead_exit(struct crypto_aead *tfm)
603 {
604 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
605
606 kfree(ctx->ipad);
607 kfree(ctx->opad);
608 if (ctx->hashalg)
609 crypto_free_shash(ctx->hashalg);
610 kfree(ctx->sdesc);
611 }
612
613 /*
614 * This is the Integrity Check Value validation (aka the authentication tag
615 * length)
616 */
617 static int otx_cpt_aead_set_authsize(struct crypto_aead *tfm,
618 unsigned int authsize)
619 {
620 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
621
622 switch (ctx->mac_type) {
623 case OTX_CPT_SHA1:
624 if (authsize != SHA1_DIGEST_SIZE &&
625 authsize != SHA1_TRUNC_DIGEST_SIZE)
626 return -EINVAL;
627
628 if (authsize == SHA1_TRUNC_DIGEST_SIZE)
629 ctx->is_trunc_hmac = true;
630 break;
631
632 case OTX_CPT_SHA256:
633 if (authsize != SHA256_DIGEST_SIZE &&
634 authsize != SHA256_TRUNC_DIGEST_SIZE)
635 return -EINVAL;
636
637 if (authsize == SHA256_TRUNC_DIGEST_SIZE)
638 ctx->is_trunc_hmac = true;
639 break;
640
641 case OTX_CPT_SHA384:
642 if (authsize != SHA384_DIGEST_SIZE &&
643 authsize != SHA384_TRUNC_DIGEST_SIZE)
644 return -EINVAL;
645
646 if (authsize == SHA384_TRUNC_DIGEST_SIZE)
647 ctx->is_trunc_hmac = true;
648 break;
649
650 case OTX_CPT_SHA512:
651 if (authsize != SHA512_DIGEST_SIZE &&
652 authsize != SHA512_TRUNC_DIGEST_SIZE)
653 return -EINVAL;
654
655 if (authsize == SHA512_TRUNC_DIGEST_SIZE)
656 ctx->is_trunc_hmac = true;
657 break;
658
659 case OTX_CPT_MAC_NULL:
660 if (ctx->cipher_type == OTX_CPT_AES_GCM) {
661 if (authsize != AES_GCM_ICV_SIZE)
662 return -EINVAL;
663 } else
664 return -EINVAL;
665 break;
666
667 default:
668 return -EINVAL;
669 }
670
671 tfm->authsize = authsize;
672 return 0;
673 }
674
675 static struct otx_cpt_sdesc *alloc_sdesc(struct crypto_shash *alg)
676 {
677 struct otx_cpt_sdesc *sdesc;
678 int size;
679
680 size = sizeof(struct shash_desc) + crypto_shash_descsize(alg);
681 sdesc = kmalloc(size, GFP_KERNEL);
682 if (!sdesc)
683 return NULL;
684
685 sdesc->shash.tfm = alg;
686
687 return sdesc;
688 }
689
690 static inline void swap_data32(void *buf, u32 len)
691 {
692 u32 *store = (u32 *) buf;
693 int i = 0;
694
695 for (i = 0 ; i < len/sizeof(u32); i++, store++)
696 *store = cpu_to_be32(*store);
697 }
698
699 static inline void swap_data64(void *buf, u32 len)
700 {
701 u64 *store = (u64 *) buf;
702 int i = 0;
703
704 for (i = 0 ; i < len/sizeof(u64); i++, store++)
705 *store = cpu_to_be64(*store);
706 }
707
708 static int copy_pad(u8 mac_type, u8 *out_pad, u8 *in_pad)
709 {
710 struct sha512_state *sha512;
711 struct sha256_state *sha256;
712 struct sha1_state *sha1;
713
714 switch (mac_type) {
715 case OTX_CPT_SHA1:
716 sha1 = (struct sha1_state *) in_pad;
717 swap_data32(sha1->state, SHA1_DIGEST_SIZE);
718 memcpy(out_pad, &sha1->state, SHA1_DIGEST_SIZE);
719 break;
720
721 case OTX_CPT_SHA256:
722 sha256 = (struct sha256_state *) in_pad;
723 swap_data32(sha256->state, SHA256_DIGEST_SIZE);
724 memcpy(out_pad, &sha256->state, SHA256_DIGEST_SIZE);
725 break;
726
727 case OTX_CPT_SHA384:
728 case OTX_CPT_SHA512:
729 sha512 = (struct sha512_state *) in_pad;
730 swap_data64(sha512->state, SHA512_DIGEST_SIZE);
731 memcpy(out_pad, &sha512->state, SHA512_DIGEST_SIZE);
732 break;
733
734 default:
735 return -EINVAL;
736 }
737
738 return 0;
739 }
740
741 static int aead_hmac_init(struct crypto_aead *cipher)
742 {
743 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
744 int state_size = crypto_shash_statesize(ctx->hashalg);
745 int ds = crypto_shash_digestsize(ctx->hashalg);
746 int bs = crypto_shash_blocksize(ctx->hashalg);
747 int authkeylen = ctx->auth_key_len;
748 u8 *ipad = NULL, *opad = NULL;
749 int ret = 0, icount = 0;
750
751 ctx->sdesc = alloc_sdesc(ctx->hashalg);
752 if (!ctx->sdesc)
753 return -ENOMEM;
754
755 ctx->ipad = kzalloc(bs, GFP_KERNEL);
756 if (!ctx->ipad) {
757 ret = -ENOMEM;
758 goto calc_fail;
759 }
760
761 ctx->opad = kzalloc(bs, GFP_KERNEL);
762 if (!ctx->opad) {
763 ret = -ENOMEM;
764 goto calc_fail;
765 }
766
767 ipad = kzalloc(state_size, GFP_KERNEL);
768 if (!ipad) {
769 ret = -ENOMEM;
770 goto calc_fail;
771 }
772
773 opad = kzalloc(state_size, GFP_KERNEL);
774 if (!opad) {
775 ret = -ENOMEM;
776 goto calc_fail;
777 }
778
779 if (authkeylen > bs) {
780 ret = crypto_shash_digest(&ctx->sdesc->shash, ctx->key,
781 authkeylen, ipad);
782 if (ret)
783 goto calc_fail;
784
785 authkeylen = ds;
786 } else {
787 memcpy(ipad, ctx->key, authkeylen);
788 }
789
790 memset(ipad + authkeylen, 0, bs - authkeylen);
791 memcpy(opad, ipad, bs);
792
793 for (icount = 0; icount < bs; icount++) {
794 ipad[icount] ^= 0x36;
795 opad[icount] ^= 0x5c;
796 }
797
798 /*
799 * Partial Hash calculated from the software
800 * algorithm is retrieved for IPAD & OPAD
801 */
802
803 /* IPAD Calculation */
804 crypto_shash_init(&ctx->sdesc->shash);
805 crypto_shash_update(&ctx->sdesc->shash, ipad, bs);
806 crypto_shash_export(&ctx->sdesc->shash, ipad);
807 ret = copy_pad(ctx->mac_type, ctx->ipad, ipad);
808 if (ret)
809 goto calc_fail;
810
811 /* OPAD Calculation */
812 crypto_shash_init(&ctx->sdesc->shash);
813 crypto_shash_update(&ctx->sdesc->shash, opad, bs);
814 crypto_shash_export(&ctx->sdesc->shash, opad);
815 ret = copy_pad(ctx->mac_type, ctx->opad, opad);
816 if (ret)
817 goto calc_fail;
818
819 kfree(ipad);
820 kfree(opad);
821
822 return 0;
823
824 calc_fail:
825 kfree(ctx->ipad);
826 ctx->ipad = NULL;
827 kfree(ctx->opad);
828 ctx->opad = NULL;
829 kfree(ipad);
830 kfree(opad);
831 kfree(ctx->sdesc);
832 ctx->sdesc = NULL;
833
834 return ret;
835 }
836
837 static int otx_cpt_aead_cbc_aes_sha_setkey(struct crypto_aead *cipher,
838 const unsigned char *key,
839 unsigned int keylen)
840 {
841 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
842 struct crypto_authenc_key_param *param;
843 int enckeylen = 0, authkeylen = 0;
844 struct rtattr *rta = (void *)key;
845 int status = -EINVAL;
846
847 if (!RTA_OK(rta, keylen))
848 goto badkey;
849
850 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
851 goto badkey;
852
853 if (RTA_PAYLOAD(rta) < sizeof(*param))
854 goto badkey;
855
856 param = RTA_DATA(rta);
857 enckeylen = be32_to_cpu(param->enckeylen);
858 key += RTA_ALIGN(rta->rta_len);
859 keylen -= RTA_ALIGN(rta->rta_len);
860 if (keylen < enckeylen)
861 goto badkey;
862
863 if (keylen > OTX_CPT_MAX_KEY_SIZE)
864 goto badkey;
865
866 authkeylen = keylen - enckeylen;
867 memcpy(ctx->key, key, keylen);
868
869 switch (enckeylen) {
870 case AES_KEYSIZE_128:
871 ctx->key_type = OTX_CPT_AES_128_BIT;
872 break;
873 case AES_KEYSIZE_192:
874 ctx->key_type = OTX_CPT_AES_192_BIT;
875 break;
876 case AES_KEYSIZE_256:
877 ctx->key_type = OTX_CPT_AES_256_BIT;
878 break;
879 default:
880 /* Invalid key length */
881 goto badkey;
882 }
883
884 ctx->enc_key_len = enckeylen;
885 ctx->auth_key_len = authkeylen;
886
887 status = aead_hmac_init(cipher);
888 if (status)
889 goto badkey;
890
891 return 0;
892 badkey:
893 return status;
894 }
895
896 static int otx_cpt_aead_ecb_null_sha_setkey(struct crypto_aead *cipher,
897 const unsigned char *key,
898 unsigned int keylen)
899 {
900 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
901 struct crypto_authenc_key_param *param;
902 struct rtattr *rta = (void *)key;
903 int enckeylen = 0;
904
905 if (!RTA_OK(rta, keylen))
906 goto badkey;
907
908 if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
909 goto badkey;
910
911 if (RTA_PAYLOAD(rta) < sizeof(*param))
912 goto badkey;
913
914 param = RTA_DATA(rta);
915 enckeylen = be32_to_cpu(param->enckeylen);
916 key += RTA_ALIGN(rta->rta_len);
917 keylen -= RTA_ALIGN(rta->rta_len);
918 if (enckeylen != 0)
919 goto badkey;
920
921 if (keylen > OTX_CPT_MAX_KEY_SIZE)
922 goto badkey;
923
924 memcpy(ctx->key, key, keylen);
925 ctx->enc_key_len = enckeylen;
926 ctx->auth_key_len = keylen;
927 return 0;
928 badkey:
929 return -EINVAL;
930 }
931
932 static int otx_cpt_aead_gcm_aes_setkey(struct crypto_aead *cipher,
933 const unsigned char *key,
934 unsigned int keylen)
935 {
936 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(cipher);
937
938 /*
939 * For aes gcm we expect to get encryption key (16, 24, 32 bytes)
940 * and salt (4 bytes)
941 */
942 switch (keylen) {
943 case AES_KEYSIZE_128 + AES_GCM_SALT_SIZE:
944 ctx->key_type = OTX_CPT_AES_128_BIT;
945 ctx->enc_key_len = AES_KEYSIZE_128;
946 break;
947 case AES_KEYSIZE_192 + AES_GCM_SALT_SIZE:
948 ctx->key_type = OTX_CPT_AES_192_BIT;
949 ctx->enc_key_len = AES_KEYSIZE_192;
950 break;
951 case AES_KEYSIZE_256 + AES_GCM_SALT_SIZE:
952 ctx->key_type = OTX_CPT_AES_256_BIT;
953 ctx->enc_key_len = AES_KEYSIZE_256;
954 break;
955 default:
956 /* Invalid key and salt length */
957 return -EINVAL;
958 }
959
960 /* Store encryption key and salt */
961 memcpy(ctx->key, key, keylen);
962
963 return 0;
964 }
965
966 static inline u32 create_aead_ctx_hdr(struct aead_request *req, u32 enc,
967 u32 *argcnt)
968 {
969 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
970 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
971 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
972 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
973 struct otx_cpt_fc_ctx *fctx = &rctx->fctx;
974 int mac_len = crypto_aead_authsize(tfm);
975 int ds;
976
977 rctx->ctrl_word.e.enc_data_offset = req->assoclen;
978
979 switch (ctx->cipher_type) {
980 case OTX_CPT_AES_CBC:
981 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_CPTR;
982 /* Copy encryption key to context */
983 memcpy(fctx->enc.encr_key, ctx->key + ctx->auth_key_len,
984 ctx->enc_key_len);
985 /* Copy IV to context */
986 memcpy(fctx->enc.encr_iv, req->iv, crypto_aead_ivsize(tfm));
987
988 ds = crypto_shash_digestsize(ctx->hashalg);
989 if (ctx->mac_type == OTX_CPT_SHA384)
990 ds = SHA512_DIGEST_SIZE;
991 if (ctx->ipad)
992 memcpy(fctx->hmac.e.ipad, ctx->ipad, ds);
993 if (ctx->opad)
994 memcpy(fctx->hmac.e.opad, ctx->opad, ds);
995 break;
996
997 case OTX_CPT_AES_GCM:
998 fctx->enc.enc_ctrl.e.iv_source = OTX_CPT_FROM_DPTR;
999 /* Copy encryption key to context */
1000 memcpy(fctx->enc.encr_key, ctx->key, ctx->enc_key_len);
1001 /* Copy salt to context */
1002 memcpy(fctx->enc.encr_iv, ctx->key + ctx->enc_key_len,
1003 AES_GCM_SALT_SIZE);
1004
1005 rctx->ctrl_word.e.iv_offset = req->assoclen - AES_GCM_IV_OFFSET;
1006 break;
1007
1008 default:
1009 /* Unknown cipher type */
1010 return -EINVAL;
1011 }
1012 rctx->ctrl_word.flags = cpu_to_be64(rctx->ctrl_word.flags);
1013
1014 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
1015 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;
1016 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_FC |
1017 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);
1018 if (enc) {
1019 req_info->req.opcode.s.minor = 2;
1020 req_info->req.param1 = req->cryptlen;
1021 req_info->req.param2 = req->cryptlen + req->assoclen;
1022 } else {
1023 req_info->req.opcode.s.minor = 3;
1024 req_info->req.param1 = req->cryptlen - mac_len;
1025 req_info->req.param2 = req->cryptlen + req->assoclen - mac_len;
1026 }
1027
1028 fctx->enc.enc_ctrl.e.enc_cipher = ctx->cipher_type;
1029 fctx->enc.enc_ctrl.e.aes_key = ctx->key_type;
1030 fctx->enc.enc_ctrl.e.mac_type = ctx->mac_type;
1031 fctx->enc.enc_ctrl.e.mac_len = mac_len;
1032 fctx->enc.enc_ctrl.flags = cpu_to_be64(fctx->enc.enc_ctrl.flags);
1033
1034 /*
1035 * Storing Packet Data Information in offset
1036 * Control Word First 8 bytes
1037 */
1038 req_info->in[*argcnt].vptr = (u8 *)&rctx->ctrl_word;
1039 req_info->in[*argcnt].size = CONTROL_WORD_LEN;
1040 req_info->req.dlen += CONTROL_WORD_LEN;
1041 ++(*argcnt);
1042
1043 req_info->in[*argcnt].vptr = (u8 *)fctx;
1044 req_info->in[*argcnt].size = sizeof(struct otx_cpt_fc_ctx);
1045 req_info->req.dlen += sizeof(struct otx_cpt_fc_ctx);
1046 ++(*argcnt);
1047
1048 return 0;
1049 }
1050
1051 static inline u32 create_hmac_ctx_hdr(struct aead_request *req, u32 *argcnt,
1052 u32 enc)
1053 {
1054 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
1055 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1056 struct otx_cpt_aead_ctx *ctx = crypto_aead_ctx(tfm);
1057 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1058
1059 req_info->ctrl.s.dma_mode = OTX_CPT_DMA_GATHER_SCATTER;
1060 req_info->ctrl.s.se_req = OTX_CPT_SE_CORE_REQ;
1061 req_info->req.opcode.s.major = OTX_CPT_MAJOR_OP_HMAC |
1062 DMA_MODE_FLAG(OTX_CPT_DMA_GATHER_SCATTER);
1063 req_info->is_trunc_hmac = ctx->is_trunc_hmac;
1064
1065 req_info->req.opcode.s.minor = 0;
1066 req_info->req.param1 = ctx->auth_key_len;
1067 req_info->req.param2 = ctx->mac_type << 8;
1068
1069 /* Add authentication key */
1070 req_info->in[*argcnt].vptr = ctx->key;
1071 req_info->in[*argcnt].size = round_up(ctx->auth_key_len, 8);
1072 req_info->req.dlen += round_up(ctx->auth_key_len, 8);
1073 ++(*argcnt);
1074
1075 return 0;
1076 }
1077
1078 static inline u32 create_aead_input_list(struct aead_request *req, u32 enc)
1079 {
1080 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
1081 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1082 u32 inputlen = req->cryptlen + req->assoclen;
1083 u32 status, argcnt = 0;
1084
1085 status = create_aead_ctx_hdr(req, enc, &argcnt);
1086 if (status)
1087 return status;
1088 update_input_data(req_info, req->src, inputlen, &argcnt);
1089 req_info->incnt = argcnt;
1090
1091 return 0;
1092 }
1093
1094 static inline u32 create_aead_output_list(struct aead_request *req, u32 enc,
1095 u32 mac_len)
1096 {
1097 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
1098 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1099 u32 argcnt = 0, outputlen = 0;
1100
1101 if (enc)
1102 outputlen = req->cryptlen + req->assoclen + mac_len;
1103 else
1104 outputlen = req->cryptlen + req->assoclen - mac_len;
1105
1106 update_output_data(req_info, req->dst, 0, outputlen, &argcnt);
1107 req_info->outcnt = argcnt;
1108
1109 return 0;
1110 }
1111
1112 static inline u32 create_aead_null_input_list(struct aead_request *req,
1113 u32 enc, u32 mac_len)
1114 {
1115 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
1116 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1117 u32 inputlen, argcnt = 0;
1118
1119 if (enc)
1120 inputlen = req->cryptlen + req->assoclen;
1121 else
1122 inputlen = req->cryptlen + req->assoclen - mac_len;
1123
1124 create_hmac_ctx_hdr(req, &argcnt, enc);
1125 update_input_data(req_info, req->src, inputlen, &argcnt);
1126 req_info->incnt = argcnt;
1127
1128 return 0;
1129 }
1130
1131 static inline u32 create_aead_null_output_list(struct aead_request *req,
1132 u32 enc, u32 mac_len)
1133 {
1134 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
1135 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1136 struct scatterlist *dst;
1137 u8 *ptr = NULL;
1138 int argcnt = 0, status, offset;
1139 u32 inputlen;
1140
1141 if (enc)
1142 inputlen = req->cryptlen + req->assoclen;
1143 else
1144 inputlen = req->cryptlen + req->assoclen - mac_len;
1145
1146 /*
1147 * If source and destination are different
1148 * then copy payload to destination
1149 */
1150 if (req->src != req->dst) {
1151
1152 ptr = kmalloc(inputlen, (req_info->areq->flags &
1153 CRYPTO_TFM_REQ_MAY_SLEEP) ?
1154 GFP_KERNEL : GFP_ATOMIC);
1155 if (!ptr) {
1156 status = -ENOMEM;
1157 goto error;
1158 }
1159
1160 status = sg_copy_to_buffer(req->src, sg_nents(req->src), ptr,
1161 inputlen);
1162 if (status != inputlen) {
1163 status = -EINVAL;
1164 goto error_free;
1165 }
1166 status = sg_copy_from_buffer(req->dst, sg_nents(req->dst), ptr,
1167 inputlen);
1168 if (status != inputlen) {
1169 status = -EINVAL;
1170 goto error_free;
1171 }
1172 kfree(ptr);
1173 }
1174
1175 if (enc) {
1176 /*
1177 * In an encryption scenario hmac needs
1178 * to be appended after payload
1179 */
1180 dst = req->dst;
1181 offset = inputlen;
1182 while (offset >= dst->length) {
1183 offset -= dst->length;
1184 dst = sg_next(dst);
1185 if (!dst) {
1186 status = -ENOENT;
1187 goto error;
1188 }
1189 }
1190
1191 update_output_data(req_info, dst, offset, mac_len, &argcnt);
1192 } else {
1193 /*
1194 * In a decryption scenario calculated hmac for received
1195 * payload needs to be compare with hmac received
1196 */
1197 status = sg_copy_buffer(req->src, sg_nents(req->src),
1198 rctx->fctx.hmac.s.hmac_recv, mac_len,
1199 inputlen, true);
1200 if (status != mac_len) {
1201 status = -EINVAL;
1202 goto error;
1203 }
1204
1205 req_info->out[argcnt].vptr = rctx->fctx.hmac.s.hmac_calc;
1206 req_info->out[argcnt].size = mac_len;
1207 argcnt++;
1208 }
1209
1210 req_info->outcnt = argcnt;
1211 return 0;
1212
1213 error_free:
1214 kfree(ptr);
1215 error:
1216 return status;
1217 }
1218
1219 static u32 cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
1220 {
1221 struct otx_cpt_req_ctx *rctx = aead_request_ctx(req);
1222 struct otx_cpt_req_info *req_info = &rctx->cpt_req;
1223 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1224 struct pci_dev *pdev;
1225 u32 status, cpu_num;
1226
1227 /* Clear control words */
1228 rctx->ctrl_word.flags = 0;
1229 rctx->fctx.enc.enc_ctrl.flags = 0;
1230
1231 req_info->callback = otx_cpt_aead_callback;
1232 req_info->areq = &req->base;
1233 req_info->req_type = reg_type;
1234 req_info->is_enc = enc;
1235 req_info->is_trunc_hmac = false;
1236
1237 switch (reg_type) {
1238 case OTX_CPT_AEAD_ENC_DEC_REQ:
1239 status = create_aead_input_list(req, enc);
1240 if (status)
1241 return status;
1242 status = create_aead_output_list(req, enc,
1243 crypto_aead_authsize(tfm));
1244 if (status)
1245 return status;
1246 break;
1247
1248 case OTX_CPT_AEAD_ENC_DEC_NULL_REQ:
1249 status = create_aead_null_input_list(req, enc,
1250 crypto_aead_authsize(tfm));
1251 if (status)
1252 return status;
1253 status = create_aead_null_output_list(req, enc,
1254 crypto_aead_authsize(tfm));
1255 if (status)
1256 return status;
1257 break;
1258
1259 default:
1260 return -EINVAL;
1261 }
1262
1263 /* Validate that request doesn't exceed maximum CPT supported size */
1264 if (req_info->req.param1 > OTX_CPT_MAX_REQ_SIZE ||
1265 req_info->req.param2 > OTX_CPT_MAX_REQ_SIZE)
1266 return -E2BIG;
1267
1268 status = get_se_device(&pdev, &cpu_num);
1269 if (status)
1270 return status;
1271
1272 req_info->ctrl.s.grp = 0;
1273
1274 status = otx_cpt_do_request(pdev, req_info, cpu_num);
1275 /*
1276 * We perform an asynchronous send and once
1277 * the request is completed the driver would
1278 * intimate through registered call back functions
1279 */
1280 return status;
1281 }
1282
1283 static int otx_cpt_aead_encrypt(struct aead_request *req)
1284 {
1285 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, true);
1286 }
1287
1288 static int otx_cpt_aead_decrypt(struct aead_request *req)
1289 {
1290 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_REQ, false);
1291 }
1292
1293 static int otx_cpt_aead_null_encrypt(struct aead_request *req)
1294 {
1295 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, true);
1296 }
1297
1298 static int otx_cpt_aead_null_decrypt(struct aead_request *req)
1299 {
1300 return cpt_aead_enc_dec(req, OTX_CPT_AEAD_ENC_DEC_NULL_REQ, false);
1301 }
1302
1303 static struct skcipher_alg otx_cpt_skciphers[] = { {
1304 .base.cra_name = "xts(aes)",
1305 .base.cra_driver_name = "cpt_xts_aes",
1306 .base.cra_flags = CRYPTO_ALG_ASYNC,
1307 .base.cra_blocksize = AES_BLOCK_SIZE,
1308 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1309 .base.cra_alignmask = 7,
1310 .base.cra_priority = 4001,
1311 .base.cra_module = THIS_MODULE,
1312
1313 .init = otx_cpt_enc_dec_init,
1314 .ivsize = AES_BLOCK_SIZE,
1315 .min_keysize = 2 * AES_MIN_KEY_SIZE,
1316 .max_keysize = 2 * AES_MAX_KEY_SIZE,
1317 .setkey = otx_cpt_skcipher_xts_setkey,
1318 .encrypt = otx_cpt_skcipher_encrypt,
1319 .decrypt = otx_cpt_skcipher_decrypt,
1320 }, {
1321 .base.cra_name = "cbc(aes)",
1322 .base.cra_driver_name = "cpt_cbc_aes",
1323 .base.cra_flags = CRYPTO_ALG_ASYNC,
1324 .base.cra_blocksize = AES_BLOCK_SIZE,
1325 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1326 .base.cra_alignmask = 7,
1327 .base.cra_priority = 4001,
1328 .base.cra_module = THIS_MODULE,
1329
1330 .init = otx_cpt_enc_dec_init,
1331 .ivsize = AES_BLOCK_SIZE,
1332 .min_keysize = AES_MIN_KEY_SIZE,
1333 .max_keysize = AES_MAX_KEY_SIZE,
1334 .setkey = otx_cpt_skcipher_cbc_aes_setkey,
1335 .encrypt = otx_cpt_skcipher_encrypt,
1336 .decrypt = otx_cpt_skcipher_decrypt,
1337 }, {
1338 .base.cra_name = "ecb(aes)",
1339 .base.cra_driver_name = "cpt_ecb_aes",
1340 .base.cra_flags = CRYPTO_ALG_ASYNC,
1341 .base.cra_blocksize = AES_BLOCK_SIZE,
1342 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1343 .base.cra_alignmask = 7,
1344 .base.cra_priority = 4001,
1345 .base.cra_module = THIS_MODULE,
1346
1347 .init = otx_cpt_enc_dec_init,
1348 .ivsize = 0,
1349 .min_keysize = AES_MIN_KEY_SIZE,
1350 .max_keysize = AES_MAX_KEY_SIZE,
1351 .setkey = otx_cpt_skcipher_ecb_aes_setkey,
1352 .encrypt = otx_cpt_skcipher_encrypt,
1353 .decrypt = otx_cpt_skcipher_decrypt,
1354 }, {
1355 .base.cra_name = "cfb(aes)",
1356 .base.cra_driver_name = "cpt_cfb_aes",
1357 .base.cra_flags = CRYPTO_ALG_ASYNC,
1358 .base.cra_blocksize = AES_BLOCK_SIZE,
1359 .base.cra_ctxsize = sizeof(struct otx_cpt_enc_ctx),
1360 .base.cra_alignmask = 7,
1361 .base.cra_priority = 4001,
1362 .base.cra_module = THIS_MODULE,
1363
1364 .init = otx_cpt_enc_dec_init,
1365 .ivsize = AES_BLOCK_SIZE,
1366 .min_keysize = AES_MIN_KEY_SIZE,
1367 .max_keysize = AES_MAX_KEY_SIZE,
1368 .setkey = otx_cpt_skcipher_cfb_aes_setkey,
1369 .encrypt = otx_cpt_skcipher_encrypt,
1370 .decrypt = otx_cpt_skcipher_decrypt,
1371 }, {
1372 .base.cra_name = "cbc(des3_ede)",
1373 .base.cra_driver_name = "cpt_cbc_des3_ede",
1374 .base.cra_flags = CRYPTO_ALG_ASYNC,
1375 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1376 .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx),
1377 .base.cra_alignmask = 7,
1378 .base.cra_priority = 4001,
1379 .base.cra_module = THIS_MODULE,
1380
1381 .init = otx_cpt_enc_dec_init,
1382 .min_keysize = DES3_EDE_KEY_SIZE,
1383 .max_keysize = DES3_EDE_KEY_SIZE,
1384 .ivsize = DES_BLOCK_SIZE,
1385 .setkey = otx_cpt_skcipher_cbc_des3_setkey,
1386 .encrypt = otx_cpt_skcipher_encrypt,
1387 .decrypt = otx_cpt_skcipher_decrypt,
1388 }, {
1389 .base.cra_name = "ecb(des3_ede)",
1390 .base.cra_driver_name = "cpt_ecb_des3_ede",
1391 .base.cra_flags = CRYPTO_ALG_ASYNC,
1392 .base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
1393 .base.cra_ctxsize = sizeof(struct otx_cpt_des3_ctx),
1394 .base.cra_alignmask = 7,
1395 .base.cra_priority = 4001,
1396 .base.cra_module = THIS_MODULE,
1397
1398 .init = otx_cpt_enc_dec_init,
1399 .min_keysize = DES3_EDE_KEY_SIZE,
1400 .max_keysize = DES3_EDE_KEY_SIZE,
1401 .ivsize = 0,
1402 .setkey = otx_cpt_skcipher_ecb_des3_setkey,
1403 .encrypt = otx_cpt_skcipher_encrypt,
1404 .decrypt = otx_cpt_skcipher_decrypt,
1405 } };
1406
1407 static struct aead_alg otx_cpt_aeads[] = { {
1408 .base = {
1409 .cra_name = "authenc(hmac(sha1),cbc(aes))",
1410 .cra_driver_name = "cpt_hmac_sha1_cbc_aes",
1411 .cra_blocksize = AES_BLOCK_SIZE,
1412 .cra_flags = CRYPTO_ALG_ASYNC,
1413 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1414 .cra_priority = 4001,
1415 .cra_alignmask = 0,
1416 .cra_module = THIS_MODULE,
1417 },
1418 .init = otx_cpt_aead_cbc_aes_sha1_init,
1419 .exit = otx_cpt_aead_exit,
1420 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1421 .setauthsize = otx_cpt_aead_set_authsize,
1422 .encrypt = otx_cpt_aead_encrypt,
1423 .decrypt = otx_cpt_aead_decrypt,
1424 .ivsize = AES_BLOCK_SIZE,
1425 .maxauthsize = SHA1_DIGEST_SIZE,
1426 }, {
1427 .base = {
1428 .cra_name = "authenc(hmac(sha256),cbc(aes))",
1429 .cra_driver_name = "cpt_hmac_sha256_cbc_aes",
1430 .cra_blocksize = AES_BLOCK_SIZE,
1431 .cra_flags = CRYPTO_ALG_ASYNC,
1432 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1433 .cra_priority = 4001,
1434 .cra_alignmask = 0,
1435 .cra_module = THIS_MODULE,
1436 },
1437 .init = otx_cpt_aead_cbc_aes_sha256_init,
1438 .exit = otx_cpt_aead_exit,
1439 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1440 .setauthsize = otx_cpt_aead_set_authsize,
1441 .encrypt = otx_cpt_aead_encrypt,
1442 .decrypt = otx_cpt_aead_decrypt,
1443 .ivsize = AES_BLOCK_SIZE,
1444 .maxauthsize = SHA256_DIGEST_SIZE,
1445 }, {
1446 .base = {
1447 .cra_name = "authenc(hmac(sha384),cbc(aes))",
1448 .cra_driver_name = "cpt_hmac_sha384_cbc_aes",
1449 .cra_blocksize = AES_BLOCK_SIZE,
1450 .cra_flags = CRYPTO_ALG_ASYNC,
1451 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1452 .cra_priority = 4001,
1453 .cra_alignmask = 0,
1454 .cra_module = THIS_MODULE,
1455 },
1456 .init = otx_cpt_aead_cbc_aes_sha384_init,
1457 .exit = otx_cpt_aead_exit,
1458 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1459 .setauthsize = otx_cpt_aead_set_authsize,
1460 .encrypt = otx_cpt_aead_encrypt,
1461 .decrypt = otx_cpt_aead_decrypt,
1462 .ivsize = AES_BLOCK_SIZE,
1463 .maxauthsize = SHA384_DIGEST_SIZE,
1464 }, {
1465 .base = {
1466 .cra_name = "authenc(hmac(sha512),cbc(aes))",
1467 .cra_driver_name = "cpt_hmac_sha512_cbc_aes",
1468 .cra_blocksize = AES_BLOCK_SIZE,
1469 .cra_flags = CRYPTO_ALG_ASYNC,
1470 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1471 .cra_priority = 4001,
1472 .cra_alignmask = 0,
1473 .cra_module = THIS_MODULE,
1474 },
1475 .init = otx_cpt_aead_cbc_aes_sha512_init,
1476 .exit = otx_cpt_aead_exit,
1477 .setkey = otx_cpt_aead_cbc_aes_sha_setkey,
1478 .setauthsize = otx_cpt_aead_set_authsize,
1479 .encrypt = otx_cpt_aead_encrypt,
1480 .decrypt = otx_cpt_aead_decrypt,
1481 .ivsize = AES_BLOCK_SIZE,
1482 .maxauthsize = SHA512_DIGEST_SIZE,
1483 }, {
1484 .base = {
1485 .cra_name = "authenc(hmac(sha1),ecb(cipher_null))",
1486 .cra_driver_name = "cpt_hmac_sha1_ecb_null",
1487 .cra_blocksize = 1,
1488 .cra_flags = CRYPTO_ALG_ASYNC,
1489 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1490 .cra_priority = 4001,
1491 .cra_alignmask = 0,
1492 .cra_module = THIS_MODULE,
1493 },
1494 .init = otx_cpt_aead_ecb_null_sha1_init,
1495 .exit = otx_cpt_aead_exit,
1496 .setkey = otx_cpt_aead_ecb_null_sha_setkey,
1497 .setauthsize = otx_cpt_aead_set_authsize,
1498 .encrypt = otx_cpt_aead_null_encrypt,
1499 .decrypt = otx_cpt_aead_null_decrypt,
1500 .ivsize = 0,
1501 .maxauthsize = SHA1_DIGEST_SIZE,
1502 }, {
1503 .base = {
1504 .cra_name = "authenc(hmac(sha256),ecb(cipher_null))",
1505 .cra_driver_name = "cpt_hmac_sha256_ecb_null",
1506 .cra_blocksize = 1,
1507 .cra_flags = CRYPTO_ALG_ASYNC,
1508 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1509 .cra_priority = 4001,
1510 .cra_alignmask = 0,
1511 .cra_module = THIS_MODULE,
1512 },
1513 .init = otx_cpt_aead_ecb_null_sha256_init,
1514 .exit = otx_cpt_aead_exit,
1515 .setkey = otx_cpt_aead_ecb_null_sha_setkey,
1516 .setauthsize = otx_cpt_aead_set_authsize,
1517 .encrypt = otx_cpt_aead_null_encrypt,
1518 .decrypt = otx_cpt_aead_null_decrypt,
1519 .ivsize = 0,
1520 .maxauthsize = SHA256_DIGEST_SIZE,
1521 }, {
1522 .base = {
1523 .cra_name = "authenc(hmac(sha384),ecb(cipher_null))",
1524 .cra_driver_name = "cpt_hmac_sha384_ecb_null",
1525 .cra_blocksize = 1,
1526 .cra_flags = CRYPTO_ALG_ASYNC,
1527 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1528 .cra_priority = 4001,
1529 .cra_alignmask = 0,
1530 .cra_module = THIS_MODULE,
1531 },
1532 .init = otx_cpt_aead_ecb_null_sha384_init,
1533 .exit = otx_cpt_aead_exit,
1534 .setkey = otx_cpt_aead_ecb_null_sha_setkey,
1535 .setauthsize = otx_cpt_aead_set_authsize,
1536 .encrypt = otx_cpt_aead_null_encrypt,
1537 .decrypt = otx_cpt_aead_null_decrypt,
1538 .ivsize = 0,
1539 .maxauthsize = SHA384_DIGEST_SIZE,
1540 }, {
1541 .base = {
1542 .cra_name = "authenc(hmac(sha512),ecb(cipher_null))",
1543 .cra_driver_name = "cpt_hmac_sha512_ecb_null",
1544 .cra_blocksize = 1,
1545 .cra_flags = CRYPTO_ALG_ASYNC,
1546 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1547 .cra_priority = 4001,
1548 .cra_alignmask = 0,
1549 .cra_module = THIS_MODULE,
1550 },
1551 .init = otx_cpt_aead_ecb_null_sha512_init,
1552 .exit = otx_cpt_aead_exit,
1553 .setkey = otx_cpt_aead_ecb_null_sha_setkey,
1554 .setauthsize = otx_cpt_aead_set_authsize,
1555 .encrypt = otx_cpt_aead_null_encrypt,
1556 .decrypt = otx_cpt_aead_null_decrypt,
1557 .ivsize = 0,
1558 .maxauthsize = SHA512_DIGEST_SIZE,
1559 }, {
1560 .base = {
1561 .cra_name = "rfc4106(gcm(aes))",
1562 .cra_driver_name = "cpt_rfc4106_gcm_aes",
1563 .cra_blocksize = 1,
1564 .cra_flags = CRYPTO_ALG_ASYNC,
1565 .cra_ctxsize = sizeof(struct otx_cpt_aead_ctx),
1566 .cra_priority = 4001,
1567 .cra_alignmask = 0,
1568 .cra_module = THIS_MODULE,
1569 },
1570 .init = otx_cpt_aead_gcm_aes_init,
1571 .exit = otx_cpt_aead_exit,
1572 .setkey = otx_cpt_aead_gcm_aes_setkey,
1573 .setauthsize = otx_cpt_aead_set_authsize,
1574 .encrypt = otx_cpt_aead_encrypt,
1575 .decrypt = otx_cpt_aead_decrypt,
1576 .ivsize = AES_GCM_IV_SIZE,
1577 .maxauthsize = AES_GCM_ICV_SIZE,
1578 } };
1579
1580 static inline int is_any_alg_used(void)
1581 {
1582 int i;
1583
1584 for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++)
1585 if (refcount_read(&otx_cpt_skciphers[i].base.cra_refcnt) != 1)
1586 return true;
1587 for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++)
1588 if (refcount_read(&otx_cpt_aeads[i].base.cra_refcnt) != 1)
1589 return true;
1590 return false;
1591 }
1592
1593 static inline int cpt_register_algs(void)
1594 {
1595 int i, err = 0;
1596
1597 if (!IS_ENABLED(CONFIG_DM_CRYPT)) {
1598 for (i = 0; i < ARRAY_SIZE(otx_cpt_skciphers); i++)
1599 otx_cpt_skciphers[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1600
1601 err = crypto_register_skciphers(otx_cpt_skciphers,
1602 ARRAY_SIZE(otx_cpt_skciphers));
1603 if (err)
1604 return err;
1605 }
1606
1607 for (i = 0; i < ARRAY_SIZE(otx_cpt_aeads); i++)
1608 otx_cpt_aeads[i].base.cra_flags &= ~CRYPTO_ALG_DEAD;
1609
1610 err = crypto_register_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads));
1611 if (err) {
1612 crypto_unregister_skciphers(otx_cpt_skciphers,
1613 ARRAY_SIZE(otx_cpt_skciphers));
1614 return err;
1615 }
1616
1617 return 0;
1618 }
1619
1620 static inline void cpt_unregister_algs(void)
1621 {
1622 crypto_unregister_skciphers(otx_cpt_skciphers,
1623 ARRAY_SIZE(otx_cpt_skciphers));
1624 crypto_unregister_aeads(otx_cpt_aeads, ARRAY_SIZE(otx_cpt_aeads));
1625 }
1626
1627 static int compare_func(const void *lptr, const void *rptr)
1628 {
1629 struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1630 struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1631
1632 if (ldesc->dev->devfn < rdesc->dev->devfn)
1633 return -1;
1634 if (ldesc->dev->devfn > rdesc->dev->devfn)
1635 return 1;
1636 return 0;
1637 }
1638
1639 static void swap_func(void *lptr, void *rptr, int size)
1640 {
1641 struct cpt_device_desc *ldesc = (struct cpt_device_desc *) lptr;
1642 struct cpt_device_desc *rdesc = (struct cpt_device_desc *) rptr;
1643 struct cpt_device_desc desc;
1644
1645 desc = *ldesc;
1646 *ldesc = *rdesc;
1647 *rdesc = desc;
1648 }
1649
1650 int otx_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
1651 enum otx_cptpf_type pf_type,
1652 enum otx_cptvf_type engine_type,
1653 int num_queues, int num_devices)
1654 {
1655 int ret = 0;
1656 int count;
1657
1658 mutex_lock(&mutex);
1659 switch (engine_type) {
1660 case OTX_CPT_SE_TYPES:
1661 count = atomic_read(&se_devices.count);
1662 if (count >= CPT_MAX_VF_NUM) {
1663 dev_err(&pdev->dev, "No space to add a new device");
1664 ret = -ENOSPC;
1665 goto err;
1666 }
1667 se_devices.desc[count].pf_type = pf_type;
1668 se_devices.desc[count].num_queues = num_queues;
1669 se_devices.desc[count++].dev = pdev;
1670 atomic_inc(&se_devices.count);
1671
1672 if (atomic_read(&se_devices.count) == num_devices &&
1673 is_crypto_registered == false) {
1674 if (cpt_register_algs()) {
1675 dev_err(&pdev->dev,
1676 "Error in registering crypto algorithms\n");
1677 ret = -EINVAL;
1678 goto err;
1679 }
1680 try_module_get(mod);
1681 is_crypto_registered = true;
1682 }
1683 sort(se_devices.desc, count, sizeof(struct cpt_device_desc),
1684 compare_func, swap_func);
1685 break;
1686
1687 case OTX_CPT_AE_TYPES:
1688 count = atomic_read(&ae_devices.count);
1689 if (count >= CPT_MAX_VF_NUM) {
1690 dev_err(&pdev->dev, "No space to a add new device");
1691 ret = -ENOSPC;
1692 goto err;
1693 }
1694 ae_devices.desc[count].pf_type = pf_type;
1695 ae_devices.desc[count].num_queues = num_queues;
1696 ae_devices.desc[count++].dev = pdev;
1697 atomic_inc(&ae_devices.count);
1698 sort(ae_devices.desc, count, sizeof(struct cpt_device_desc),
1699 compare_func, swap_func);
1700 break;
1701
1702 default:
1703 dev_err(&pdev->dev, "Unknown VF type %d\n", engine_type);
1704 ret = BAD_OTX_CPTVF_TYPE;
1705 }
1706 err:
1707 mutex_unlock(&mutex);
1708 return ret;
1709 }
1710
1711 void otx_cpt_crypto_exit(struct pci_dev *pdev, struct module *mod,
1712 enum otx_cptvf_type engine_type)
1713 {
1714 struct cpt_device_table *dev_tbl;
1715 bool dev_found = false;
1716 int i, j, count;
1717
1718 mutex_lock(&mutex);
1719
1720 dev_tbl = (engine_type == OTX_CPT_AE_TYPES) ? &ae_devices : &se_devices;
1721 count = atomic_read(&dev_tbl->count);
1722 for (i = 0; i < count; i++)
1723 if (pdev == dev_tbl->desc[i].dev) {
1724 for (j = i; j < count-1; j++)
1725 dev_tbl->desc[j] = dev_tbl->desc[j+1];
1726 dev_found = true;
1727 break;
1728 }
1729
1730 if (!dev_found) {
1731 dev_err(&pdev->dev, "%s device not found", __func__);
1732 goto exit;
1733 }
1734
1735 if (engine_type != OTX_CPT_AE_TYPES) {
1736 if (atomic_dec_and_test(&se_devices.count) &&
1737 !is_any_alg_used()) {
1738 cpt_unregister_algs();
1739 module_put(mod);
1740 is_crypto_registered = false;
1741 }
1742 } else
1743 atomic_dec(&ae_devices.count);
1744 exit:
1745 mutex_unlock(&mutex);
1746 }
Linux with added FPC-III support