?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'akpm' (patches from Andrew)
[linux/fpc-iii.git] / drivers / crypto / hisilicon / sec2 / sec_crypto.c
bloba2cfcc9ccd942e8dde7cd79d6259954b88dcbe77
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2019 HiSilicon Limited. */
3
4 #include <crypto/aes.h>
5 #include <crypto/algapi.h>
6 #include <crypto/authenc.h>
7 #include <crypto/des.h>
8 #include <crypto/hash.h>
9 #include <crypto/internal/aead.h>
10 #include <crypto/sha.h>
11 #include <crypto/skcipher.h>
12 #include <crypto/xts.h>
13 #include <linux/crypto.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/idr.h>
16
17 #include "sec.h"
18 #include "sec_crypto.h"
19
20 #define SEC_PRIORITY 4001
21 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE)
22 #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE)
23 #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE)
24 #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE)
25
26 /* SEC sqe(bd) bit operational relative MACRO */
27 #define SEC_DE_OFFSET 1
28 #define SEC_CIPHER_OFFSET 4
29 #define SEC_SCENE_OFFSET 3
30 #define SEC_DST_SGL_OFFSET 2
31 #define SEC_SRC_SGL_OFFSET 7
32 #define SEC_CKEY_OFFSET 9
33 #define SEC_CMODE_OFFSET 12
34 #define SEC_AKEY_OFFSET 5
35 #define SEC_AEAD_ALG_OFFSET 11
36 #define SEC_AUTH_OFFSET 6
37
38 #define SEC_FLAG_OFFSET 7
39 #define SEC_FLAG_MASK 0x0780
40 #define SEC_TYPE_MASK 0x0F
41 #define SEC_DONE_MASK 0x0001
42
43 #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH)
44 #define SEC_SGL_SGE_NR 128
45 #define SEC_CTX_DEV(ctx) (&(ctx)->sec->qm.pdev->dev)
46 #define SEC_CIPHER_AUTH 0xfe
47 #define SEC_AUTH_CIPHER 0x1
48 #define SEC_MAX_MAC_LEN 64
49 #define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH)
50 #define SEC_SQE_LEN_RATE 4
51 #define SEC_SQE_CFLAG 2
52 #define SEC_SQE_AEAD_FLAG 3
53 #define SEC_SQE_DONE 0x1
54
55 static atomic_t sec_active_devs;
56
57 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
58 static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
59 {
60 if (req->c_req.encrypt)
61 return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
62 ctx->hlf_q_num;
63
64 return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
65 ctx->hlf_q_num;
66 }
67
68 static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
69 {
70 if (req->c_req.encrypt)
71 atomic_dec(&ctx->enc_qcyclic);
72 else
73 atomic_dec(&ctx->dec_qcyclic);
74 }
75
76 static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
77 {
78 int req_id;
79
80 mutex_lock(&qp_ctx->req_lock);
81
82 req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL,
83 0, QM_Q_DEPTH, GFP_ATOMIC);
84 mutex_unlock(&qp_ctx->req_lock);
85 if (unlikely(req_id < 0)) {
86 dev_err(SEC_CTX_DEV(req->ctx), "alloc req id fail!\n");
87 return req_id;
88 }
89
90 req->qp_ctx = qp_ctx;
91 qp_ctx->req_list[req_id] = req;
92 return req_id;
93 }
94
95 static void sec_free_req_id(struct sec_req *req)
96 {
97 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
98 int req_id = req->req_id;
99
100 if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) {
101 dev_err(SEC_CTX_DEV(req->ctx), "free request id invalid!\n");
102 return;
103 }
104
105 qp_ctx->req_list[req_id] = NULL;
106 req->qp_ctx = NULL;
107
108 mutex_lock(&qp_ctx->req_lock);
109 idr_remove(&qp_ctx->req_idr, req_id);
110 mutex_unlock(&qp_ctx->req_lock);
111 }
112
113 static int sec_aead_verify(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
114 {
115 struct aead_request *aead_req = req->aead_req.aead_req;
116 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req);
117 u8 *mac_out = qp_ctx->res[req->req_id].out_mac;
118 size_t authsize = crypto_aead_authsize(tfm);
119 u8 *mac = mac_out + SEC_MAX_MAC_LEN;
120 struct scatterlist *sgl = aead_req->src;
121 size_t sz;
122
123 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize,
124 aead_req->cryptlen + aead_req->assoclen -
125 authsize);
126 if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) {
127 dev_err(SEC_CTX_DEV(req->ctx), "aead verify failure!\n");
128 return -EBADMSG;
129 }
130
131 return 0;
132 }
133
134 static void sec_req_cb(struct hisi_qp *qp, void *resp)
135 {
136 struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
137 struct sec_sqe *bd = resp;
138 struct sec_ctx *ctx;
139 struct sec_req *req;
140 u16 done, flag;
141 int err = 0;
142 u8 type;
143
144 type = bd->type_cipher_auth & SEC_TYPE_MASK;
145 if (unlikely(type != SEC_BD_TYPE2)) {
146 pr_err("err bd type [%d]\n", type);
147 return;
148 }
149
150 req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)];
151 req->err_type = bd->type2.error_type;
152 ctx = req->ctx;
153 done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK;
154 flag = (le16_to_cpu(bd->type2.done_flag) &
155 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET;
156 if (unlikely(req->err_type || done != SEC_SQE_DONE ||
157 (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) ||
158 (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) {
159 dev_err(SEC_CTX_DEV(ctx),
160 "err_type[%d],done[%d],flag[%d]\n",
161 req->err_type, done, flag);
162 err = -EIO;
163 }
164
165 if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt)
166 err = sec_aead_verify(req, qp_ctx);
167
168 atomic64_inc(&ctx->sec->debug.dfx.recv_cnt);
169
170 ctx->req_op->buf_unmap(ctx, req);
171
172 ctx->req_op->callback(ctx, req, err);
173 }
174
175 static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
176 {
177 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
178 int ret;
179
180 mutex_lock(&qp_ctx->req_lock);
181 ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
182 mutex_unlock(&qp_ctx->req_lock);
183 atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
184
185 if (unlikely(ret == -EBUSY))
186 return -ENOBUFS;
187
188 if (!ret) {
189 if (req->fake_busy)
190 ret = -EBUSY;
191 else
192 ret = -EINPROGRESS;
193 }
194
195 return ret;
196 }
197
198 /* Get DMA memory resources */
199 static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
200 {
201 int i;
202
203 res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ,
204 &res->c_ivin_dma, GFP_KERNEL);
205 if (!res->c_ivin)
206 return -ENOMEM;
207
208 for (i = 1; i < QM_Q_DEPTH; i++) {
209 res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE;
210 res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE;
211 }
212
213 return 0;
214 }
215
216 static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res)
217 {
218 if (res->c_ivin)
219 dma_free_coherent(dev, SEC_TOTAL_IV_SZ,
220 res->c_ivin, res->c_ivin_dma);
221 }
222
223 static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res)
224 {
225 int i;
226
227 res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1,
228 &res->out_mac_dma, GFP_KERNEL);
229 if (!res->out_mac)
230 return -ENOMEM;
231
232 for (i = 1; i < QM_Q_DEPTH; i++) {
233 res[i].out_mac_dma = res->out_mac_dma +
234 i * (SEC_MAX_MAC_LEN << 1);
235 res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1);
236 }
237
238 return 0;
239 }
240
241 static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res)
242 {
243 if (res->out_mac)
244 dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1,
245 res->out_mac, res->out_mac_dma);
246 }
247
248 static int sec_alg_resource_alloc(struct sec_ctx *ctx,
249 struct sec_qp_ctx *qp_ctx)
250 {
251 struct device *dev = SEC_CTX_DEV(ctx);
252 struct sec_alg_res *res = qp_ctx->res;
253 int ret;
254
255 ret = sec_alloc_civ_resource(dev, res);
256 if (ret)
257 return ret;
258
259 if (ctx->alg_type == SEC_AEAD) {
260 ret = sec_alloc_mac_resource(dev, res);
261 if (ret)
262 goto get_fail;
263 }
264
265 return 0;
266 get_fail:
267 sec_free_civ_resource(dev, res);
268
269 return ret;
270 }
271
272 static void sec_alg_resource_free(struct sec_ctx *ctx,
273 struct sec_qp_ctx *qp_ctx)
274 {
275 struct device *dev = SEC_CTX_DEV(ctx);
276
277 sec_free_civ_resource(dev, qp_ctx->res);
278
279 if (ctx->alg_type == SEC_AEAD)
280 sec_free_mac_resource(dev, qp_ctx->res);
281 }
282
283 static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx,
284 int qp_ctx_id, int alg_type)
285 {
286 struct device *dev = SEC_CTX_DEV(ctx);
287 struct sec_qp_ctx *qp_ctx;
288 struct hisi_qp *qp;
289 int ret = -ENOMEM;
290
291 qp = hisi_qm_create_qp(qm, alg_type);
292 if (IS_ERR(qp))
293 return PTR_ERR(qp);
294
295 qp_ctx = &ctx->qp_ctx[qp_ctx_id];
296 qp->req_type = 0;
297 qp->qp_ctx = qp_ctx;
298 qp->req_cb = sec_req_cb;
299 qp_ctx->qp = qp;
300 qp_ctx->ctx = ctx;
301
302 mutex_init(&qp_ctx->req_lock);
303 atomic_set(&qp_ctx->pending_reqs, 0);
304 idr_init(&qp_ctx->req_idr);
305
306 qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH,
307 SEC_SGL_SGE_NR);
308 if (IS_ERR(qp_ctx->c_in_pool)) {
309 dev_err(dev, "fail to create sgl pool for input!\n");
310 goto err_destroy_idr;
311 }
312
313 qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH,
314 SEC_SGL_SGE_NR);
315 if (IS_ERR(qp_ctx->c_out_pool)) {
316 dev_err(dev, "fail to create sgl pool for output!\n");
317 goto err_free_c_in_pool;
318 }
319
320 ret = sec_alg_resource_alloc(ctx, qp_ctx);
321 if (ret)
322 goto err_free_c_out_pool;
323
324 ret = hisi_qm_start_qp(qp, 0);
325 if (ret < 0)
326 goto err_queue_free;
327
328 return 0;
329
330 err_queue_free:
331 sec_alg_resource_free(ctx, qp_ctx);
332 err_free_c_out_pool:
333 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
334 err_free_c_in_pool:
335 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
336 err_destroy_idr:
337 idr_destroy(&qp_ctx->req_idr);
338 hisi_qm_release_qp(qp);
339
340 return ret;
341 }
342
343 static void sec_release_qp_ctx(struct sec_ctx *ctx,
344 struct sec_qp_ctx *qp_ctx)
345 {
346 struct device *dev = SEC_CTX_DEV(ctx);
347
348 hisi_qm_stop_qp(qp_ctx->qp);
349 sec_alg_resource_free(ctx, qp_ctx);
350
351 hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool);
352 hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool);
353
354 idr_destroy(&qp_ctx->req_idr);
355 hisi_qm_release_qp(qp_ctx->qp);
356 }
357
358 static int sec_ctx_base_init(struct sec_ctx *ctx)
359 {
360 struct sec_dev *sec;
361 int i, ret;
362
363 sec = sec_find_device(cpu_to_node(smp_processor_id()));
364 if (!sec) {
365 pr_err("Can not find proper Hisilicon SEC device!\n");
366 return -ENODEV;
367 }
368 ctx->sec = sec;
369 ctx->hlf_q_num = sec->ctx_q_num >> 1;
370
371 /* Half of queue depth is taken as fake requests limit in the queue. */
372 ctx->fake_req_limit = QM_Q_DEPTH >> 1;
373 ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
374 GFP_KERNEL);
375 if (!ctx->qp_ctx)
376 return -ENOMEM;
377
378 for (i = 0; i < sec->ctx_q_num; i++) {
379 ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0);
380 if (ret)
381 goto err_sec_release_qp_ctx;
382 }
383
384 return 0;
385 err_sec_release_qp_ctx:
386 for (i = i - 1; i >= 0; i--)
387 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
388
389 kfree(ctx->qp_ctx);
390 return ret;
391 }
392
393 static void sec_ctx_base_uninit(struct sec_ctx *ctx)
394 {
395 int i;
396
397 for (i = 0; i < ctx->sec->ctx_q_num; i++)
398 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]);
399
400 kfree(ctx->qp_ctx);
401 }
402
403 static int sec_cipher_init(struct sec_ctx *ctx)
404 {
405 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
406
407 c_ctx->c_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
408 &c_ctx->c_key_dma, GFP_KERNEL);
409 if (!c_ctx->c_key)
410 return -ENOMEM;
411
412 return 0;
413 }
414
415 static void sec_cipher_uninit(struct sec_ctx *ctx)
416 {
417 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
418
419 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE);
420 dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
421 c_ctx->c_key, c_ctx->c_key_dma);
422 }
423
424 static int sec_auth_init(struct sec_ctx *ctx)
425 {
426 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
427
428 a_ctx->a_key = dma_alloc_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
429 &a_ctx->a_key_dma, GFP_KERNEL);
430 if (!a_ctx->a_key)
431 return -ENOMEM;
432
433 return 0;
434 }
435
436 static void sec_auth_uninit(struct sec_ctx *ctx)
437 {
438 struct sec_auth_ctx *a_ctx = &ctx->a_ctx;
439
440 memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE);
441 dma_free_coherent(SEC_CTX_DEV(ctx), SEC_MAX_KEY_SIZE,
442 a_ctx->a_key, a_ctx->a_key_dma);
443 }
444
445 static int sec_skcipher_init(struct crypto_skcipher *tfm)
446 {
447 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
448 int ret;
449
450 ctx = crypto_skcipher_ctx(tfm);
451 ctx->alg_type = SEC_SKCIPHER;
452 crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
453 ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
454 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
455 dev_err(SEC_CTX_DEV(ctx), "get error skcipher iv size!\n");
456 return -EINVAL;
457 }
458
459 ret = sec_ctx_base_init(ctx);
460 if (ret)
461 return ret;
462
463 ret = sec_cipher_init(ctx);
464 if (ret)
465 goto err_cipher_init;
466
467 return 0;
468 err_cipher_init:
469 sec_ctx_base_uninit(ctx);
470
471 return ret;
472 }
473
474 static void sec_skcipher_uninit(struct crypto_skcipher *tfm)
475 {
476 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
477
478 sec_cipher_uninit(ctx);
479 sec_ctx_base_uninit(ctx);
480 }
481
482 static int sec_skcipher_3des_setkey(struct sec_cipher_ctx *c_ctx,
483 const u32 keylen,
484 const enum sec_cmode c_mode)
485 {
486 switch (keylen) {
487 case SEC_DES3_2KEY_SIZE:
488 c_ctx->c_key_len = SEC_CKEY_3DES_2KEY;
489 break;
490 case SEC_DES3_3KEY_SIZE:
491 c_ctx->c_key_len = SEC_CKEY_3DES_3KEY;
492 break;
493 default:
494 return -EINVAL;
495 }
496
497 return 0;
498 }
499
500 static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx,
501 const u32 keylen,
502 const enum sec_cmode c_mode)
503 {
504 if (c_mode == SEC_CMODE_XTS) {
505 switch (keylen) {
506 case SEC_XTS_MIN_KEY_SIZE:
507 c_ctx->c_key_len = SEC_CKEY_128BIT;
508 break;
509 case SEC_XTS_MAX_KEY_SIZE:
510 c_ctx->c_key_len = SEC_CKEY_256BIT;
511 break;
512 default:
513 pr_err("hisi_sec2: xts mode key error!\n");
514 return -EINVAL;
515 }
516 } else {
517 switch (keylen) {
518 case AES_KEYSIZE_128:
519 c_ctx->c_key_len = SEC_CKEY_128BIT;
520 break;
521 case AES_KEYSIZE_192:
522 c_ctx->c_key_len = SEC_CKEY_192BIT;
523 break;
524 case AES_KEYSIZE_256:
525 c_ctx->c_key_len = SEC_CKEY_256BIT;
526 break;
527 default:
528 pr_err("hisi_sec2: aes key error!\n");
529 return -EINVAL;
530 }
531 }
532
533 return 0;
534 }
535
536 static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
537 const u32 keylen, const enum sec_calg c_alg,
538 const enum sec_cmode c_mode)
539 {
540 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
541 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
542 int ret;
543
544 if (c_mode == SEC_CMODE_XTS) {
545 ret = xts_verify_key(tfm, key, keylen);
546 if (ret) {
547 dev_err(SEC_CTX_DEV(ctx), "xts mode key err!\n");
548 return ret;
549 }
550 }
551
552 c_ctx->c_alg = c_alg;
553 c_ctx->c_mode = c_mode;
554
555 switch (c_alg) {
556 case SEC_CALG_3DES:
557 ret = sec_skcipher_3des_setkey(c_ctx, keylen, c_mode);
558 break;
559 case SEC_CALG_AES:
560 case SEC_CALG_SM4:
561 ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode);
562 break;
563 default:
564 return -EINVAL;
565 }
566
567 if (ret) {
568 dev_err(SEC_CTX_DEV(ctx), "set sec key err!\n");
569 return ret;
570 }
571
572 memcpy(c_ctx->c_key, key, keylen);
573
574 return 0;
575 }
576
577 #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \
578 static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\
579 u32 keylen) \
580 { \
581 return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \
582 }
583
584 GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB)
585 GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC)
586 GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS)
587
588 GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB)
589 GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC)
590
591 GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS)
592 GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC)
593
594 static int sec_cipher_map(struct device *dev, struct sec_req *req,
595 struct scatterlist *src, struct scatterlist *dst)
596 {
597 struct sec_cipher_req *c_req = &req->c_req;
598 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
599
600 c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src,
601 qp_ctx->c_in_pool,
602 req->req_id,
603 &c_req->c_in_dma);
604
605 if (IS_ERR(c_req->c_in)) {
606 dev_err(dev, "fail to dma map input sgl buffers!\n");
607 return PTR_ERR(c_req->c_in);
608 }
609
610 if (dst == src) {
611 c_req->c_out = c_req->c_in;
612 c_req->c_out_dma = c_req->c_in_dma;
613 } else {
614 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst,
615 qp_ctx->c_out_pool,
616 req->req_id,
617 &c_req->c_out_dma);
618
619 if (IS_ERR(c_req->c_out)) {
620 dev_err(dev, "fail to dma map output sgl buffers!\n");
621 hisi_acc_sg_buf_unmap(dev, src, c_req->c_in);
622 return PTR_ERR(c_req->c_out);
623 }
624 }
625
626 return 0;
627 }
628
629 static void sec_cipher_unmap(struct device *dev, struct sec_cipher_req *req,
630 struct scatterlist *src, struct scatterlist *dst)
631 {
632 if (dst != src)
633 hisi_acc_sg_buf_unmap(dev, src, req->c_in);
634
635 hisi_acc_sg_buf_unmap(dev, dst, req->c_out);
636 }
637
638 static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
639 {
640 struct skcipher_request *sq = req->c_req.sk_req;
641
642 return sec_cipher_map(SEC_CTX_DEV(ctx), req, sq->src, sq->dst);
643 }
644
645 static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
646 {
647 struct device *dev = SEC_CTX_DEV(ctx);
648 struct sec_cipher_req *c_req = &req->c_req;
649 struct skcipher_request *sk_req = c_req->sk_req;
650
651 sec_cipher_unmap(dev, c_req, sk_req->src, sk_req->dst);
652 }
653
654 static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx,
655 struct crypto_authenc_keys *keys)
656 {
657 switch (keys->enckeylen) {
658 case AES_KEYSIZE_128:
659 c_ctx->c_key_len = SEC_CKEY_128BIT;
660 break;
661 case AES_KEYSIZE_192:
662 c_ctx->c_key_len = SEC_CKEY_192BIT;
663 break;
664 case AES_KEYSIZE_256:
665 c_ctx->c_key_len = SEC_CKEY_256BIT;
666 break;
667 default:
668 pr_err("hisi_sec2: aead aes key error!\n");
669 return -EINVAL;
670 }
671 memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen);
672
673 return 0;
674 }
675
676 static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx,
677 struct crypto_authenc_keys *keys)
678 {
679 struct crypto_shash *hash_tfm = ctx->hash_tfm;
680 SHASH_DESC_ON_STACK(shash, hash_tfm);
681 int blocksize, ret;
682
683 if (!keys->authkeylen) {
684 pr_err("hisi_sec2: aead auth key error!\n");
685 return -EINVAL;
686 }
687
688 blocksize = crypto_shash_blocksize(hash_tfm);
689 if (keys->authkeylen > blocksize) {
690 ret = crypto_shash_digest(shash, keys->authkey,
691 keys->authkeylen, ctx->a_key);
692 if (ret) {
693 pr_err("hisi_sec2: aead auth digest error!\n");
694 return -EINVAL;
695 }
696 ctx->a_key_len = blocksize;
697 } else {
698 memcpy(ctx->a_key, keys->authkey, keys->authkeylen);
699 ctx->a_key_len = keys->authkeylen;
700 }
701
702 return 0;
703 }
704
705 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key,
706 const u32 keylen, const enum sec_hash_alg a_alg,
707 const enum sec_calg c_alg,
708 const enum sec_mac_len mac_len,
709 const enum sec_cmode c_mode)
710 {
711 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
712 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
713 struct crypto_authenc_keys keys;
714 int ret;
715
716 ctx->a_ctx.a_alg = a_alg;
717 ctx->c_ctx.c_alg = c_alg;
718 ctx->a_ctx.mac_len = mac_len;
719 c_ctx->c_mode = c_mode;
720
721 if (crypto_authenc_extractkeys(&keys, key, keylen))
722 goto bad_key;
723
724 ret = sec_aead_aes_set_key(c_ctx, &keys);
725 if (ret) {
726 dev_err(SEC_CTX_DEV(ctx), "set sec cipher key err!\n");
727 goto bad_key;
728 }
729
730 ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys);
731 if (ret) {
732 dev_err(SEC_CTX_DEV(ctx), "set sec auth key err!\n");
733 goto bad_key;
734 }
735
736 return 0;
737 bad_key:
738 memzero_explicit(&keys, sizeof(struct crypto_authenc_keys));
739
740 return -EINVAL;
741 }
742
743
744 #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \
745 static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \
746 u32 keylen) \
747 { \
748 return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\
749 }
750
751 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1,
752 SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC)
753 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256,
754 SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC)
755 GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512,
756 SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC)
757
758 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req)
759 {
760 struct aead_request *aq = req->aead_req.aead_req;
761
762 return sec_cipher_map(SEC_CTX_DEV(ctx), req, aq->src, aq->dst);
763 }
764
765 static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req)
766 {
767 struct device *dev = SEC_CTX_DEV(ctx);
768 struct sec_cipher_req *cq = &req->c_req;
769 struct aead_request *aq = req->aead_req.aead_req;
770
771 sec_cipher_unmap(dev, cq, aq->src, aq->dst);
772 }
773
774 static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req)
775 {
776 int ret;
777
778 ret = ctx->req_op->buf_map(ctx, req);
779 if (unlikely(ret))
780 return ret;
781
782 ctx->req_op->do_transfer(ctx, req);
783
784 ret = ctx->req_op->bd_fill(ctx, req);
785 if (unlikely(ret))
786 goto unmap_req_buf;
787
788 return ret;
789
790 unmap_req_buf:
791 ctx->req_op->buf_unmap(ctx, req);
792
793 return ret;
794 }
795
796 static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req)
797 {
798 ctx->req_op->buf_unmap(ctx, req);
799 }
800
801 static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
802 {
803 struct skcipher_request *sk_req = req->c_req.sk_req;
804 u8 *c_ivin = req->qp_ctx->res[req->req_id].c_ivin;
805
806 memcpy(c_ivin, sk_req->iv, ctx->c_ctx.ivsize);
807 }
808
809 static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
810 {
811 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx;
812 struct sec_cipher_req *c_req = &req->c_req;
813 struct sec_sqe *sec_sqe = &req->sec_sqe;
814 u8 scene, sa_type, da_type;
815 u8 bd_type, cipher;
816 u8 de = 0;
817
818 memset(sec_sqe, 0, sizeof(struct sec_sqe));
819
820 sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
821 sec_sqe->type2.c_ivin_addr =
822 cpu_to_le64(req->qp_ctx->res[req->req_id].c_ivin_dma);
823 sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma);
824 sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
825
826 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
827 SEC_CMODE_OFFSET);
828 sec_sqe->type2.c_alg = c_ctx->c_alg;
829 sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) <<
830 SEC_CKEY_OFFSET);
831
832 bd_type = SEC_BD_TYPE2;
833 if (c_req->encrypt)
834 cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET;
835 else
836 cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET;
837 sec_sqe->type_cipher_auth = bd_type | cipher;
838
839 sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET;
840 scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
841 if (c_req->c_in_dma != c_req->c_out_dma)
842 de = 0x1 << SEC_DE_OFFSET;
843
844 sec_sqe->sds_sa_type = (de | scene | sa_type);
845
846 /* Just set DST address type */
847 da_type = SEC_SGL << SEC_DST_SGL_OFFSET;
848 sec_sqe->sdm_addr_type |= da_type;
849
850 sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
851 sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
852
853 return 0;
854 }
855
856 static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type)
857 {
858 struct aead_request *aead_req = req->aead_req.aead_req;
859 struct skcipher_request *sk_req = req->c_req.sk_req;
860 u32 iv_size = req->ctx->c_ctx.ivsize;
861 struct scatterlist *sgl;
862 unsigned int cryptlen;
863 size_t sz;
864 u8 *iv;
865
866 if (req->c_req.encrypt)
867 sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
868 else
869 sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
870
871 if (alg_type == SEC_SKCIPHER) {
872 iv = sk_req->iv;
873 cryptlen = sk_req->cryptlen;
874 } else {
875 iv = aead_req->iv;
876 cryptlen = aead_req->cryptlen;
877 }
878
879 sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size,
880 cryptlen - iv_size);
881 if (unlikely(sz != iv_size))
882 dev_err(SEC_CTX_DEV(req->ctx), "copy output iv error!\n");
883 }
884
885 static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
886 int err)
887 {
888 struct skcipher_request *sk_req = req->c_req.sk_req;
889 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
890
891 atomic_dec(&qp_ctx->pending_reqs);
892 sec_free_req_id(req);
893
894 /* IV output at encrypto of CBC mode */
895 if (!err && ctx->c_ctx.c_mode == SEC_CMODE_CBC && req->c_req.encrypt)
896 sec_update_iv(req, SEC_SKCIPHER);
897
898 if (req->fake_busy)
899 sk_req->base.complete(&sk_req->base, -EINPROGRESS);
900
901 sk_req->base.complete(&sk_req->base, err);
902 }
903
904 static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req)
905 {
906 struct aead_request *aead_req = req->aead_req.aead_req;
907 u8 *c_ivin = req->qp_ctx->res[req->req_id].c_ivin;
908
909 memcpy(c_ivin, aead_req->iv, ctx->c_ctx.ivsize);
910 }
911
912 static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir,
913 struct sec_req *req, struct sec_sqe *sec_sqe)
914 {
915 struct sec_aead_req *a_req = &req->aead_req;
916 struct sec_cipher_req *c_req = &req->c_req;
917 struct aead_request *aq = a_req->aead_req;
918
919 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma);
920
921 sec_sqe->type2.mac_key_alg =
922 cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE);
923
924 sec_sqe->type2.mac_key_alg |=
925 cpu_to_le32((u32)((ctx->a_key_len) /
926 SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET);
927
928 sec_sqe->type2.mac_key_alg |=
929 cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET);
930
931 sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET;
932
933 if (dir)
934 sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH;
935 else
936 sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER;
937
938 sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen);
939
940 sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen);
941
942 sec_sqe->type2.mac_addr =
943 cpu_to_le64(req->qp_ctx->res[req->req_id].out_mac_dma);
944 }
945
946 static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req)
947 {
948 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
949 struct sec_sqe *sec_sqe = &req->sec_sqe;
950 int ret;
951
952 ret = sec_skcipher_bd_fill(ctx, req);
953 if (unlikely(ret)) {
954 dev_err(SEC_CTX_DEV(ctx), "skcipher bd fill is error!\n");
955 return ret;
956 }
957
958 sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe);
959
960 return 0;
961 }
962
963 static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err)
964 {
965 struct aead_request *a_req = req->aead_req.aead_req;
966 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
967 struct sec_cipher_req *c_req = &req->c_req;
968 size_t authsize = crypto_aead_authsize(tfm);
969 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
970 size_t sz;
971
972 atomic_dec(&qp_ctx->pending_reqs);
973
974 if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
975 sec_update_iv(req, SEC_AEAD);
976
977 /* Copy output mac */
978 if (!err && c_req->encrypt) {
979 struct scatterlist *sgl = a_req->dst;
980
981 sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl),
982 qp_ctx->res[req->req_id].out_mac,
983 authsize, a_req->cryptlen +
984 a_req->assoclen);
985
986 if (unlikely(sz != authsize)) {
987 dev_err(SEC_CTX_DEV(req->ctx), "copy out mac err!\n");
988 err = -EINVAL;
989 }
990 }
991
992 sec_free_req_id(req);
993
994 if (req->fake_busy)
995 a_req->base.complete(&a_req->base, -EINPROGRESS);
996
997 a_req->base.complete(&a_req->base, err);
998 }
999
1000 static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1001 {
1002 struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1003
1004 atomic_dec(&qp_ctx->pending_reqs);
1005 sec_free_req_id(req);
1006 sec_free_queue_id(ctx, req);
1007 }
1008
1009 static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1010 {
1011 struct sec_qp_ctx *qp_ctx;
1012 int queue_id;
1013
1014 /* To load balance */
1015 queue_id = sec_alloc_queue_id(ctx, req);
1016 qp_ctx = &ctx->qp_ctx[queue_id];
1017
1018 req->req_id = sec_alloc_req_id(req, qp_ctx);
1019 if (unlikely(req->req_id < 0)) {
1020 sec_free_queue_id(ctx, req);
1021 return req->req_id;
1022 }
1023
1024 if (ctx->fake_req_limit <= atomic_inc_return(&qp_ctx->pending_reqs))
1025 req->fake_busy = true;
1026 else
1027 req->fake_busy = false;
1028
1029 return 0;
1030 }
1031
1032 static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1033 {
1034 int ret;
1035
1036 ret = sec_request_init(ctx, req);
1037 if (unlikely(ret))
1038 return ret;
1039
1040 ret = sec_request_transfer(ctx, req);
1041 if (unlikely(ret))
1042 goto err_uninit_req;
1043
1044 /* Output IV as decrypto */
1045 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt)
1046 sec_update_iv(req, ctx->alg_type);
1047
1048 ret = ctx->req_op->bd_send(ctx, req);
1049 if (unlikely(ret != -EBUSY && ret != -EINPROGRESS)) {
1050 dev_err_ratelimited(SEC_CTX_DEV(ctx), "send sec request failed!\n");
1051 goto err_send_req;
1052 }
1053
1054 return ret;
1055
1056 err_send_req:
1057 /* As failing, restore the IV from user */
1058 if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
1059 if (ctx->alg_type == SEC_SKCIPHER)
1060 memcpy(req->c_req.sk_req->iv,
1061 req->qp_ctx->res[req->req_id].c_ivin,
1062 ctx->c_ctx.ivsize);
1063 else
1064 memcpy(req->aead_req.aead_req->iv,
1065 req->qp_ctx->res[req->req_id].c_ivin,
1066 ctx->c_ctx.ivsize);
1067 }
1068
1069 sec_request_untransfer(ctx, req);
1070 err_uninit_req:
1071 sec_request_uninit(ctx, req);
1072
1073 return ret;
1074 }
1075
1076 static const struct sec_req_op sec_skcipher_req_ops = {
1077 .buf_map = sec_skcipher_sgl_map,
1078 .buf_unmap = sec_skcipher_sgl_unmap,
1079 .do_transfer = sec_skcipher_copy_iv,
1080 .bd_fill = sec_skcipher_bd_fill,
1081 .bd_send = sec_bd_send,
1082 .callback = sec_skcipher_callback,
1083 .process = sec_process,
1084 };
1085
1086 static const struct sec_req_op sec_aead_req_ops = {
1087 .buf_map = sec_aead_sgl_map,
1088 .buf_unmap = sec_aead_sgl_unmap,
1089 .do_transfer = sec_aead_copy_iv,
1090 .bd_fill = sec_aead_bd_fill,
1091 .bd_send = sec_bd_send,
1092 .callback = sec_aead_callback,
1093 .process = sec_process,
1094 };
1095
1096 static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm)
1097 {
1098 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
1099
1100 ctx->req_op = &sec_skcipher_req_ops;
1101
1102 return sec_skcipher_init(tfm);
1103 }
1104
1105 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm)
1106 {
1107 sec_skcipher_uninit(tfm);
1108 }
1109
1110 static int sec_aead_init(struct crypto_aead *tfm)
1111 {
1112 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1113 int ret;
1114
1115 crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
1116 ctx->alg_type = SEC_AEAD;
1117 ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
1118 if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
1119 dev_err(SEC_CTX_DEV(ctx), "get error aead iv size!\n");
1120 return -EINVAL;
1121 }
1122
1123 ctx->req_op = &sec_aead_req_ops;
1124 ret = sec_ctx_base_init(ctx);
1125 if (ret)
1126 return ret;
1127
1128 ret = sec_auth_init(ctx);
1129 if (ret)
1130 goto err_auth_init;
1131
1132 ret = sec_cipher_init(ctx);
1133 if (ret)
1134 goto err_cipher_init;
1135
1136 return ret;
1137
1138 err_cipher_init:
1139 sec_auth_uninit(ctx);
1140 err_auth_init:
1141 sec_ctx_base_uninit(ctx);
1142
1143 return ret;
1144 }
1145
1146 static void sec_aead_exit(struct crypto_aead *tfm)
1147 {
1148 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1149
1150 sec_cipher_uninit(ctx);
1151 sec_auth_uninit(ctx);
1152 sec_ctx_base_uninit(ctx);
1153 }
1154
1155 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name)
1156 {
1157 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1158 struct sec_auth_ctx *auth_ctx = &ctx->a_ctx;
1159 int ret;
1160
1161 ret = sec_aead_init(tfm);
1162 if (ret) {
1163 pr_err("hisi_sec2: aead init error!\n");
1164 return ret;
1165 }
1166
1167 auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1168 if (IS_ERR(auth_ctx->hash_tfm)) {
1169 dev_err(SEC_CTX_DEV(ctx), "aead alloc shash error!\n");
1170 sec_aead_exit(tfm);
1171 return PTR_ERR(auth_ctx->hash_tfm);
1172 }
1173
1174 return 0;
1175 }
1176
1177 static void sec_aead_ctx_exit(struct crypto_aead *tfm)
1178 {
1179 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1180
1181 crypto_free_shash(ctx->a_ctx.hash_tfm);
1182 sec_aead_exit(tfm);
1183 }
1184
1185 static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm)
1186 {
1187 return sec_aead_ctx_init(tfm, "sha1");
1188 }
1189
1190 static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm)
1191 {
1192 return sec_aead_ctx_init(tfm, "sha256");
1193 }
1194
1195 static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm)
1196 {
1197 return sec_aead_ctx_init(tfm, "sha512");
1198 }
1199
1200 static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
1201 {
1202 struct skcipher_request *sk_req = sreq->c_req.sk_req;
1203 struct device *dev = SEC_CTX_DEV(ctx);
1204 u8 c_alg = ctx->c_ctx.c_alg;
1205
1206 if (unlikely(!sk_req->src || !sk_req->dst)) {
1207 dev_err(dev, "skcipher input param error!\n");
1208 return -EINVAL;
1209 }
1210 sreq->c_req.c_len = sk_req->cryptlen;
1211 if (c_alg == SEC_CALG_3DES) {
1212 if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) {
1213 dev_err(dev, "skcipher 3des input length error!\n");
1214 return -EINVAL;
1215 }
1216 return 0;
1217 } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) {
1218 if (unlikely(sk_req->cryptlen & (AES_BLOCK_SIZE - 1))) {
1219 dev_err(dev, "skcipher aes input length error!\n");
1220 return -EINVAL;
1221 }
1222 return 0;
1223 }
1224
1225 dev_err(dev, "skcipher algorithm error!\n");
1226 return -EINVAL;
1227 }
1228
1229 static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
1230 {
1231 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
1232 struct sec_req *req = skcipher_request_ctx(sk_req);
1233 struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
1234 int ret;
1235
1236 if (!sk_req->cryptlen)
1237 return 0;
1238
1239 req->c_req.sk_req = sk_req;
1240 req->c_req.encrypt = encrypt;
1241 req->ctx = ctx;
1242
1243 ret = sec_skcipher_param_check(ctx, req);
1244 if (unlikely(ret))
1245 return -EINVAL;
1246
1247 return ctx->req_op->process(ctx, req);
1248 }
1249
1250 static int sec_skcipher_encrypt(struct skcipher_request *sk_req)
1251 {
1252 return sec_skcipher_crypto(sk_req, true);
1253 }
1254
1255 static int sec_skcipher_decrypt(struct skcipher_request *sk_req)
1256 {
1257 return sec_skcipher_crypto(sk_req, false);
1258 }
1259
1260 #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \
1261 sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\
1262 {\
1263 .base = {\
1264 .cra_name = sec_cra_name,\
1265 .cra_driver_name = "hisi_sec_"sec_cra_name,\
1266 .cra_priority = SEC_PRIORITY,\
1267 .cra_flags = CRYPTO_ALG_ASYNC,\
1268 .cra_blocksize = blk_size,\
1269 .cra_ctxsize = sizeof(struct sec_ctx),\
1270 .cra_module = THIS_MODULE,\
1271 },\
1272 .init = ctx_init,\
1273 .exit = ctx_exit,\
1274 .setkey = sec_set_key,\
1275 .decrypt = sec_skcipher_decrypt,\
1276 .encrypt = sec_skcipher_encrypt,\
1277 .min_keysize = sec_min_key_size,\
1278 .max_keysize = sec_max_key_size,\
1279 .ivsize = iv_size,\
1280 },
1281
1282 #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \
1283 max_key_size, blk_size, iv_size) \
1284 SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \
1285 sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size)
1286
1287 static struct skcipher_alg sec_skciphers[] = {
1288 SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb,
1289 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
1290 AES_BLOCK_SIZE, 0)
1291
1292 SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc,
1293 AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE,
1294 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
1295
1296 SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts,
1297 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE,
1298 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
1299
1300 SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb,
1301 SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE,
1302 DES3_EDE_BLOCK_SIZE, 0)
1303
1304 SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc,
1305 SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE,
1306 DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE)
1307
1308 SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts,
1309 SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE,
1310 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
1311
1312 SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc,
1313 AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE,
1314 AES_BLOCK_SIZE, AES_BLOCK_SIZE)
1315 };
1316
1317 static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq)
1318 {
1319 u8 c_alg = ctx->c_ctx.c_alg;
1320 struct aead_request *req = sreq->aead_req.aead_req;
1321 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1322 size_t authsize = crypto_aead_authsize(tfm);
1323
1324 if (unlikely(!req->src || !req->dst || !req->cryptlen)) {
1325 dev_err(SEC_CTX_DEV(ctx), "aead input param error!\n");
1326 return -EINVAL;
1327 }
1328
1329 /* Support AES only */
1330 if (unlikely(c_alg != SEC_CALG_AES)) {
1331 dev_err(SEC_CTX_DEV(ctx), "aead crypto alg error!\n");
1332 return -EINVAL;
1333
1334 }
1335 if (sreq->c_req.encrypt)
1336 sreq->c_req.c_len = req->cryptlen;
1337 else
1338 sreq->c_req.c_len = req->cryptlen - authsize;
1339
1340 if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) {
1341 dev_err(SEC_CTX_DEV(ctx), "aead crypto length error!\n");
1342 return -EINVAL;
1343 }
1344
1345 return 0;
1346 }
1347
1348 static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
1349 {
1350 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
1351 struct sec_req *req = aead_request_ctx(a_req);
1352 struct sec_ctx *ctx = crypto_aead_ctx(tfm);
1353 int ret;
1354
1355 req->aead_req.aead_req = a_req;
1356 req->c_req.encrypt = encrypt;
1357 req->ctx = ctx;
1358
1359 ret = sec_aead_param_check(ctx, req);
1360 if (unlikely(ret))
1361 return -EINVAL;
1362
1363 return ctx->req_op->process(ctx, req);
1364 }
1365
1366 static int sec_aead_encrypt(struct aead_request *a_req)
1367 {
1368 return sec_aead_crypto(a_req, true);
1369 }
1370
1371 static int sec_aead_decrypt(struct aead_request *a_req)
1372 {
1373 return sec_aead_crypto(a_req, false);
1374 }
1375
1376 #define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\
1377 ctx_exit, blk_size, iv_size, max_authsize)\
1378 {\
1379 .base = {\
1380 .cra_name = sec_cra_name,\
1381 .cra_driver_name = "hisi_sec_"sec_cra_name,\
1382 .cra_priority = SEC_PRIORITY,\
1383 .cra_flags = CRYPTO_ALG_ASYNC,\
1384 .cra_blocksize = blk_size,\
1385 .cra_ctxsize = sizeof(struct sec_ctx),\
1386 .cra_module = THIS_MODULE,\
1387 },\
1388 .init = ctx_init,\
1389 .exit = ctx_exit,\
1390 .setkey = sec_set_key,\
1391 .decrypt = sec_aead_decrypt,\
1392 .encrypt = sec_aead_encrypt,\
1393 .ivsize = iv_size,\
1394 .maxauthsize = max_authsize,\
1395 }
1396
1397 #define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\
1398 SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\
1399 sec_aead_ctx_exit, blksize, ivsize, authsize)
1400
1401 static struct aead_alg sec_aeads[] = {
1402 SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))",
1403 sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init,
1404 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE),
1405
1406 SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))",
1407 sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init,
1408 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE),
1409
1410 SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))",
1411 sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init,
1412 AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE),
1413 };
1414
1415 int sec_register_to_crypto(void)
1416 {
1417 int ret = 0;
1418
1419 /* To avoid repeat register */
1420 if (atomic_add_return(1, &sec_active_devs) == 1) {
1421 ret = crypto_register_skciphers(sec_skciphers,
1422 ARRAY_SIZE(sec_skciphers));
1423 if (ret)
1424 return ret;
1425
1426 ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
1427 if (ret)
1428 goto reg_aead_fail;
1429 }
1430
1431 return ret;
1432
1433 reg_aead_fail:
1434 crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers));
1435
1436 return ret;
1437 }
1438
1439 void sec_unregister_from_crypto(void)
1440 {
1441 if (atomic_sub_return(1, &sec_active_devs) == 0) {
1442 crypto_unregister_skciphers(sec_skciphers,
1443 ARRAY_SIZE(sec_skciphers));
1444 crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads));
1445 }
1446 }
Linux with added FPC-III support