?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
staging: rtl8192u: remove redundant assignment to pointer crypt
[linux/fpc-iii.git] / drivers / crypto / mediatek / mtk-aes.c
blobb7477ee32ca098969d8345e3512e657cdfba74ba
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cryptographic API.
4 *
5 * Driver for EIP97 AES acceleration.
6 *
7 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
8 *
9 * Some ideas are from atmel-aes.c drivers.
10 */
11
12 #include <crypto/aes.h>
13 #include <crypto/gcm.h>
14 #include "mtk-platform.h"
15
16 #define AES_QUEUE_SIZE 512
17 #define AES_BUF_ORDER 2
18 #define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \
19 & ~(AES_BLOCK_SIZE - 1))
20 #define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \
21 AES_BLOCK_SIZE * 2)
22 #define AES_MAX_CT_SIZE 6
23
24 #define AES_CT_CTRL_HDR cpu_to_le32(0x00220000)
25
26 /* AES-CBC/ECB/CTR command token */
27 #define AES_CMD0 cpu_to_le32(0x05000000)
28 #define AES_CMD1 cpu_to_le32(0x2d060000)
29 #define AES_CMD2 cpu_to_le32(0xe4a63806)
30 /* AES-GCM command token */
31 #define AES_GCM_CMD0 cpu_to_le32(0x0b000000)
32 #define AES_GCM_CMD1 cpu_to_le32(0xa0800000)
33 #define AES_GCM_CMD2 cpu_to_le32(0x25000010)
34 #define AES_GCM_CMD3 cpu_to_le32(0x0f020000)
35 #define AES_GCM_CMD4 cpu_to_le32(0x21e60000)
36 #define AES_GCM_CMD5 cpu_to_le32(0x40e60000)
37 #define AES_GCM_CMD6 cpu_to_le32(0xd0070000)
38
39 /* AES transform information word 0 fields */
40 #define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0)
41 #define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0)
42 #define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0)
43 #define AES_TFM_GCM_IN cpu_to_le32(0xf << 0)
44 #define AES_TFM_SIZE(x) cpu_to_le32((x) << 8)
45 #define AES_TFM_128BITS cpu_to_le32(0xb << 16)
46 #define AES_TFM_192BITS cpu_to_le32(0xd << 16)
47 #define AES_TFM_256BITS cpu_to_le32(0xf << 16)
48 #define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21)
49 #define AES_TFM_GHASH cpu_to_le32(0x4 << 23)
50 /* AES transform information word 1 fields */
51 #define AES_TFM_ECB cpu_to_le32(0x0 << 0)
52 #define AES_TFM_CBC cpu_to_le32(0x1 << 0)
53 #define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */
54 #define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */
55 #define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */
56 #define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */
57 #define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10)
58 #define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17)
59
60 /* AES flags */
61 #define AES_FLAGS_CIPHER_MSK GENMASK(2, 0)
62 #define AES_FLAGS_ECB BIT(0)
63 #define AES_FLAGS_CBC BIT(1)
64 #define AES_FLAGS_CTR BIT(2)
65 #define AES_FLAGS_GCM BIT(3)
66 #define AES_FLAGS_ENCRYPT BIT(4)
67 #define AES_FLAGS_BUSY BIT(5)
68
69 #define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26))
70
71 /**
72 * mtk_aes_info - hardware information of AES
73 * @cmd: command token, hardware instruction
74 * @tfm: transform state of cipher algorithm.
75 * @state: contains keys and initial vectors.
76 *
77 * Memory layout of GCM buffer:
78 * /-----------\
79 * | AES KEY | 128/196/256 bits
80 * |-----------|
81 * | HASH KEY | a string 128 zero bits encrypted using the block cipher
82 * |-----------|
83 * | IVs | 4 * 4 bytes
84 * \-----------/
85 *
86 * The engine requires all these info to do:
87 * - Commands decoding and control of the engine's data path.
88 * - Coordinating hardware data fetch and store operations.
89 * - Result token construction and output.
90 */
91 struct mtk_aes_info {
92 __le32 cmd[AES_MAX_CT_SIZE];
93 __le32 tfm[2];
94 __le32 state[AES_MAX_STATE_BUF_SIZE];
95 };
96
97 struct mtk_aes_reqctx {
98 u64 mode;
99 };
100
101 struct mtk_aes_base_ctx {
102 struct mtk_cryp *cryp;
103 u32 keylen;
104 __le32 keymode;
105
106 mtk_aes_fn start;
107
108 struct mtk_aes_info info;
109 dma_addr_t ct_dma;
110 dma_addr_t tfm_dma;
111
112 __le32 ct_hdr;
113 u32 ct_size;
114 };
115
116 struct mtk_aes_ctx {
117 struct mtk_aes_base_ctx base;
118 };
119
120 struct mtk_aes_ctr_ctx {
121 struct mtk_aes_base_ctx base;
122
123 u32 iv[AES_BLOCK_SIZE / sizeof(u32)];
124 size_t offset;
125 struct scatterlist src[2];
126 struct scatterlist dst[2];
127 };
128
129 struct mtk_aes_gcm_ctx {
130 struct mtk_aes_base_ctx base;
131
132 u32 authsize;
133 size_t textlen;
134
135 struct crypto_skcipher *ctr;
136 };
137
138 struct mtk_aes_drv {
139 struct list_head dev_list;
140 /* Device list lock */
141 spinlock_t lock;
142 };
143
144 static struct mtk_aes_drv mtk_aes = {
145 .dev_list = LIST_HEAD_INIT(mtk_aes.dev_list),
146 .lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock),
147 };
148
149 static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset)
150 {
151 return readl_relaxed(cryp->base + offset);
152 }
153
154 static inline void mtk_aes_write(struct mtk_cryp *cryp,
155 u32 offset, u32 value)
156 {
157 writel_relaxed(value, cryp->base + offset);
158 }
159
160 static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx)
161 {
162 struct mtk_cryp *cryp = NULL;
163 struct mtk_cryp *tmp;
164
165 spin_lock_bh(&mtk_aes.lock);
166 if (!ctx->cryp) {
167 list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) {
168 cryp = tmp;
169 break;
170 }
171 ctx->cryp = cryp;
172 } else {
173 cryp = ctx->cryp;
174 }
175 spin_unlock_bh(&mtk_aes.lock);
176
177 return cryp;
178 }
179
180 static inline size_t mtk_aes_padlen(size_t len)
181 {
182 len &= AES_BLOCK_SIZE - 1;
183 return len ? AES_BLOCK_SIZE - len : 0;
184 }
185
186 static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len,
187 struct mtk_aes_dma *dma)
188 {
189 int nents;
190
191 if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
192 return false;
193
194 for (nents = 0; sg; sg = sg_next(sg), ++nents) {
195 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
196 return false;
197
198 if (len <= sg->length) {
199 if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
200 return false;
201
202 dma->nents = nents + 1;
203 dma->remainder = sg->length - len;
204 sg->length = len;
205 return true;
206 }
207
208 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
209 return false;
210
211 len -= sg->length;
212 }
213
214 return false;
215 }
216
217 static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,
218 const struct mtk_aes_reqctx *rctx)
219 {
220 /* Clear all but persistent flags and set request flags. */
221 aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;
222 }
223
224 static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
225 {
226 struct scatterlist *sg = dma->sg;
227 int nents = dma->nents;
228
229 if (!dma->remainder)
230 return;
231
232 while (--nents > 0 && sg)
233 sg = sg_next(sg);
234
235 if (!sg)
236 return;
237
238 sg->length += dma->remainder;
239 }
240
241 static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size)
242 {
243 int i;
244
245 for (i = 0; i < SIZE_IN_WORDS(size); i++)
246 dst[i] = cpu_to_le32(src[i]);
247 }
248
249 static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size)
250 {
251 int i;
252
253 for (i = 0; i < SIZE_IN_WORDS(size); i++)
254 dst[i] = cpu_to_be32(src[i]);
255 }
256
257 static inline int mtk_aes_complete(struct mtk_cryp *cryp,
258 struct mtk_aes_rec *aes,
259 int err)
260 {
261 aes->flags &= ~AES_FLAGS_BUSY;
262 aes->areq->complete(aes->areq, err);
263 /* Handle new request */
264 tasklet_schedule(&aes->queue_task);
265 return err;
266 }
267
268 /*
269 * Write descriptors for processing. This will configure the engine, load
270 * the transform information and then start the packet processing.
271 */
272 static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
273 {
274 struct mtk_ring *ring = cryp->ring[aes->id];
275 struct mtk_desc *cmd = NULL, *res = NULL;
276 struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;
277 u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;
278 int nents;
279
280 /* Write command descriptors */
281 for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) {
282 cmd = ring->cmd_next;
283 cmd->hdr = MTK_DESC_BUF_LEN(ssg->length);
284 cmd->buf = cpu_to_le32(sg_dma_address(ssg));
285
286 if (nents == 0) {
287 cmd->hdr |= MTK_DESC_FIRST |
288 MTK_DESC_CT_LEN(aes->ctx->ct_size);
289 cmd->ct = cpu_to_le32(aes->ctx->ct_dma);
290 cmd->ct_hdr = aes->ctx->ct_hdr;
291 cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);
292 }
293
294 /* Shift ring buffer and check boundary */
295 if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM)
296 ring->cmd_next = ring->cmd_base;
297 }
298 cmd->hdr |= MTK_DESC_LAST;
299
300 /* Prepare result descriptors */
301 for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) {
302 res = ring->res_next;
303 res->hdr = MTK_DESC_BUF_LEN(dsg->length);
304 res->buf = cpu_to_le32(sg_dma_address(dsg));
305
306 if (nents == 0)
307 res->hdr |= MTK_DESC_FIRST;
308
309 /* Shift ring buffer and check boundary */
310 if (++ring->res_next == ring->res_base + MTK_DESC_NUM)
311 ring->res_next = ring->res_base;
312 }
313 res->hdr |= MTK_DESC_LAST;
314
315 /* Pointer to current result descriptor */
316 ring->res_prev = res;
317
318 /* Prepare enough space for authenticated tag */
319 if (aes->flags & AES_FLAGS_GCM)
320 res->hdr += AES_BLOCK_SIZE;
321
322 /*
323 * Make sure that all changes to the DMA ring are done before we
324 * start engine.
325 */
326 wmb();
327 /* Start DMA transfer */
328 mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));
329 mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));
330
331 return -EINPROGRESS;
332 }
333
334 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
335 {
336 struct mtk_aes_base_ctx *ctx = aes->ctx;
337
338 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),
339 DMA_TO_DEVICE);
340
341 if (aes->src.sg == aes->dst.sg) {
342 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
343 DMA_BIDIRECTIONAL);
344
345 if (aes->src.sg != &aes->aligned_sg)
346 mtk_aes_restore_sg(&aes->src);
347 } else {
348 dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,
349 DMA_FROM_DEVICE);
350
351 if (aes->dst.sg != &aes->aligned_sg)
352 mtk_aes_restore_sg(&aes->dst);
353
354 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
355 DMA_TO_DEVICE);
356
357 if (aes->src.sg != &aes->aligned_sg)
358 mtk_aes_restore_sg(&aes->src);
359 }
360
361 if (aes->dst.sg == &aes->aligned_sg)
362 sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),
363 aes->buf, aes->total);
364 }
365
366 static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
367 {
368 struct mtk_aes_base_ctx *ctx = aes->ctx;
369 struct mtk_aes_info *info = &ctx->info;
370
371 ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
372 DMA_TO_DEVICE);
373 if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
374 goto exit;
375
376 ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd);
377
378 if (aes->src.sg == aes->dst.sg) {
379 aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
380 aes->src.nents,
381 DMA_BIDIRECTIONAL);
382 aes->dst.sg_len = aes->src.sg_len;
383 if (unlikely(!aes->src.sg_len))
384 goto sg_map_err;
385 } else {
386 aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
387 aes->src.nents, DMA_TO_DEVICE);
388 if (unlikely(!aes->src.sg_len))
389 goto sg_map_err;
390
391 aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
392 aes->dst.nents, DMA_FROM_DEVICE);
393 if (unlikely(!aes->dst.sg_len)) {
394 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
395 DMA_TO_DEVICE);
396 goto sg_map_err;
397 }
398 }
399
400 return mtk_aes_xmit(cryp, aes);
401
402 sg_map_err:
403 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE);
404 exit:
405 return mtk_aes_complete(cryp, aes, -EINVAL);
406 }
407
408 /* Initialize transform information of CBC/ECB/CTR mode */
409 static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
410 size_t len)
411 {
412 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
413 struct mtk_aes_base_ctx *ctx = aes->ctx;
414 struct mtk_aes_info *info = &ctx->info;
415 u32 cnt = 0;
416
417 ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
418 info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len);
419 info->cmd[cnt++] = AES_CMD1;
420
421 info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode;
422 if (aes->flags & AES_FLAGS_ENCRYPT)
423 info->tfm[0] |= AES_TFM_BASIC_OUT;
424 else
425 info->tfm[0] |= AES_TFM_BASIC_IN;
426
427 switch (aes->flags & AES_FLAGS_CIPHER_MSK) {
428 case AES_FLAGS_CBC:
429 info->tfm[1] = AES_TFM_CBC;
430 break;
431 case AES_FLAGS_ECB:
432 info->tfm[1] = AES_TFM_ECB;
433 goto ecb;
434 case AES_FLAGS_CTR:
435 info->tfm[1] = AES_TFM_CTR_LOAD;
436 goto ctr;
437
438 default:
439 /* Should not happen... */
440 return;
441 }
442
443 mtk_aes_write_state_le(info->state + ctx->keylen, req->info,
444 AES_BLOCK_SIZE);
445 ctr:
446 info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE));
447 info->tfm[1] |= AES_TFM_FULL_IV;
448 info->cmd[cnt++] = AES_CMD2;
449 ecb:
450 ctx->ct_size = cnt;
451 }
452
453 static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
454 struct scatterlist *src, struct scatterlist *dst,
455 size_t len)
456 {
457 size_t padlen = 0;
458 bool src_aligned, dst_aligned;
459
460 aes->total = len;
461 aes->src.sg = src;
462 aes->dst.sg = dst;
463 aes->real_dst = dst;
464
465 src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
466 if (src == dst)
467 dst_aligned = src_aligned;
468 else
469 dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
470
471 if (!src_aligned || !dst_aligned) {
472 padlen = mtk_aes_padlen(len);
473
474 if (len + padlen > AES_BUF_SIZE)
475 return mtk_aes_complete(cryp, aes, -ENOMEM);
476
477 if (!src_aligned) {
478 sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
479 aes->src.sg = &aes->aligned_sg;
480 aes->src.nents = 1;
481 aes->src.remainder = 0;
482 }
483
484 if (!dst_aligned) {
485 aes->dst.sg = &aes->aligned_sg;
486 aes->dst.nents = 1;
487 aes->dst.remainder = 0;
488 }
489
490 sg_init_table(&aes->aligned_sg, 1);
491 sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);
492 }
493
494 mtk_aes_info_init(cryp, aes, len + padlen);
495
496 return mtk_aes_map(cryp, aes);
497 }
498
499 static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
500 struct crypto_async_request *new_areq)
501 {
502 struct mtk_aes_rec *aes = cryp->aes[id];
503 struct crypto_async_request *areq, *backlog;
504 struct mtk_aes_base_ctx *ctx;
505 unsigned long flags;
506 int ret = 0;
507
508 spin_lock_irqsave(&aes->lock, flags);
509 if (new_areq)
510 ret = crypto_enqueue_request(&aes->queue, new_areq);
511 if (aes->flags & AES_FLAGS_BUSY) {
512 spin_unlock_irqrestore(&aes->lock, flags);
513 return ret;
514 }
515 backlog = crypto_get_backlog(&aes->queue);
516 areq = crypto_dequeue_request(&aes->queue);
517 if (areq)
518 aes->flags |= AES_FLAGS_BUSY;
519 spin_unlock_irqrestore(&aes->lock, flags);
520
521 if (!areq)
522 return ret;
523
524 if (backlog)
525 backlog->complete(backlog, -EINPROGRESS);
526
527 ctx = crypto_tfm_ctx(areq->tfm);
528
529 aes->areq = areq;
530 aes->ctx = ctx;
531
532 return ctx->start(cryp, aes);
533 }
534
535 static int mtk_aes_transfer_complete(struct mtk_cryp *cryp,
536 struct mtk_aes_rec *aes)
537 {
538 return mtk_aes_complete(cryp, aes, 0);
539 }
540
541 static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
542 {
543 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
544 struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
545
546 mtk_aes_set_mode(aes, rctx);
547 aes->resume = mtk_aes_transfer_complete;
548
549 return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes);
550 }
551
552 static inline struct mtk_aes_ctr_ctx *
553 mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx)
554 {
555 return container_of(ctx, struct mtk_aes_ctr_ctx, base);
556 }
557
558 static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
559 {
560 struct mtk_aes_base_ctx *ctx = aes->ctx;
561 struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx);
562 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
563 struct scatterlist *src, *dst;
564 u32 start, end, ctr, blocks;
565 size_t datalen;
566 bool fragmented = false;
567
568 /* Check for transfer completion. */
569 cctx->offset += aes->total;
570 if (cctx->offset >= req->nbytes)
571 return mtk_aes_transfer_complete(cryp, aes);
572
573 /* Compute data length. */
574 datalen = req->nbytes - cctx->offset;
575 blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
576 ctr = be32_to_cpu(cctx->iv[3]);
577
578 /* Check 32bit counter overflow. */
579 start = ctr;
580 end = start + blocks - 1;
581 if (end < start) {
582 ctr |= 0xffffffff;
583 datalen = AES_BLOCK_SIZE * -start;
584 fragmented = true;
585 }
586
587 /* Jump to offset. */
588 src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset);
589 dst = ((req->src == req->dst) ? src :
590 scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset));
591
592 /* Write IVs into transform state buffer. */
593 mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv,
594 AES_BLOCK_SIZE);
595
596 if (unlikely(fragmented)) {
597 /*
598 * Increment the counter manually to cope with the hardware
599 * counter overflow.
600 */
601 cctx->iv[3] = cpu_to_be32(ctr);
602 crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE);
603 }
604
605 return mtk_aes_dma(cryp, aes, src, dst, datalen);
606 }
607
608 static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
609 {
610 struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);
611 struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq);
612 struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req);
613
614 mtk_aes_set_mode(aes, rctx);
615
616 memcpy(cctx->iv, req->info, AES_BLOCK_SIZE);
617 cctx->offset = 0;
618 aes->total = 0;
619 aes->resume = mtk_aes_ctr_transfer;
620
621 return mtk_aes_ctr_transfer(cryp, aes);
622 }
623
624 /* Check and set the AES key to transform state buffer */
625 static int mtk_aes_setkey(struct crypto_ablkcipher *tfm,
626 const u8 *key, u32 keylen)
627 {
628 struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm);
629
630 switch (keylen) {
631 case AES_KEYSIZE_128:
632 ctx->keymode = AES_TFM_128BITS;
633 break;
634 case AES_KEYSIZE_192:
635 ctx->keymode = AES_TFM_192BITS;
636 break;
637 case AES_KEYSIZE_256:
638 ctx->keymode = AES_TFM_256BITS;
639 break;
640
641 default:
642 crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
643 return -EINVAL;
644 }
645
646 ctx->keylen = SIZE_IN_WORDS(keylen);
647 mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen);
648
649 return 0;
650 }
651
652 static int mtk_aes_crypt(struct ablkcipher_request *req, u64 mode)
653 {
654 struct mtk_aes_base_ctx *ctx;
655 struct mtk_aes_reqctx *rctx;
656
657 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
658 rctx = ablkcipher_request_ctx(req);
659 rctx->mode = mode;
660
661 return mtk_aes_handle_queue(ctx->cryp, !(mode & AES_FLAGS_ENCRYPT),
662 &req->base);
663 }
664
665 static int mtk_aes_ecb_encrypt(struct ablkcipher_request *req)
666 {
667 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB);
668 }
669
670 static int mtk_aes_ecb_decrypt(struct ablkcipher_request *req)
671 {
672 return mtk_aes_crypt(req, AES_FLAGS_ECB);
673 }
674
675 static int mtk_aes_cbc_encrypt(struct ablkcipher_request *req)
676 {
677 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC);
678 }
679
680 static int mtk_aes_cbc_decrypt(struct ablkcipher_request *req)
681 {
682 return mtk_aes_crypt(req, AES_FLAGS_CBC);
683 }
684
685 static int mtk_aes_ctr_encrypt(struct ablkcipher_request *req)
686 {
687 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR);
688 }
689
690 static int mtk_aes_ctr_decrypt(struct ablkcipher_request *req)
691 {
692 return mtk_aes_crypt(req, AES_FLAGS_CTR);
693 }
694
695 static int mtk_aes_cra_init(struct crypto_tfm *tfm)
696 {
697 struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
698 struct mtk_cryp *cryp = NULL;
699
700 cryp = mtk_aes_find_dev(&ctx->base);
701 if (!cryp) {
702 pr_err("can't find crypto device\n");
703 return -ENODEV;
704 }
705
706 tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
707 ctx->base.start = mtk_aes_start;
708 return 0;
709 }
710
711 static int mtk_aes_ctr_cra_init(struct crypto_tfm *tfm)
712 {
713 struct mtk_aes_ctx *ctx = crypto_tfm_ctx(tfm);
714 struct mtk_cryp *cryp = NULL;
715
716 cryp = mtk_aes_find_dev(&ctx->base);
717 if (!cryp) {
718 pr_err("can't find crypto device\n");
719 return -ENODEV;
720 }
721
722 tfm->crt_ablkcipher.reqsize = sizeof(struct mtk_aes_reqctx);
723 ctx->base.start = mtk_aes_ctr_start;
724 return 0;
725 }
726
727 static struct crypto_alg aes_algs[] = {
728 {
729 .cra_name = "cbc(aes)",
730 .cra_driver_name = "cbc-aes-mtk",
731 .cra_priority = 400,
732 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
733 CRYPTO_ALG_ASYNC,
734 .cra_init = mtk_aes_cra_init,
735 .cra_blocksize = AES_BLOCK_SIZE,
736 .cra_ctxsize = sizeof(struct mtk_aes_ctx),
737 .cra_alignmask = 0xf,
738 .cra_type = &crypto_ablkcipher_type,
739 .cra_module = THIS_MODULE,
740 .cra_u.ablkcipher = {
741 .min_keysize = AES_MIN_KEY_SIZE,
742 .max_keysize = AES_MAX_KEY_SIZE,
743 .setkey = mtk_aes_setkey,
744 .encrypt = mtk_aes_cbc_encrypt,
745 .decrypt = mtk_aes_cbc_decrypt,
746 .ivsize = AES_BLOCK_SIZE,
747 }
748 },
749 {
750 .cra_name = "ecb(aes)",
751 .cra_driver_name = "ecb-aes-mtk",
752 .cra_priority = 400,
753 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
754 CRYPTO_ALG_ASYNC,
755 .cra_init = mtk_aes_cra_init,
756 .cra_blocksize = AES_BLOCK_SIZE,
757 .cra_ctxsize = sizeof(struct mtk_aes_ctx),
758 .cra_alignmask = 0xf,
759 .cra_type = &crypto_ablkcipher_type,
760 .cra_module = THIS_MODULE,
761 .cra_u.ablkcipher = {
762 .min_keysize = AES_MIN_KEY_SIZE,
763 .max_keysize = AES_MAX_KEY_SIZE,
764 .setkey = mtk_aes_setkey,
765 .encrypt = mtk_aes_ecb_encrypt,
766 .decrypt = mtk_aes_ecb_decrypt,
767 }
768 },
769 {
770 .cra_name = "ctr(aes)",
771 .cra_driver_name = "ctr-aes-mtk",
772 .cra_priority = 400,
773 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
774 CRYPTO_ALG_ASYNC,
775 .cra_init = mtk_aes_ctr_cra_init,
776 .cra_blocksize = 1,
777 .cra_ctxsize = sizeof(struct mtk_aes_ctr_ctx),
778 .cra_alignmask = 0xf,
779 .cra_type = &crypto_ablkcipher_type,
780 .cra_module = THIS_MODULE,
781 .cra_u.ablkcipher = {
782 .min_keysize = AES_MIN_KEY_SIZE,
783 .max_keysize = AES_MAX_KEY_SIZE,
784 .ivsize = AES_BLOCK_SIZE,
785 .setkey = mtk_aes_setkey,
786 .encrypt = mtk_aes_ctr_encrypt,
787 .decrypt = mtk_aes_ctr_decrypt,
788 }
789 },
790 };
791
792 static inline struct mtk_aes_gcm_ctx *
793 mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx)
794 {
795 return container_of(ctx, struct mtk_aes_gcm_ctx, base);
796 }
797
798 /*
799 * Engine will verify and compare tag automatically, so we just need
800 * to check returned status which stored in the result descriptor.
801 */
802 static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp,
803 struct mtk_aes_rec *aes)
804 {
805 u32 status = cryp->ring[aes->id]->res_prev->ct;
806
807 return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?
808 -EBADMSG : 0);
809 }
810
811 /* Initialize transform information of GCM mode */
812 static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp,
813 struct mtk_aes_rec *aes,
814 size_t len)
815 {
816 struct aead_request *req = aead_request_cast(aes->areq);
817 struct mtk_aes_base_ctx *ctx = aes->ctx;
818 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
819 struct mtk_aes_info *info = &ctx->info;
820 u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
821 u32 cnt = 0;
822
823 ctx->ct_hdr = AES_CT_CTRL_HDR | len;
824
825 info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen);
826 info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen);
827 info->cmd[cnt++] = AES_GCM_CMD2;
828 info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen);
829
830 if (aes->flags & AES_FLAGS_ENCRYPT) {
831 info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize);
832 info->tfm[0] = AES_TFM_GCM_OUT;
833 } else {
834 info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize);
835 info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize);
836 info->tfm[0] = AES_TFM_GCM_IN;
837 }
838 ctx->ct_size = cnt;
839
840 info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE(
841 ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) |
842 ctx->keymode;
843 info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV |
844 AES_TFM_ENC_HASH;
845
846 mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS(
847 AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize);
848 }
849
850 static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
851 struct scatterlist *src, struct scatterlist *dst,
852 size_t len)
853 {
854 bool src_aligned, dst_aligned;
855
856 aes->src.sg = src;
857 aes->dst.sg = dst;
858 aes->real_dst = dst;
859
860 src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
861 if (src == dst)
862 dst_aligned = src_aligned;
863 else
864 dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
865
866 if (!src_aligned || !dst_aligned) {
867 if (aes->total > AES_BUF_SIZE)
868 return mtk_aes_complete(cryp, aes, -ENOMEM);
869
870 if (!src_aligned) {
871 sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
872 aes->src.sg = &aes->aligned_sg;
873 aes->src.nents = 1;
874 aes->src.remainder = 0;
875 }
876
877 if (!dst_aligned) {
878 aes->dst.sg = &aes->aligned_sg;
879 aes->dst.nents = 1;
880 aes->dst.remainder = 0;
881 }
882
883 sg_init_table(&aes->aligned_sg, 1);
884 sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);
885 }
886
887 mtk_aes_gcm_info_init(cryp, aes, len);
888
889 return mtk_aes_map(cryp, aes);
890 }
891
892 /* Todo: GMAC */
893 static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
894 {
895 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);
896 struct aead_request *req = aead_request_cast(aes->areq);
897 struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
898 u32 len = req->assoclen + req->cryptlen;
899
900 mtk_aes_set_mode(aes, rctx);
901
902 if (aes->flags & AES_FLAGS_ENCRYPT) {
903 u32 tag[4];
904
905 aes->resume = mtk_aes_transfer_complete;
906 /* Compute total process length. */
907 aes->total = len + gctx->authsize;
908 /* Compute text length. */
909 gctx->textlen = req->cryptlen;
910 /* Hardware will append authenticated tag to output buffer */
911 scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1);
912 } else {
913 aes->resume = mtk_aes_gcm_tag_verify;
914 aes->total = len;
915 gctx->textlen = req->cryptlen - gctx->authsize;
916 }
917
918 return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);
919 }
920
921 static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
922 {
923 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
924 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
925 struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
926
927 /* Empty messages are not supported yet */
928 if (!gctx->textlen && !req->assoclen)
929 return -EINVAL;
930
931 rctx->mode = AES_FLAGS_GCM | mode;
932
933 return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT),
934 &req->base);
935 }
936
937 /*
938 * Because of the hardware limitation, we need to pre-calculate key(H)
939 * for the GHASH operation. The result of the encryption operation
940 * need to be stored in the transform state buffer.
941 */
942 static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
943 u32 keylen)
944 {
945 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
946 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
947 struct crypto_skcipher *ctr = gctx->ctr;
948 struct {
949 u32 hash[4];
950 u8 iv[8];
951
952 struct crypto_wait wait;
953
954 struct scatterlist sg[1];
955 struct skcipher_request req;
956 } *data;
957 int err;
958
959 switch (keylen) {
960 case AES_KEYSIZE_128:
961 ctx->keymode = AES_TFM_128BITS;
962 break;
963 case AES_KEYSIZE_192:
964 ctx->keymode = AES_TFM_192BITS;
965 break;
966 case AES_KEYSIZE_256:
967 ctx->keymode = AES_TFM_256BITS;
968 break;
969
970 default:
971 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
972 return -EINVAL;
973 }
974
975 ctx->keylen = SIZE_IN_WORDS(keylen);
976
977 /* Same as crypto_gcm_setkey() from crypto/gcm.c */
978 crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
979 crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
980 CRYPTO_TFM_REQ_MASK);
981 err = crypto_skcipher_setkey(ctr, key, keylen);
982 crypto_aead_set_flags(aead, crypto_skcipher_get_flags(ctr) &
983 CRYPTO_TFM_RES_MASK);
984 if (err)
985 return err;
986
987 data = kzalloc(sizeof(*data) + crypto_skcipher_reqsize(ctr),
988 GFP_KERNEL);
989 if (!data)
990 return -ENOMEM;
991
992 crypto_init_wait(&data->wait);
993 sg_init_one(data->sg, &data->hash, AES_BLOCK_SIZE);
994 skcipher_request_set_tfm(&data->req, ctr);
995 skcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
996 CRYPTO_TFM_REQ_MAY_BACKLOG,
997 crypto_req_done, &data->wait);
998 skcipher_request_set_crypt(&data->req, data->sg, data->sg,
999 AES_BLOCK_SIZE, data->iv);
1000
1001 err = crypto_wait_req(crypto_skcipher_encrypt(&data->req),
1002 &data->wait);
1003 if (err)
1004 goto out;
1005
1006 /* Write key into state buffer */
1007 mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen);
1008 /* Write key(H) into state buffer */
1009 mtk_aes_write_state_be(ctx->info.state + ctx->keylen, data->hash,
1010 AES_BLOCK_SIZE);
1011 out:
1012 kzfree(data);
1013 return err;
1014 }
1015
1016 static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead,
1017 u32 authsize)
1018 {
1019 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
1020 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
1021
1022 /* Same as crypto_gcm_authsize() from crypto/gcm.c */
1023 switch (authsize) {
1024 case 8:
1025 case 12:
1026 case 16:
1027 break;
1028 default:
1029 return -EINVAL;
1030 }
1031
1032 gctx->authsize = authsize;
1033 return 0;
1034 }
1035
1036 static int mtk_aes_gcm_encrypt(struct aead_request *req)
1037 {
1038 return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
1039 }
1040
1041 static int mtk_aes_gcm_decrypt(struct aead_request *req)
1042 {
1043 return mtk_aes_gcm_crypt(req, 0);
1044 }
1045
1046 static int mtk_aes_gcm_init(struct crypto_aead *aead)
1047 {
1048 struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
1049 struct mtk_cryp *cryp = NULL;
1050
1051 cryp = mtk_aes_find_dev(&ctx->base);
1052 if (!cryp) {
1053 pr_err("can't find crypto device\n");
1054 return -ENODEV;
1055 }
1056
1057 ctx->ctr = crypto_alloc_skcipher("ctr(aes)", 0,
1058 CRYPTO_ALG_ASYNC);
1059 if (IS_ERR(ctx->ctr)) {
1060 pr_err("Error allocating ctr(aes)\n");
1061 return PTR_ERR(ctx->ctr);
1062 }
1063
1064 crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx));
1065 ctx->base.start = mtk_aes_gcm_start;
1066 return 0;
1067 }
1068
1069 static void mtk_aes_gcm_exit(struct crypto_aead *aead)
1070 {
1071 struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
1072
1073 crypto_free_skcipher(ctx->ctr);
1074 }
1075
1076 static struct aead_alg aes_gcm_alg = {
1077 .setkey = mtk_aes_gcm_setkey,
1078 .setauthsize = mtk_aes_gcm_setauthsize,
1079 .encrypt = mtk_aes_gcm_encrypt,
1080 .decrypt = mtk_aes_gcm_decrypt,
1081 .init = mtk_aes_gcm_init,
1082 .exit = mtk_aes_gcm_exit,
1083 .ivsize = GCM_AES_IV_SIZE,
1084 .maxauthsize = AES_BLOCK_SIZE,
1085
1086 .base = {
1087 .cra_name = "gcm(aes)",
1088 .cra_driver_name = "gcm-aes-mtk",
1089 .cra_priority = 400,
1090 .cra_flags = CRYPTO_ALG_ASYNC,
1091 .cra_blocksize = 1,
1092 .cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx),
1093 .cra_alignmask = 0xf,
1094 .cra_module = THIS_MODULE,
1095 },
1096 };
1097
1098 static void mtk_aes_queue_task(unsigned long data)
1099 {
1100 struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;
1101
1102 mtk_aes_handle_queue(aes->cryp, aes->id, NULL);
1103 }
1104
1105 static void mtk_aes_done_task(unsigned long data)
1106 {
1107 struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;
1108 struct mtk_cryp *cryp = aes->cryp;
1109
1110 mtk_aes_unmap(cryp, aes);
1111 aes->resume(cryp, aes);
1112 }
1113
1114 static irqreturn_t mtk_aes_irq(int irq, void *dev_id)
1115 {
1116 struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id;
1117 struct mtk_cryp *cryp = aes->cryp;
1118 u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id));
1119
1120 mtk_aes_write(cryp, RDR_STAT(aes->id), val);
1121
1122 if (likely(AES_FLAGS_BUSY & aes->flags)) {
1123 mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST);
1124 mtk_aes_write(cryp, RDR_THRESH(aes->id),
1125 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1126
1127 tasklet_schedule(&aes->done_task);
1128 } else {
1129 dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
1130 }
1131 return IRQ_HANDLED;
1132 }
1133
1134 /*
1135 * The purpose of creating encryption and decryption records is
1136 * to process outbound/inbound data in parallel, it can improve
1137 * performance in most use cases, such as IPSec VPN, especially
1138 * under heavy network traffic.
1139 */
1140 static int mtk_aes_record_init(struct mtk_cryp *cryp)
1141 {
1142 struct mtk_aes_rec **aes = cryp->aes;
1143 int i, err = -ENOMEM;
1144
1145 for (i = 0; i < MTK_REC_NUM; i++) {
1146 aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL);
1147 if (!aes[i])
1148 goto err_cleanup;
1149
1150 aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL,
1151 AES_BUF_ORDER);
1152 if (!aes[i]->buf)
1153 goto err_cleanup;
1154
1155 aes[i]->cryp = cryp;
1156
1157 spin_lock_init(&aes[i]->lock);
1158 crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE);
1159
1160 tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task,
1161 (unsigned long)aes[i]);
1162 tasklet_init(&aes[i]->done_task, mtk_aes_done_task,
1163 (unsigned long)aes[i]);
1164 }
1165
1166 /* Link to ring0 and ring1 respectively */
1167 aes[0]->id = MTK_RING0;
1168 aes[1]->id = MTK_RING1;
1169
1170 return 0;
1171
1172 err_cleanup:
1173 for (; i--; ) {
1174 free_page((unsigned long)aes[i]->buf);
1175 kfree(aes[i]);
1176 }
1177
1178 return err;
1179 }
1180
1181 static void mtk_aes_record_free(struct mtk_cryp *cryp)
1182 {
1183 int i;
1184
1185 for (i = 0; i < MTK_REC_NUM; i++) {
1186 tasklet_kill(&cryp->aes[i]->done_task);
1187 tasklet_kill(&cryp->aes[i]->queue_task);
1188
1189 free_page((unsigned long)cryp->aes[i]->buf);
1190 kfree(cryp->aes[i]);
1191 }
1192 }
1193
1194 static void mtk_aes_unregister_algs(void)
1195 {
1196 int i;
1197
1198 crypto_unregister_aead(&aes_gcm_alg);
1199
1200 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1201 crypto_unregister_alg(&aes_algs[i]);
1202 }
1203
1204 static int mtk_aes_register_algs(void)
1205 {
1206 int err, i;
1207
1208 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1209 err = crypto_register_alg(&aes_algs[i]);
1210 if (err)
1211 goto err_aes_algs;
1212 }
1213
1214 err = crypto_register_aead(&aes_gcm_alg);
1215 if (err)
1216 goto err_aes_algs;
1217
1218 return 0;
1219
1220 err_aes_algs:
1221 for (; i--; )
1222 crypto_unregister_alg(&aes_algs[i]);
1223
1224 return err;
1225 }
1226
1227 int mtk_cipher_alg_register(struct mtk_cryp *cryp)
1228 {
1229 int ret;
1230
1231 INIT_LIST_HEAD(&cryp->aes_list);
1232
1233 /* Initialize two cipher records */
1234 ret = mtk_aes_record_init(cryp);
1235 if (ret)
1236 goto err_record;
1237
1238 ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq,
1239 0, "mtk-aes", cryp->aes[0]);
1240 if (ret) {
1241 dev_err(cryp->dev, "unable to request AES irq.\n");
1242 goto err_res;
1243 }
1244
1245 ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq,
1246 0, "mtk-aes", cryp->aes[1]);
1247 if (ret) {
1248 dev_err(cryp->dev, "unable to request AES irq.\n");
1249 goto err_res;
1250 }
1251
1252 /* Enable ring0 and ring1 interrupt */
1253 mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0);
1254 mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1);
1255
1256 spin_lock(&mtk_aes.lock);
1257 list_add_tail(&cryp->aes_list, &mtk_aes.dev_list);
1258 spin_unlock(&mtk_aes.lock);
1259
1260 ret = mtk_aes_register_algs();
1261 if (ret)
1262 goto err_algs;
1263
1264 return 0;
1265
1266 err_algs:
1267 spin_lock(&mtk_aes.lock);
1268 list_del(&cryp->aes_list);
1269 spin_unlock(&mtk_aes.lock);
1270 err_res:
1271 mtk_aes_record_free(cryp);
1272 err_record:
1273
1274 dev_err(cryp->dev, "mtk-aes initialization failed.\n");
1275 return ret;
1276 }
1277
1278 void mtk_cipher_alg_release(struct mtk_cryp *cryp)
1279 {
1280 spin_lock(&mtk_aes.lock);
1281 list_del(&cryp->aes_list);
1282 spin_unlock(&mtk_aes.lock);
1283
1284 mtk_aes_unregister_algs();
1285 mtk_aes_record_free(cryp);
1286 }
Linux with added FPC-III support