?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'regmap-fix-v5.11-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux/fpc-iii.git] / drivers / crypto / mediatek / mtk-aes.c
blob7323066724c3b94958e32167b160017b8359b131
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Cryptographic API.
4 *
5 * Driver for EIP97 AES acceleration.
6 *
7 * Copyright (c) 2016 Ryder Lee <ryder.lee@mediatek.com>
8 *
9 * Some ideas are from atmel-aes.c drivers.
10 */
11
12 #include <crypto/aes.h>
13 #include <crypto/gcm.h>
14 #include <crypto/internal/skcipher.h>
15 #include "mtk-platform.h"
16
17 #define AES_QUEUE_SIZE 512
18 #define AES_BUF_ORDER 2
19 #define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \
20 & ~(AES_BLOCK_SIZE - 1))
21 #define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \
22 AES_BLOCK_SIZE * 2)
23 #define AES_MAX_CT_SIZE 6
24
25 #define AES_CT_CTRL_HDR cpu_to_le32(0x00220000)
26
27 /* AES-CBC/ECB/CTR/OFB/CFB command token */
28 #define AES_CMD0 cpu_to_le32(0x05000000)
29 #define AES_CMD1 cpu_to_le32(0x2d060000)
30 #define AES_CMD2 cpu_to_le32(0xe4a63806)
31 /* AES-GCM command token */
32 #define AES_GCM_CMD0 cpu_to_le32(0x0b000000)
33 #define AES_GCM_CMD1 cpu_to_le32(0xa0800000)
34 #define AES_GCM_CMD2 cpu_to_le32(0x25000010)
35 #define AES_GCM_CMD3 cpu_to_le32(0x0f020000)
36 #define AES_GCM_CMD4 cpu_to_le32(0x21e60000)
37 #define AES_GCM_CMD5 cpu_to_le32(0x40e60000)
38 #define AES_GCM_CMD6 cpu_to_le32(0xd0070000)
39
40 /* AES transform information word 0 fields */
41 #define AES_TFM_BASIC_OUT cpu_to_le32(0x4 << 0)
42 #define AES_TFM_BASIC_IN cpu_to_le32(0x5 << 0)
43 #define AES_TFM_GCM_OUT cpu_to_le32(0x6 << 0)
44 #define AES_TFM_GCM_IN cpu_to_le32(0xf << 0)
45 #define AES_TFM_SIZE(x) cpu_to_le32((x) << 8)
46 #define AES_TFM_128BITS cpu_to_le32(0xb << 16)
47 #define AES_TFM_192BITS cpu_to_le32(0xd << 16)
48 #define AES_TFM_256BITS cpu_to_le32(0xf << 16)
49 #define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21)
50 #define AES_TFM_GHASH cpu_to_le32(0x4 << 23)
51 /* AES transform information word 1 fields */
52 #define AES_TFM_ECB cpu_to_le32(0x0 << 0)
53 #define AES_TFM_CBC cpu_to_le32(0x1 << 0)
54 #define AES_TFM_OFB cpu_to_le32(0x4 << 0)
55 #define AES_TFM_CFB128 cpu_to_le32(0x5 << 0)
56 #define AES_TFM_CTR_INIT cpu_to_le32(0x2 << 0) /* init counter to 1 */
57 #define AES_TFM_CTR_LOAD cpu_to_le32(0x6 << 0) /* load/reuse counter */
58 #define AES_TFM_3IV cpu_to_le32(0x7 << 5) /* using IV 0-2 */
59 #define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */
60 #define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10)
61 #define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17)
62
63 /* AES flags */
64 #define AES_FLAGS_CIPHER_MSK GENMASK(4, 0)
65 #define AES_FLAGS_ECB BIT(0)
66 #define AES_FLAGS_CBC BIT(1)
67 #define AES_FLAGS_CTR BIT(2)
68 #define AES_FLAGS_OFB BIT(3)
69 #define AES_FLAGS_CFB128 BIT(4)
70 #define AES_FLAGS_GCM BIT(5)
71 #define AES_FLAGS_ENCRYPT BIT(6)
72 #define AES_FLAGS_BUSY BIT(7)
73
74 #define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26))
75
76 /**
77 * mtk_aes_info - hardware information of AES
78 * @cmd: command token, hardware instruction
79 * @tfm: transform state of cipher algorithm.
80 * @state: contains keys and initial vectors.
81 *
82 * Memory layout of GCM buffer:
83 * /-----------\
84 * | AES KEY | 128/196/256 bits
85 * |-----------|
86 * | HASH KEY | a string 128 zero bits encrypted using the block cipher
87 * |-----------|
88 * | IVs | 4 * 4 bytes
89 * \-----------/
90 *
91 * The engine requires all these info to do:
92 * - Commands decoding and control of the engine's data path.
93 * - Coordinating hardware data fetch and store operations.
94 * - Result token construction and output.
95 */
96 struct mtk_aes_info {
97 __le32 cmd[AES_MAX_CT_SIZE];
98 __le32 tfm[2];
99 __le32 state[AES_MAX_STATE_BUF_SIZE];
100 };
101
102 struct mtk_aes_reqctx {
103 u64 mode;
104 };
105
106 struct mtk_aes_base_ctx {
107 struct mtk_cryp *cryp;
108 u32 keylen;
109 __le32 key[12];
110 __le32 keymode;
111
112 mtk_aes_fn start;
113
114 struct mtk_aes_info info;
115 dma_addr_t ct_dma;
116 dma_addr_t tfm_dma;
117
118 __le32 ct_hdr;
119 u32 ct_size;
120 };
121
122 struct mtk_aes_ctx {
123 struct mtk_aes_base_ctx base;
124 };
125
126 struct mtk_aes_ctr_ctx {
127 struct mtk_aes_base_ctx base;
128
129 __be32 iv[AES_BLOCK_SIZE / sizeof(u32)];
130 size_t offset;
131 struct scatterlist src[2];
132 struct scatterlist dst[2];
133 };
134
135 struct mtk_aes_gcm_ctx {
136 struct mtk_aes_base_ctx base;
137
138 u32 authsize;
139 size_t textlen;
140 };
141
142 struct mtk_aes_drv {
143 struct list_head dev_list;
144 /* Device list lock */
145 spinlock_t lock;
146 };
147
148 static struct mtk_aes_drv mtk_aes = {
149 .dev_list = LIST_HEAD_INIT(mtk_aes.dev_list),
150 .lock = __SPIN_LOCK_UNLOCKED(mtk_aes.lock),
151 };
152
153 static inline u32 mtk_aes_read(struct mtk_cryp *cryp, u32 offset)
154 {
155 return readl_relaxed(cryp->base + offset);
156 }
157
158 static inline void mtk_aes_write(struct mtk_cryp *cryp,
159 u32 offset, u32 value)
160 {
161 writel_relaxed(value, cryp->base + offset);
162 }
163
164 static struct mtk_cryp *mtk_aes_find_dev(struct mtk_aes_base_ctx *ctx)
165 {
166 struct mtk_cryp *cryp = NULL;
167 struct mtk_cryp *tmp;
168
169 spin_lock_bh(&mtk_aes.lock);
170 if (!ctx->cryp) {
171 list_for_each_entry(tmp, &mtk_aes.dev_list, aes_list) {
172 cryp = tmp;
173 break;
174 }
175 ctx->cryp = cryp;
176 } else {
177 cryp = ctx->cryp;
178 }
179 spin_unlock_bh(&mtk_aes.lock);
180
181 return cryp;
182 }
183
184 static inline size_t mtk_aes_padlen(size_t len)
185 {
186 len &= AES_BLOCK_SIZE - 1;
187 return len ? AES_BLOCK_SIZE - len : 0;
188 }
189
190 static bool mtk_aes_check_aligned(struct scatterlist *sg, size_t len,
191 struct mtk_aes_dma *dma)
192 {
193 int nents;
194
195 if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
196 return false;
197
198 for (nents = 0; sg; sg = sg_next(sg), ++nents) {
199 if (!IS_ALIGNED(sg->offset, sizeof(u32)))
200 return false;
201
202 if (len <= sg->length) {
203 if (!IS_ALIGNED(len, AES_BLOCK_SIZE))
204 return false;
205
206 dma->nents = nents + 1;
207 dma->remainder = sg->length - len;
208 sg->length = len;
209 return true;
210 }
211
212 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
213 return false;
214
215 len -= sg->length;
216 }
217
218 return false;
219 }
220
221 static inline void mtk_aes_set_mode(struct mtk_aes_rec *aes,
222 const struct mtk_aes_reqctx *rctx)
223 {
224 /* Clear all but persistent flags and set request flags. */
225 aes->flags = (aes->flags & AES_FLAGS_BUSY) | rctx->mode;
226 }
227
228 static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma)
229 {
230 struct scatterlist *sg = dma->sg;
231 int nents = dma->nents;
232
233 if (!dma->remainder)
234 return;
235
236 while (--nents > 0 && sg)
237 sg = sg_next(sg);
238
239 if (!sg)
240 return;
241
242 sg->length += dma->remainder;
243 }
244
245 static inline int mtk_aes_complete(struct mtk_cryp *cryp,
246 struct mtk_aes_rec *aes,
247 int err)
248 {
249 aes->flags &= ~AES_FLAGS_BUSY;
250 aes->areq->complete(aes->areq, err);
251 /* Handle new request */
252 tasklet_schedule(&aes->queue_task);
253 return err;
254 }
255
256 /*
257 * Write descriptors for processing. This will configure the engine, load
258 * the transform information and then start the packet processing.
259 */
260 static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
261 {
262 struct mtk_ring *ring = cryp->ring[aes->id];
263 struct mtk_desc *cmd = NULL, *res = NULL;
264 struct scatterlist *ssg = aes->src.sg, *dsg = aes->dst.sg;
265 u32 slen = aes->src.sg_len, dlen = aes->dst.sg_len;
266 int nents;
267
268 /* Write command descriptors */
269 for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) {
270 cmd = ring->cmd_next;
271 cmd->hdr = MTK_DESC_BUF_LEN(ssg->length);
272 cmd->buf = cpu_to_le32(sg_dma_address(ssg));
273
274 if (nents == 0) {
275 cmd->hdr |= MTK_DESC_FIRST |
276 MTK_DESC_CT_LEN(aes->ctx->ct_size);
277 cmd->ct = cpu_to_le32(aes->ctx->ct_dma);
278 cmd->ct_hdr = aes->ctx->ct_hdr;
279 cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma);
280 }
281
282 /* Shift ring buffer and check boundary */
283 if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM)
284 ring->cmd_next = ring->cmd_base;
285 }
286 cmd->hdr |= MTK_DESC_LAST;
287
288 /* Prepare result descriptors */
289 for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) {
290 res = ring->res_next;
291 res->hdr = MTK_DESC_BUF_LEN(dsg->length);
292 res->buf = cpu_to_le32(sg_dma_address(dsg));
293
294 if (nents == 0)
295 res->hdr |= MTK_DESC_FIRST;
296
297 /* Shift ring buffer and check boundary */
298 if (++ring->res_next == ring->res_base + MTK_DESC_NUM)
299 ring->res_next = ring->res_base;
300 }
301 res->hdr |= MTK_DESC_LAST;
302
303 /* Pointer to current result descriptor */
304 ring->res_prev = res;
305
306 /* Prepare enough space for authenticated tag */
307 if (aes->flags & AES_FLAGS_GCM)
308 le32_add_cpu(&res->hdr, AES_BLOCK_SIZE);
309
310 /*
311 * Make sure that all changes to the DMA ring are done before we
312 * start engine.
313 */
314 wmb();
315 /* Start DMA transfer */
316 mtk_aes_write(cryp, RDR_PREP_COUNT(aes->id), MTK_DESC_CNT(dlen));
317 mtk_aes_write(cryp, CDR_PREP_COUNT(aes->id), MTK_DESC_CNT(slen));
318
319 return -EINPROGRESS;
320 }
321
322 static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
323 {
324 struct mtk_aes_base_ctx *ctx = aes->ctx;
325
326 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info),
327 DMA_TO_DEVICE);
328
329 if (aes->src.sg == aes->dst.sg) {
330 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
331 DMA_BIDIRECTIONAL);
332
333 if (aes->src.sg != &aes->aligned_sg)
334 mtk_aes_restore_sg(&aes->src);
335 } else {
336 dma_unmap_sg(cryp->dev, aes->dst.sg, aes->dst.nents,
337 DMA_FROM_DEVICE);
338
339 if (aes->dst.sg != &aes->aligned_sg)
340 mtk_aes_restore_sg(&aes->dst);
341
342 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
343 DMA_TO_DEVICE);
344
345 if (aes->src.sg != &aes->aligned_sg)
346 mtk_aes_restore_sg(&aes->src);
347 }
348
349 if (aes->dst.sg == &aes->aligned_sg)
350 sg_copy_from_buffer(aes->real_dst, sg_nents(aes->real_dst),
351 aes->buf, aes->total);
352 }
353
354 static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
355 {
356 struct mtk_aes_base_ctx *ctx = aes->ctx;
357 struct mtk_aes_info *info = &ctx->info;
358
359 ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info),
360 DMA_TO_DEVICE);
361 if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma)))
362 goto exit;
363
364 ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd);
365
366 if (aes->src.sg == aes->dst.sg) {
367 aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
368 aes->src.nents,
369 DMA_BIDIRECTIONAL);
370 aes->dst.sg_len = aes->src.sg_len;
371 if (unlikely(!aes->src.sg_len))
372 goto sg_map_err;
373 } else {
374 aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg,
375 aes->src.nents, DMA_TO_DEVICE);
376 if (unlikely(!aes->src.sg_len))
377 goto sg_map_err;
378
379 aes->dst.sg_len = dma_map_sg(cryp->dev, aes->dst.sg,
380 aes->dst.nents, DMA_FROM_DEVICE);
381 if (unlikely(!aes->dst.sg_len)) {
382 dma_unmap_sg(cryp->dev, aes->src.sg, aes->src.nents,
383 DMA_TO_DEVICE);
384 goto sg_map_err;
385 }
386 }
387
388 return mtk_aes_xmit(cryp, aes);
389
390 sg_map_err:
391 dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE);
392 exit:
393 return mtk_aes_complete(cryp, aes, -EINVAL);
394 }
395
396 /* Initialize transform information of CBC/ECB/CTR/OFB/CFB mode */
397 static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
398 size_t len)
399 {
400 struct skcipher_request *req = skcipher_request_cast(aes->areq);
401 struct mtk_aes_base_ctx *ctx = aes->ctx;
402 struct mtk_aes_info *info = &ctx->info;
403 u32 cnt = 0;
404
405 ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
406 info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len);
407 info->cmd[cnt++] = AES_CMD1;
408
409 info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode;
410 if (aes->flags & AES_FLAGS_ENCRYPT)
411 info->tfm[0] |= AES_TFM_BASIC_OUT;
412 else
413 info->tfm[0] |= AES_TFM_BASIC_IN;
414
415 switch (aes->flags & AES_FLAGS_CIPHER_MSK) {
416 case AES_FLAGS_CBC:
417 info->tfm[1] = AES_TFM_CBC;
418 break;
419 case AES_FLAGS_ECB:
420 info->tfm[1] = AES_TFM_ECB;
421 goto ecb;
422 case AES_FLAGS_CTR:
423 info->tfm[1] = AES_TFM_CTR_LOAD;
424 goto ctr;
425 case AES_FLAGS_OFB:
426 info->tfm[1] = AES_TFM_OFB;
427 break;
428 case AES_FLAGS_CFB128:
429 info->tfm[1] = AES_TFM_CFB128;
430 break;
431 default:
432 /* Should not happen... */
433 return;
434 }
435
436 memcpy(info->state + ctx->keylen, req->iv, AES_BLOCK_SIZE);
437 ctr:
438 le32_add_cpu(&info->tfm[0],
439 le32_to_cpu(AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE))));
440 info->tfm[1] |= AES_TFM_FULL_IV;
441 info->cmd[cnt++] = AES_CMD2;
442 ecb:
443 ctx->ct_size = cnt;
444 }
445
446 static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
447 struct scatterlist *src, struct scatterlist *dst,
448 size_t len)
449 {
450 size_t padlen = 0;
451 bool src_aligned, dst_aligned;
452
453 aes->total = len;
454 aes->src.sg = src;
455 aes->dst.sg = dst;
456 aes->real_dst = dst;
457
458 src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
459 if (src == dst)
460 dst_aligned = src_aligned;
461 else
462 dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
463
464 if (!src_aligned || !dst_aligned) {
465 padlen = mtk_aes_padlen(len);
466
467 if (len + padlen > AES_BUF_SIZE)
468 return mtk_aes_complete(cryp, aes, -ENOMEM);
469
470 if (!src_aligned) {
471 sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
472 aes->src.sg = &aes->aligned_sg;
473 aes->src.nents = 1;
474 aes->src.remainder = 0;
475 }
476
477 if (!dst_aligned) {
478 aes->dst.sg = &aes->aligned_sg;
479 aes->dst.nents = 1;
480 aes->dst.remainder = 0;
481 }
482
483 sg_init_table(&aes->aligned_sg, 1);
484 sg_set_buf(&aes->aligned_sg, aes->buf, len + padlen);
485 }
486
487 mtk_aes_info_init(cryp, aes, len + padlen);
488
489 return mtk_aes_map(cryp, aes);
490 }
491
492 static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id,
493 struct crypto_async_request *new_areq)
494 {
495 struct mtk_aes_rec *aes = cryp->aes[id];
496 struct crypto_async_request *areq, *backlog;
497 struct mtk_aes_base_ctx *ctx;
498 unsigned long flags;
499 int ret = 0;
500
501 spin_lock_irqsave(&aes->lock, flags);
502 if (new_areq)
503 ret = crypto_enqueue_request(&aes->queue, new_areq);
504 if (aes->flags & AES_FLAGS_BUSY) {
505 spin_unlock_irqrestore(&aes->lock, flags);
506 return ret;
507 }
508 backlog = crypto_get_backlog(&aes->queue);
509 areq = crypto_dequeue_request(&aes->queue);
510 if (areq)
511 aes->flags |= AES_FLAGS_BUSY;
512 spin_unlock_irqrestore(&aes->lock, flags);
513
514 if (!areq)
515 return ret;
516
517 if (backlog)
518 backlog->complete(backlog, -EINPROGRESS);
519
520 ctx = crypto_tfm_ctx(areq->tfm);
521 /* Write key into state buffer */
522 memcpy(ctx->info.state, ctx->key, sizeof(ctx->key));
523
524 aes->areq = areq;
525 aes->ctx = ctx;
526
527 return ctx->start(cryp, aes);
528 }
529
530 static int mtk_aes_transfer_complete(struct mtk_cryp *cryp,
531 struct mtk_aes_rec *aes)
532 {
533 return mtk_aes_complete(cryp, aes, 0);
534 }
535
536 static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
537 {
538 struct skcipher_request *req = skcipher_request_cast(aes->areq);
539 struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req);
540
541 mtk_aes_set_mode(aes, rctx);
542 aes->resume = mtk_aes_transfer_complete;
543
544 return mtk_aes_dma(cryp, aes, req->src, req->dst, req->cryptlen);
545 }
546
547 static inline struct mtk_aes_ctr_ctx *
548 mtk_aes_ctr_ctx_cast(struct mtk_aes_base_ctx *ctx)
549 {
550 return container_of(ctx, struct mtk_aes_ctr_ctx, base);
551 }
552
553 static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
554 {
555 struct mtk_aes_base_ctx *ctx = aes->ctx;
556 struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx);
557 struct skcipher_request *req = skcipher_request_cast(aes->areq);
558 struct scatterlist *src, *dst;
559 u32 start, end, ctr, blocks;
560 size_t datalen;
561 bool fragmented = false;
562
563 /* Check for transfer completion. */
564 cctx->offset += aes->total;
565 if (cctx->offset >= req->cryptlen)
566 return mtk_aes_transfer_complete(cryp, aes);
567
568 /* Compute data length. */
569 datalen = req->cryptlen - cctx->offset;
570 blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
571 ctr = be32_to_cpu(cctx->iv[3]);
572
573 /* Check 32bit counter overflow. */
574 start = ctr;
575 end = start + blocks - 1;
576 if (end < start) {
577 ctr = 0xffffffff;
578 datalen = AES_BLOCK_SIZE * -start;
579 fragmented = true;
580 }
581
582 /* Jump to offset. */
583 src = scatterwalk_ffwd(cctx->src, req->src, cctx->offset);
584 dst = ((req->src == req->dst) ? src :
585 scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset));
586
587 /* Write IVs into transform state buffer. */
588 memcpy(ctx->info.state + ctx->keylen, cctx->iv, AES_BLOCK_SIZE);
589
590 if (unlikely(fragmented)) {
591 /*
592 * Increment the counter manually to cope with the hardware
593 * counter overflow.
594 */
595 cctx->iv[3] = cpu_to_be32(ctr);
596 crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE);
597 }
598
599 return mtk_aes_dma(cryp, aes, src, dst, datalen);
600 }
601
602 static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
603 {
604 struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(aes->ctx);
605 struct skcipher_request *req = skcipher_request_cast(aes->areq);
606 struct mtk_aes_reqctx *rctx = skcipher_request_ctx(req);
607
608 mtk_aes_set_mode(aes, rctx);
609
610 memcpy(cctx->iv, req->iv, AES_BLOCK_SIZE);
611 cctx->offset = 0;
612 aes->total = 0;
613 aes->resume = mtk_aes_ctr_transfer;
614
615 return mtk_aes_ctr_transfer(cryp, aes);
616 }
617
618 /* Check and set the AES key to transform state buffer */
619 static int mtk_aes_setkey(struct crypto_skcipher *tfm,
620 const u8 *key, u32 keylen)
621 {
622 struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm);
623
624 switch (keylen) {
625 case AES_KEYSIZE_128:
626 ctx->keymode = AES_TFM_128BITS;
627 break;
628 case AES_KEYSIZE_192:
629 ctx->keymode = AES_TFM_192BITS;
630 break;
631 case AES_KEYSIZE_256:
632 ctx->keymode = AES_TFM_256BITS;
633 break;
634
635 default:
636 return -EINVAL;
637 }
638
639 ctx->keylen = SIZE_IN_WORDS(keylen);
640 memcpy(ctx->key, key, keylen);
641
642 return 0;
643 }
644
645 static int mtk_aes_crypt(struct skcipher_request *req, u64 mode)
646 {
647 struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
648 struct mtk_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher);
649 struct mtk_aes_reqctx *rctx;
650 struct mtk_cryp *cryp;
651
652 cryp = mtk_aes_find_dev(ctx);
653 if (!cryp)
654 return -ENODEV;
655
656 rctx = skcipher_request_ctx(req);
657 rctx->mode = mode;
658
659 return mtk_aes_handle_queue(cryp, !(mode & AES_FLAGS_ENCRYPT),
660 &req->base);
661 }
662
663 static int mtk_aes_ecb_encrypt(struct skcipher_request *req)
664 {
665 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_ECB);
666 }
667
668 static int mtk_aes_ecb_decrypt(struct skcipher_request *req)
669 {
670 return mtk_aes_crypt(req, AES_FLAGS_ECB);
671 }
672
673 static int mtk_aes_cbc_encrypt(struct skcipher_request *req)
674 {
675 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CBC);
676 }
677
678 static int mtk_aes_cbc_decrypt(struct skcipher_request *req)
679 {
680 return mtk_aes_crypt(req, AES_FLAGS_CBC);
681 }
682
683 static int mtk_aes_ctr_encrypt(struct skcipher_request *req)
684 {
685 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CTR);
686 }
687
688 static int mtk_aes_ctr_decrypt(struct skcipher_request *req)
689 {
690 return mtk_aes_crypt(req, AES_FLAGS_CTR);
691 }
692
693 static int mtk_aes_ofb_encrypt(struct skcipher_request *req)
694 {
695 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_OFB);
696 }
697
698 static int mtk_aes_ofb_decrypt(struct skcipher_request *req)
699 {
700 return mtk_aes_crypt(req, AES_FLAGS_OFB);
701 }
702
703 static int mtk_aes_cfb_encrypt(struct skcipher_request *req)
704 {
705 return mtk_aes_crypt(req, AES_FLAGS_ENCRYPT | AES_FLAGS_CFB128);
706 }
707
708 static int mtk_aes_cfb_decrypt(struct skcipher_request *req)
709 {
710 return mtk_aes_crypt(req, AES_FLAGS_CFB128);
711 }
712
713 static int mtk_aes_init_tfm(struct crypto_skcipher *tfm)
714 {
715 struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
716
717 crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx));
718 ctx->base.start = mtk_aes_start;
719 return 0;
720 }
721
722 static int mtk_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
723 {
724 struct mtk_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
725
726 crypto_skcipher_set_reqsize(tfm, sizeof(struct mtk_aes_reqctx));
727 ctx->base.start = mtk_aes_ctr_start;
728 return 0;
729 }
730
731 static struct skcipher_alg aes_algs[] = {
732 {
733 .base.cra_name = "cbc(aes)",
734 .base.cra_driver_name = "cbc-aes-mtk",
735 .base.cra_priority = 400,
736 .base.cra_flags = CRYPTO_ALG_ASYNC,
737 .base.cra_blocksize = AES_BLOCK_SIZE,
738 .base.cra_ctxsize = sizeof(struct mtk_aes_ctx),
739 .base.cra_alignmask = 0xf,
740 .base.cra_module = THIS_MODULE,
741
742 .min_keysize = AES_MIN_KEY_SIZE,
743 .max_keysize = AES_MAX_KEY_SIZE,
744 .setkey = mtk_aes_setkey,
745 .encrypt = mtk_aes_cbc_encrypt,
746 .decrypt = mtk_aes_cbc_decrypt,
747 .ivsize = AES_BLOCK_SIZE,
748 .init = mtk_aes_init_tfm,
749 },
750 {
751 .base.cra_name = "ecb(aes)",
752 .base.cra_driver_name = "ecb-aes-mtk",
753 .base.cra_priority = 400,
754 .base.cra_flags = CRYPTO_ALG_ASYNC,
755 .base.cra_blocksize = AES_BLOCK_SIZE,
756 .base.cra_ctxsize = sizeof(struct mtk_aes_ctx),
757 .base.cra_alignmask = 0xf,
758 .base.cra_module = THIS_MODULE,
759
760 .min_keysize = AES_MIN_KEY_SIZE,
761 .max_keysize = AES_MAX_KEY_SIZE,
762 .setkey = mtk_aes_setkey,
763 .encrypt = mtk_aes_ecb_encrypt,
764 .decrypt = mtk_aes_ecb_decrypt,
765 .init = mtk_aes_init_tfm,
766 },
767 {
768 .base.cra_name = "ctr(aes)",
769 .base.cra_driver_name = "ctr-aes-mtk",
770 .base.cra_priority = 400,
771 .base.cra_flags = CRYPTO_ALG_ASYNC,
772 .base.cra_blocksize = 1,
773 .base.cra_ctxsize = sizeof(struct mtk_aes_ctx),
774 .base.cra_alignmask = 0xf,
775 .base.cra_module = THIS_MODULE,
776
777 .min_keysize = AES_MIN_KEY_SIZE,
778 .max_keysize = AES_MAX_KEY_SIZE,
779 .ivsize = AES_BLOCK_SIZE,
780 .setkey = mtk_aes_setkey,
781 .encrypt = mtk_aes_ctr_encrypt,
782 .decrypt = mtk_aes_ctr_decrypt,
783 .init = mtk_aes_ctr_init_tfm,
784 },
785 {
786 .base.cra_name = "ofb(aes)",
787 .base.cra_driver_name = "ofb-aes-mtk",
788 .base.cra_priority = 400,
789 .base.cra_flags = CRYPTO_ALG_ASYNC,
790 .base.cra_blocksize = AES_BLOCK_SIZE,
791 .base.cra_ctxsize = sizeof(struct mtk_aes_ctx),
792 .base.cra_alignmask = 0xf,
793 .base.cra_module = THIS_MODULE,
794
795 .min_keysize = AES_MIN_KEY_SIZE,
796 .max_keysize = AES_MAX_KEY_SIZE,
797 .ivsize = AES_BLOCK_SIZE,
798 .setkey = mtk_aes_setkey,
799 .encrypt = mtk_aes_ofb_encrypt,
800 .decrypt = mtk_aes_ofb_decrypt,
801 },
802 {
803 .base.cra_name = "cfb(aes)",
804 .base.cra_driver_name = "cfb-aes-mtk",
805 .base.cra_priority = 400,
806 .base.cra_flags = CRYPTO_ALG_ASYNC,
807 .base.cra_blocksize = 1,
808 .base.cra_ctxsize = sizeof(struct mtk_aes_ctx),
809 .base.cra_alignmask = 0xf,
810 .base.cra_module = THIS_MODULE,
811
812 .min_keysize = AES_MIN_KEY_SIZE,
813 .max_keysize = AES_MAX_KEY_SIZE,
814 .ivsize = AES_BLOCK_SIZE,
815 .setkey = mtk_aes_setkey,
816 .encrypt = mtk_aes_cfb_encrypt,
817 .decrypt = mtk_aes_cfb_decrypt,
818 },
819 };
820
821 static inline struct mtk_aes_gcm_ctx *
822 mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx)
823 {
824 return container_of(ctx, struct mtk_aes_gcm_ctx, base);
825 }
826
827 /*
828 * Engine will verify and compare tag automatically, so we just need
829 * to check returned status which stored in the result descriptor.
830 */
831 static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp,
832 struct mtk_aes_rec *aes)
833 {
834 __le32 status = cryp->ring[aes->id]->res_prev->ct;
835
836 return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ?
837 -EBADMSG : 0);
838 }
839
840 /* Initialize transform information of GCM mode */
841 static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp,
842 struct mtk_aes_rec *aes,
843 size_t len)
844 {
845 struct aead_request *req = aead_request_cast(aes->areq);
846 struct mtk_aes_base_ctx *ctx = aes->ctx;
847 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
848 struct mtk_aes_info *info = &ctx->info;
849 u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req));
850 u32 cnt = 0;
851
852 ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len);
853
854 info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen);
855 info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen);
856 info->cmd[cnt++] = AES_GCM_CMD2;
857 info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen);
858
859 if (aes->flags & AES_FLAGS_ENCRYPT) {
860 info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize);
861 info->tfm[0] = AES_TFM_GCM_OUT;
862 } else {
863 info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize);
864 info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize);
865 info->tfm[0] = AES_TFM_GCM_IN;
866 }
867 ctx->ct_size = cnt;
868
869 info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE(
870 ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) |
871 ctx->keymode;
872 info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV |
873 AES_TFM_ENC_HASH;
874
875 memcpy(info->state + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE),
876 req->iv, ivsize);
877 }
878
879 static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes,
880 struct scatterlist *src, struct scatterlist *dst,
881 size_t len)
882 {
883 bool src_aligned, dst_aligned;
884
885 aes->src.sg = src;
886 aes->dst.sg = dst;
887 aes->real_dst = dst;
888
889 src_aligned = mtk_aes_check_aligned(src, len, &aes->src);
890 if (src == dst)
891 dst_aligned = src_aligned;
892 else
893 dst_aligned = mtk_aes_check_aligned(dst, len, &aes->dst);
894
895 if (!src_aligned || !dst_aligned) {
896 if (aes->total > AES_BUF_SIZE)
897 return mtk_aes_complete(cryp, aes, -ENOMEM);
898
899 if (!src_aligned) {
900 sg_copy_to_buffer(src, sg_nents(src), aes->buf, len);
901 aes->src.sg = &aes->aligned_sg;
902 aes->src.nents = 1;
903 aes->src.remainder = 0;
904 }
905
906 if (!dst_aligned) {
907 aes->dst.sg = &aes->aligned_sg;
908 aes->dst.nents = 1;
909 aes->dst.remainder = 0;
910 }
911
912 sg_init_table(&aes->aligned_sg, 1);
913 sg_set_buf(&aes->aligned_sg, aes->buf, aes->total);
914 }
915
916 mtk_aes_gcm_info_init(cryp, aes, len);
917
918 return mtk_aes_map(cryp, aes);
919 }
920
921 /* Todo: GMAC */
922 static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes)
923 {
924 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(aes->ctx);
925 struct aead_request *req = aead_request_cast(aes->areq);
926 struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
927 u32 len = req->assoclen + req->cryptlen;
928
929 mtk_aes_set_mode(aes, rctx);
930
931 if (aes->flags & AES_FLAGS_ENCRYPT) {
932 u32 tag[4];
933
934 aes->resume = mtk_aes_transfer_complete;
935 /* Compute total process length. */
936 aes->total = len + gctx->authsize;
937 /* Hardware will append authenticated tag to output buffer */
938 scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1);
939 } else {
940 aes->resume = mtk_aes_gcm_tag_verify;
941 aes->total = len;
942 }
943
944 return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len);
945 }
946
947 static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode)
948 {
949 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
950 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
951 struct mtk_aes_reqctx *rctx = aead_request_ctx(req);
952 struct mtk_cryp *cryp;
953 bool enc = !!(mode & AES_FLAGS_ENCRYPT);
954
955 cryp = mtk_aes_find_dev(ctx);
956 if (!cryp)
957 return -ENODEV;
958
959 /* Compute text length. */
960 gctx->textlen = req->cryptlen - (enc ? 0 : gctx->authsize);
961
962 /* Empty messages are not supported yet */
963 if (!gctx->textlen && !req->assoclen)
964 return -EINVAL;
965
966 rctx->mode = AES_FLAGS_GCM | mode;
967
968 return mtk_aes_handle_queue(cryp, enc, &req->base);
969 }
970
971 /*
972 * Because of the hardware limitation, we need to pre-calculate key(H)
973 * for the GHASH operation. The result of the encryption operation
974 * need to be stored in the transform state buffer.
975 */
976 static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
977 u32 keylen)
978 {
979 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
980 union {
981 u32 x32[SIZE_IN_WORDS(AES_BLOCK_SIZE)];
982 u8 x8[AES_BLOCK_SIZE];
983 } hash = {};
984 struct crypto_aes_ctx aes_ctx;
985 int err;
986 int i;
987
988 switch (keylen) {
989 case AES_KEYSIZE_128:
990 ctx->keymode = AES_TFM_128BITS;
991 break;
992 case AES_KEYSIZE_192:
993 ctx->keymode = AES_TFM_192BITS;
994 break;
995 case AES_KEYSIZE_256:
996 ctx->keymode = AES_TFM_256BITS;
997 break;
998
999 default:
1000 return -EINVAL;
1001 }
1002
1003 ctx->keylen = SIZE_IN_WORDS(keylen);
1004
1005 err = aes_expandkey(&aes_ctx, key, keylen);
1006 if (err)
1007 return err;
1008
1009 aes_encrypt(&aes_ctx, hash.x8, hash.x8);
1010 memzero_explicit(&aes_ctx, sizeof(aes_ctx));
1011
1012 memcpy(ctx->key, key, keylen);
1013
1014 /* Why do we need to do this? */
1015 for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++)
1016 hash.x32[i] = swab32(hash.x32[i]);
1017
1018 memcpy(ctx->key + ctx->keylen, &hash, AES_BLOCK_SIZE);
1019
1020 return 0;
1021 }
1022
1023 static int mtk_aes_gcm_setauthsize(struct crypto_aead *aead,
1024 u32 authsize)
1025 {
1026 struct mtk_aes_base_ctx *ctx = crypto_aead_ctx(aead);
1027 struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx);
1028
1029 /* Same as crypto_gcm_authsize() from crypto/gcm.c */
1030 switch (authsize) {
1031 case 8:
1032 case 12:
1033 case 16:
1034 break;
1035 default:
1036 return -EINVAL;
1037 }
1038
1039 gctx->authsize = authsize;
1040 return 0;
1041 }
1042
1043 static int mtk_aes_gcm_encrypt(struct aead_request *req)
1044 {
1045 return mtk_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
1046 }
1047
1048 static int mtk_aes_gcm_decrypt(struct aead_request *req)
1049 {
1050 return mtk_aes_gcm_crypt(req, 0);
1051 }
1052
1053 static int mtk_aes_gcm_init(struct crypto_aead *aead)
1054 {
1055 struct mtk_aes_gcm_ctx *ctx = crypto_aead_ctx(aead);
1056
1057 crypto_aead_set_reqsize(aead, sizeof(struct mtk_aes_reqctx));
1058 ctx->base.start = mtk_aes_gcm_start;
1059 return 0;
1060 }
1061
1062 static struct aead_alg aes_gcm_alg = {
1063 .setkey = mtk_aes_gcm_setkey,
1064 .setauthsize = mtk_aes_gcm_setauthsize,
1065 .encrypt = mtk_aes_gcm_encrypt,
1066 .decrypt = mtk_aes_gcm_decrypt,
1067 .init = mtk_aes_gcm_init,
1068 .ivsize = GCM_AES_IV_SIZE,
1069 .maxauthsize = AES_BLOCK_SIZE,
1070
1071 .base = {
1072 .cra_name = "gcm(aes)",
1073 .cra_driver_name = "gcm-aes-mtk",
1074 .cra_priority = 400,
1075 .cra_flags = CRYPTO_ALG_ASYNC,
1076 .cra_blocksize = 1,
1077 .cra_ctxsize = sizeof(struct mtk_aes_gcm_ctx),
1078 .cra_alignmask = 0xf,
1079 .cra_module = THIS_MODULE,
1080 },
1081 };
1082
1083 static void mtk_aes_queue_task(unsigned long data)
1084 {
1085 struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;
1086
1087 mtk_aes_handle_queue(aes->cryp, aes->id, NULL);
1088 }
1089
1090 static void mtk_aes_done_task(unsigned long data)
1091 {
1092 struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data;
1093 struct mtk_cryp *cryp = aes->cryp;
1094
1095 mtk_aes_unmap(cryp, aes);
1096 aes->resume(cryp, aes);
1097 }
1098
1099 static irqreturn_t mtk_aes_irq(int irq, void *dev_id)
1100 {
1101 struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id;
1102 struct mtk_cryp *cryp = aes->cryp;
1103 u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id));
1104
1105 mtk_aes_write(cryp, RDR_STAT(aes->id), val);
1106
1107 if (likely(AES_FLAGS_BUSY & aes->flags)) {
1108 mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST);
1109 mtk_aes_write(cryp, RDR_THRESH(aes->id),
1110 MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE);
1111
1112 tasklet_schedule(&aes->done_task);
1113 } else {
1114 dev_warn(cryp->dev, "AES interrupt when no active requests.\n");
1115 }
1116 return IRQ_HANDLED;
1117 }
1118
1119 /*
1120 * The purpose of creating encryption and decryption records is
1121 * to process outbound/inbound data in parallel, it can improve
1122 * performance in most use cases, such as IPSec VPN, especially
1123 * under heavy network traffic.
1124 */
1125 static int mtk_aes_record_init(struct mtk_cryp *cryp)
1126 {
1127 struct mtk_aes_rec **aes = cryp->aes;
1128 int i, err = -ENOMEM;
1129
1130 for (i = 0; i < MTK_REC_NUM; i++) {
1131 aes[i] = kzalloc(sizeof(**aes), GFP_KERNEL);
1132 if (!aes[i])
1133 goto err_cleanup;
1134
1135 aes[i]->buf = (void *)__get_free_pages(GFP_KERNEL,
1136 AES_BUF_ORDER);
1137 if (!aes[i]->buf)
1138 goto err_cleanup;
1139
1140 aes[i]->cryp = cryp;
1141
1142 spin_lock_init(&aes[i]->lock);
1143 crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE);
1144
1145 tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task,
1146 (unsigned long)aes[i]);
1147 tasklet_init(&aes[i]->done_task, mtk_aes_done_task,
1148 (unsigned long)aes[i]);
1149 }
1150
1151 /* Link to ring0 and ring1 respectively */
1152 aes[0]->id = MTK_RING0;
1153 aes[1]->id = MTK_RING1;
1154
1155 return 0;
1156
1157 err_cleanup:
1158 for (; i--; ) {
1159 free_page((unsigned long)aes[i]->buf);
1160 kfree(aes[i]);
1161 }
1162
1163 return err;
1164 }
1165
1166 static void mtk_aes_record_free(struct mtk_cryp *cryp)
1167 {
1168 int i;
1169
1170 for (i = 0; i < MTK_REC_NUM; i++) {
1171 tasklet_kill(&cryp->aes[i]->done_task);
1172 tasklet_kill(&cryp->aes[i]->queue_task);
1173
1174 free_page((unsigned long)cryp->aes[i]->buf);
1175 kfree(cryp->aes[i]);
1176 }
1177 }
1178
1179 static void mtk_aes_unregister_algs(void)
1180 {
1181 int i;
1182
1183 crypto_unregister_aead(&aes_gcm_alg);
1184
1185 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1186 crypto_unregister_skcipher(&aes_algs[i]);
1187 }
1188
1189 static int mtk_aes_register_algs(void)
1190 {
1191 int err, i;
1192
1193 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1194 err = crypto_register_skcipher(&aes_algs[i]);
1195 if (err)
1196 goto err_aes_algs;
1197 }
1198
1199 err = crypto_register_aead(&aes_gcm_alg);
1200 if (err)
1201 goto err_aes_algs;
1202
1203 return 0;
1204
1205 err_aes_algs:
1206 for (; i--; )
1207 crypto_unregister_skcipher(&aes_algs[i]);
1208
1209 return err;
1210 }
1211
1212 int mtk_cipher_alg_register(struct mtk_cryp *cryp)
1213 {
1214 int ret;
1215
1216 INIT_LIST_HEAD(&cryp->aes_list);
1217
1218 /* Initialize two cipher records */
1219 ret = mtk_aes_record_init(cryp);
1220 if (ret)
1221 goto err_record;
1222
1223 ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq,
1224 0, "mtk-aes", cryp->aes[0]);
1225 if (ret) {
1226 dev_err(cryp->dev, "unable to request AES irq.\n");
1227 goto err_res;
1228 }
1229
1230 ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq,
1231 0, "mtk-aes", cryp->aes[1]);
1232 if (ret) {
1233 dev_err(cryp->dev, "unable to request AES irq.\n");
1234 goto err_res;
1235 }
1236
1237 /* Enable ring0 and ring1 interrupt */
1238 mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0);
1239 mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1);
1240
1241 spin_lock(&mtk_aes.lock);
1242 list_add_tail(&cryp->aes_list, &mtk_aes.dev_list);
1243 spin_unlock(&mtk_aes.lock);
1244
1245 ret = mtk_aes_register_algs();
1246 if (ret)
1247 goto err_algs;
1248
1249 return 0;
1250
1251 err_algs:
1252 spin_lock(&mtk_aes.lock);
1253 list_del(&cryp->aes_list);
1254 spin_unlock(&mtk_aes.lock);
1255 err_res:
1256 mtk_aes_record_free(cryp);
1257 err_record:
1258
1259 dev_err(cryp->dev, "mtk-aes initialization failed.\n");
1260 return ret;
1261 }
1262
1263 void mtk_cipher_alg_release(struct mtk_cryp *cryp)
1264 {
1265 spin_lock(&mtk_aes.lock);
1266 list_del(&cryp->aes_list);
1267 spin_unlock(&mtk_aes.lock);
1268
1269 mtk_aes_unregister_algs();
1270 mtk_aes_record_free(cryp);
1271 }
Linux with added FPC-III support