4 * Support for Samsung S5PV210 HW acceleration.
6 * Copyright (C) 2011 NetUP Inc. All rights reserved.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as published
10 * by the Free Software Foundation.
14 #include <linux/delay.h>
15 #include <linux/err.h>
16 #include <linux/module.h>
17 #include <linux/init.h>
18 #include <linux/errno.h>
19 #include <linux/kernel.h>
20 #include <linux/clk.h>
21 #include <linux/platform_device.h>
22 #include <linux/scatterlist.h>
23 #include <linux/dma-mapping.h>
25 #include <linux/crypto.h>
26 #include <linux/interrupt.h>
28 #include <crypto/algapi.h>
29 #include <crypto/aes.h>
30 #include <crypto/ctr.h>
35 #define _SBF(s, v) ((v) << (s))
36 #define _BIT(b) _SBF(b, 1)
38 /* Feed control registers */
39 #define SSS_REG_FCINTSTAT 0x0000
40 #define SSS_FCINTSTAT_BRDMAINT _BIT(3)
41 #define SSS_FCINTSTAT_BTDMAINT _BIT(2)
42 #define SSS_FCINTSTAT_HRDMAINT _BIT(1)
43 #define SSS_FCINTSTAT_PKDMAINT _BIT(0)
45 #define SSS_REG_FCINTENSET 0x0004
46 #define SSS_FCINTENSET_BRDMAINTENSET _BIT(3)
47 #define SSS_FCINTENSET_BTDMAINTENSET _BIT(2)
48 #define SSS_FCINTENSET_HRDMAINTENSET _BIT(1)
49 #define SSS_FCINTENSET_PKDMAINTENSET _BIT(0)
51 #define SSS_REG_FCINTENCLR 0x0008
52 #define SSS_FCINTENCLR_BRDMAINTENCLR _BIT(3)
53 #define SSS_FCINTENCLR_BTDMAINTENCLR _BIT(2)
54 #define SSS_FCINTENCLR_HRDMAINTENCLR _BIT(1)
55 #define SSS_FCINTENCLR_PKDMAINTENCLR _BIT(0)
57 #define SSS_REG_FCINTPEND 0x000C
58 #define SSS_FCINTPEND_BRDMAINTP _BIT(3)
59 #define SSS_FCINTPEND_BTDMAINTP _BIT(2)
60 #define SSS_FCINTPEND_HRDMAINTP _BIT(1)
61 #define SSS_FCINTPEND_PKDMAINTP _BIT(0)
63 #define SSS_REG_FCFIFOSTAT 0x0010
64 #define SSS_FCFIFOSTAT_BRFIFOFUL _BIT(7)
65 #define SSS_FCFIFOSTAT_BRFIFOEMP _BIT(6)
66 #define SSS_FCFIFOSTAT_BTFIFOFUL _BIT(5)
67 #define SSS_FCFIFOSTAT_BTFIFOEMP _BIT(4)
68 #define SSS_FCFIFOSTAT_HRFIFOFUL _BIT(3)
69 #define SSS_FCFIFOSTAT_HRFIFOEMP _BIT(2)
70 #define SSS_FCFIFOSTAT_PKFIFOFUL _BIT(1)
71 #define SSS_FCFIFOSTAT_PKFIFOEMP _BIT(0)
73 #define SSS_REG_FCFIFOCTRL 0x0014
74 #define SSS_FCFIFOCTRL_DESSEL _BIT(2)
75 #define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00)
76 #define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01)
77 #define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02)
79 #define SSS_REG_FCBRDMAS 0x0020
80 #define SSS_REG_FCBRDMAL 0x0024
81 #define SSS_REG_FCBRDMAC 0x0028
82 #define SSS_FCBRDMAC_BYTESWAP _BIT(1)
83 #define SSS_FCBRDMAC_FLUSH _BIT(0)
85 #define SSS_REG_FCBTDMAS 0x0030
86 #define SSS_REG_FCBTDMAL 0x0034
87 #define SSS_REG_FCBTDMAC 0x0038
88 #define SSS_FCBTDMAC_BYTESWAP _BIT(1)
89 #define SSS_FCBTDMAC_FLUSH _BIT(0)
91 #define SSS_REG_FCHRDMAS 0x0040
92 #define SSS_REG_FCHRDMAL 0x0044
93 #define SSS_REG_FCHRDMAC 0x0048
94 #define SSS_FCHRDMAC_BYTESWAP _BIT(1)
95 #define SSS_FCHRDMAC_FLUSH _BIT(0)
97 #define SSS_REG_FCPKDMAS 0x0050
98 #define SSS_REG_FCPKDMAL 0x0054
99 #define SSS_REG_FCPKDMAC 0x0058
100 #define SSS_FCPKDMAC_BYTESWAP _BIT(3)
101 #define SSS_FCPKDMAC_DESCEND _BIT(2)
102 #define SSS_FCPKDMAC_TRANSMIT _BIT(1)
103 #define SSS_FCPKDMAC_FLUSH _BIT(0)
105 #define SSS_REG_FCPKDMAO 0x005C
108 #define SSS_REG_AES_CONTROL 0x4000
109 #define SSS_AES_BYTESWAP_DI _BIT(11)
110 #define SSS_AES_BYTESWAP_DO _BIT(10)
111 #define SSS_AES_BYTESWAP_IV _BIT(9)
112 #define SSS_AES_BYTESWAP_CNT _BIT(8)
113 #define SSS_AES_BYTESWAP_KEY _BIT(7)
114 #define SSS_AES_KEY_CHANGE_MODE _BIT(6)
115 #define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00)
116 #define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01)
117 #define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02)
118 #define SSS_AES_FIFO_MODE _BIT(3)
119 #define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00)
120 #define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01)
121 #define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02)
122 #define SSS_AES_MODE_DECRYPT _BIT(0)
124 #define SSS_REG_AES_STATUS 0x4004
125 #define SSS_AES_BUSY _BIT(2)
126 #define SSS_AES_INPUT_READY _BIT(1)
127 #define SSS_AES_OUTPUT_READY _BIT(0)
129 #define SSS_REG_AES_IN_DATA(s) (0x4010 + (s << 2))
130 #define SSS_REG_AES_OUT_DATA(s) (0x4020 + (s << 2))
131 #define SSS_REG_AES_IV_DATA(s) (0x4030 + (s << 2))
132 #define SSS_REG_AES_CNT_DATA(s) (0x4040 + (s << 2))
133 #define SSS_REG_AES_KEY_DATA(s) (0x4080 + (s << 2))
135 #define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg))
136 #define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg))
137 #define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg))
139 /* HW engine modes */
140 #define FLAGS_AES_DECRYPT _BIT(0)
141 #define FLAGS_AES_MODE_MASK _SBF(1, 0x03)
142 #define FLAGS_AES_CBC _SBF(1, 0x01)
143 #define FLAGS_AES_CTR _SBF(1, 0x02)
145 #define AES_KEY_LEN 16
146 #define CRYPTO_QUEUE_LEN 1
148 struct s5p_aes_reqctx
{
153 struct s5p_aes_dev
*dev
;
155 uint8_t aes_key
[AES_MAX_KEY_SIZE
];
156 uint8_t nonce
[CTR_RFC3686_NONCE_SIZE
];
163 void __iomem
*ioaddr
;
167 struct ablkcipher_request
*req
;
168 struct s5p_aes_ctx
*ctx
;
169 struct scatterlist
*sg_src
;
170 struct scatterlist
*sg_dst
;
172 struct tasklet_struct tasklet
;
173 struct crypto_queue queue
;
178 static struct s5p_aes_dev
*s5p_dev
;
180 static void s5p_set_dma_indata(struct s5p_aes_dev
*dev
, struct scatterlist
*sg
)
182 SSS_WRITE(dev
, FCBRDMAS
, sg_dma_address(sg
));
183 SSS_WRITE(dev
, FCBRDMAL
, sg_dma_len(sg
));
186 static void s5p_set_dma_outdata(struct s5p_aes_dev
*dev
, struct scatterlist
*sg
)
188 SSS_WRITE(dev
, FCBTDMAS
, sg_dma_address(sg
));
189 SSS_WRITE(dev
, FCBTDMAL
, sg_dma_len(sg
));
192 static void s5p_aes_complete(struct s5p_aes_dev
*dev
, int err
)
194 /* holding a lock outside */
195 dev
->req
->base
.complete(&dev
->req
->base
, err
);
199 static void s5p_unset_outdata(struct s5p_aes_dev
*dev
)
201 dma_unmap_sg(dev
->dev
, dev
->sg_dst
, 1, DMA_FROM_DEVICE
);
204 static void s5p_unset_indata(struct s5p_aes_dev
*dev
)
206 dma_unmap_sg(dev
->dev
, dev
->sg_src
, 1, DMA_TO_DEVICE
);
209 static int s5p_set_outdata(struct s5p_aes_dev
*dev
, struct scatterlist
*sg
)
213 if (!IS_ALIGNED(sg_dma_len(sg
), AES_BLOCK_SIZE
)) {
217 if (!sg_dma_len(sg
)) {
222 err
= dma_map_sg(dev
->dev
, sg
, 1, DMA_FROM_DEVICE
);
235 static int s5p_set_indata(struct s5p_aes_dev
*dev
, struct scatterlist
*sg
)
239 if (!IS_ALIGNED(sg_dma_len(sg
), AES_BLOCK_SIZE
)) {
243 if (!sg_dma_len(sg
)) {
248 err
= dma_map_sg(dev
->dev
, sg
, 1, DMA_TO_DEVICE
);
261 static void s5p_aes_tx(struct s5p_aes_dev
*dev
)
265 s5p_unset_outdata(dev
);
267 if (!sg_is_last(dev
->sg_dst
)) {
268 err
= s5p_set_outdata(dev
, sg_next(dev
->sg_dst
));
270 s5p_aes_complete(dev
, err
);
274 s5p_set_dma_outdata(dev
, dev
->sg_dst
);
276 s5p_aes_complete(dev
, err
);
279 static void s5p_aes_rx(struct s5p_aes_dev
*dev
)
283 s5p_unset_indata(dev
);
285 if (!sg_is_last(dev
->sg_src
)) {
286 err
= s5p_set_indata(dev
, sg_next(dev
->sg_src
));
288 s5p_aes_complete(dev
, err
);
292 s5p_set_dma_indata(dev
, dev
->sg_src
);
296 static irqreturn_t
s5p_aes_interrupt(int irq
, void *dev_id
)
298 struct platform_device
*pdev
= dev_id
;
299 struct s5p_aes_dev
*dev
= platform_get_drvdata(pdev
);
303 spin_lock_irqsave(&dev
->lock
, flags
);
305 if (irq
== dev
->irq_fc
) {
306 status
= SSS_READ(dev
, FCINTSTAT
);
307 if (status
& SSS_FCINTSTAT_BRDMAINT
)
309 if (status
& SSS_FCINTSTAT_BTDMAINT
)
312 SSS_WRITE(dev
, FCINTPEND
, status
);
315 spin_unlock_irqrestore(&dev
->lock
, flags
);
320 static void s5p_set_aes(struct s5p_aes_dev
*dev
,
321 uint8_t *key
, uint8_t *iv
, unsigned int keylen
)
323 void __iomem
*keystart
;
325 memcpy(dev
->ioaddr
+ SSS_REG_AES_IV_DATA(0), iv
, 0x10);
327 if (keylen
== AES_KEYSIZE_256
)
328 keystart
= dev
->ioaddr
+ SSS_REG_AES_KEY_DATA(0);
329 else if (keylen
== AES_KEYSIZE_192
)
330 keystart
= dev
->ioaddr
+ SSS_REG_AES_KEY_DATA(2);
332 keystart
= dev
->ioaddr
+ SSS_REG_AES_KEY_DATA(4);
334 memcpy(keystart
, key
, keylen
);
337 static void s5p_aes_crypt_start(struct s5p_aes_dev
*dev
, unsigned long mode
)
339 struct ablkcipher_request
*req
= dev
->req
;
341 uint32_t aes_control
;
345 aes_control
= SSS_AES_KEY_CHANGE_MODE
;
346 if (mode
& FLAGS_AES_DECRYPT
)
347 aes_control
|= SSS_AES_MODE_DECRYPT
;
349 if ((mode
& FLAGS_AES_MODE_MASK
) == FLAGS_AES_CBC
)
350 aes_control
|= SSS_AES_CHAIN_MODE_CBC
;
351 else if ((mode
& FLAGS_AES_MODE_MASK
) == FLAGS_AES_CTR
)
352 aes_control
|= SSS_AES_CHAIN_MODE_CTR
;
354 if (dev
->ctx
->keylen
== AES_KEYSIZE_192
)
355 aes_control
|= SSS_AES_KEY_SIZE_192
;
356 else if (dev
->ctx
->keylen
== AES_KEYSIZE_256
)
357 aes_control
|= SSS_AES_KEY_SIZE_256
;
359 aes_control
|= SSS_AES_FIFO_MODE
;
361 /* as a variant it is possible to use byte swapping on DMA side */
362 aes_control
|= SSS_AES_BYTESWAP_DI
363 | SSS_AES_BYTESWAP_DO
364 | SSS_AES_BYTESWAP_IV
365 | SSS_AES_BYTESWAP_KEY
366 | SSS_AES_BYTESWAP_CNT
;
368 spin_lock_irqsave(&dev
->lock
, flags
);
370 SSS_WRITE(dev
, FCINTENCLR
,
371 SSS_FCINTENCLR_BTDMAINTENCLR
| SSS_FCINTENCLR_BRDMAINTENCLR
);
372 SSS_WRITE(dev
, FCFIFOCTRL
, 0x00);
374 err
= s5p_set_indata(dev
, req
->src
);
378 err
= s5p_set_outdata(dev
, req
->dst
);
382 SSS_WRITE(dev
, AES_CONTROL
, aes_control
);
383 s5p_set_aes(dev
, dev
->ctx
->aes_key
, req
->info
, dev
->ctx
->keylen
);
385 s5p_set_dma_indata(dev
, req
->src
);
386 s5p_set_dma_outdata(dev
, req
->dst
);
388 SSS_WRITE(dev
, FCINTENSET
,
389 SSS_FCINTENSET_BTDMAINTENSET
| SSS_FCINTENSET_BRDMAINTENSET
);
391 spin_unlock_irqrestore(&dev
->lock
, flags
);
396 s5p_unset_indata(dev
);
399 s5p_aes_complete(dev
, err
);
400 spin_unlock_irqrestore(&dev
->lock
, flags
);
403 static void s5p_tasklet_cb(unsigned long data
)
405 struct s5p_aes_dev
*dev
= (struct s5p_aes_dev
*)data
;
406 struct crypto_async_request
*async_req
, *backlog
;
407 struct s5p_aes_reqctx
*reqctx
;
410 spin_lock_irqsave(&dev
->lock
, flags
);
411 backlog
= crypto_get_backlog(&dev
->queue
);
412 async_req
= crypto_dequeue_request(&dev
->queue
);
413 spin_unlock_irqrestore(&dev
->lock
, flags
);
419 backlog
->complete(backlog
, -EINPROGRESS
);
421 dev
->req
= ablkcipher_request_cast(async_req
);
422 dev
->ctx
= crypto_tfm_ctx(dev
->req
->base
.tfm
);
423 reqctx
= ablkcipher_request_ctx(dev
->req
);
425 s5p_aes_crypt_start(dev
, reqctx
->mode
);
428 static int s5p_aes_handle_req(struct s5p_aes_dev
*dev
,
429 struct ablkcipher_request
*req
)
434 spin_lock_irqsave(&dev
->lock
, flags
);
437 spin_unlock_irqrestore(&dev
->lock
, flags
);
442 err
= ablkcipher_enqueue_request(&dev
->queue
, req
);
443 spin_unlock_irqrestore(&dev
->lock
, flags
);
445 tasklet_schedule(&dev
->tasklet
);
451 static int s5p_aes_crypt(struct ablkcipher_request
*req
, unsigned long mode
)
453 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(req
);
454 struct s5p_aes_ctx
*ctx
= crypto_ablkcipher_ctx(tfm
);
455 struct s5p_aes_reqctx
*reqctx
= ablkcipher_request_ctx(req
);
456 struct s5p_aes_dev
*dev
= ctx
->dev
;
458 if (!IS_ALIGNED(req
->nbytes
, AES_BLOCK_SIZE
)) {
459 pr_err("request size is not exact amount of AES blocks\n");
465 return s5p_aes_handle_req(dev
, req
);
468 static int s5p_aes_setkey(struct crypto_ablkcipher
*cipher
,
469 const uint8_t *key
, unsigned int keylen
)
471 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
472 struct s5p_aes_ctx
*ctx
= crypto_tfm_ctx(tfm
);
474 if (keylen
!= AES_KEYSIZE_128
&&
475 keylen
!= AES_KEYSIZE_192
&&
476 keylen
!= AES_KEYSIZE_256
)
479 memcpy(ctx
->aes_key
, key
, keylen
);
480 ctx
->keylen
= keylen
;
485 static int s5p_aes_ecb_encrypt(struct ablkcipher_request
*req
)
487 return s5p_aes_crypt(req
, 0);
490 static int s5p_aes_ecb_decrypt(struct ablkcipher_request
*req
)
492 return s5p_aes_crypt(req
, FLAGS_AES_DECRYPT
);
495 static int s5p_aes_cbc_encrypt(struct ablkcipher_request
*req
)
497 return s5p_aes_crypt(req
, FLAGS_AES_CBC
);
500 static int s5p_aes_cbc_decrypt(struct ablkcipher_request
*req
)
502 return s5p_aes_crypt(req
, FLAGS_AES_DECRYPT
| FLAGS_AES_CBC
);
505 static int s5p_aes_cra_init(struct crypto_tfm
*tfm
)
507 struct s5p_aes_ctx
*ctx
= crypto_tfm_ctx(tfm
);
510 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct s5p_aes_reqctx
);
515 static struct crypto_alg algs
[] = {
517 .cra_name
= "ecb(aes)",
518 .cra_driver_name
= "ecb-aes-s5p",
520 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
522 CRYPTO_ALG_KERN_DRIVER_ONLY
,
523 .cra_blocksize
= AES_BLOCK_SIZE
,
524 .cra_ctxsize
= sizeof(struct s5p_aes_ctx
),
525 .cra_alignmask
= 0x0f,
526 .cra_type
= &crypto_ablkcipher_type
,
527 .cra_module
= THIS_MODULE
,
528 .cra_init
= s5p_aes_cra_init
,
529 .cra_u
.ablkcipher
= {
530 .min_keysize
= AES_MIN_KEY_SIZE
,
531 .max_keysize
= AES_MAX_KEY_SIZE
,
532 .setkey
= s5p_aes_setkey
,
533 .encrypt
= s5p_aes_ecb_encrypt
,
534 .decrypt
= s5p_aes_ecb_decrypt
,
538 .cra_name
= "cbc(aes)",
539 .cra_driver_name
= "cbc-aes-s5p",
541 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
543 CRYPTO_ALG_KERN_DRIVER_ONLY
,
544 .cra_blocksize
= AES_BLOCK_SIZE
,
545 .cra_ctxsize
= sizeof(struct s5p_aes_ctx
),
546 .cra_alignmask
= 0x0f,
547 .cra_type
= &crypto_ablkcipher_type
,
548 .cra_module
= THIS_MODULE
,
549 .cra_init
= s5p_aes_cra_init
,
550 .cra_u
.ablkcipher
= {
551 .min_keysize
= AES_MIN_KEY_SIZE
,
552 .max_keysize
= AES_MAX_KEY_SIZE
,
553 .ivsize
= AES_BLOCK_SIZE
,
554 .setkey
= s5p_aes_setkey
,
555 .encrypt
= s5p_aes_cbc_encrypt
,
556 .decrypt
= s5p_aes_cbc_decrypt
,
561 static int s5p_aes_probe(struct platform_device
*pdev
)
563 int i
, j
, err
= -ENODEV
;
564 struct s5p_aes_dev
*pdata
;
565 struct device
*dev
= &pdev
->dev
;
566 struct resource
*res
;
571 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
575 pdata
= devm_kzalloc(dev
, sizeof(*pdata
), GFP_KERNEL
);
579 if (!devm_request_mem_region(dev
, res
->start
,
580 resource_size(res
), pdev
->name
))
583 pdata
->clk
= clk_get(dev
, "secss");
584 if (IS_ERR(pdata
->clk
)) {
585 dev_err(dev
, "failed to find secss clock source\n");
589 clk_enable(pdata
->clk
);
591 spin_lock_init(&pdata
->lock
);
592 pdata
->ioaddr
= devm_ioremap(dev
, res
->start
,
595 pdata
->irq_hash
= platform_get_irq_byname(pdev
, "hash");
596 if (pdata
->irq_hash
< 0) {
597 err
= pdata
->irq_hash
;
598 dev_warn(dev
, "hash interrupt is not available.\n");
601 err
= devm_request_irq(dev
, pdata
->irq_hash
, s5p_aes_interrupt
,
602 IRQF_SHARED
, pdev
->name
, pdev
);
604 dev_warn(dev
, "hash interrupt is not available.\n");
608 pdata
->irq_fc
= platform_get_irq_byname(pdev
, "feed control");
609 if (pdata
->irq_fc
< 0) {
611 dev_warn(dev
, "feed control interrupt is not available.\n");
614 err
= devm_request_irq(dev
, pdata
->irq_fc
, s5p_aes_interrupt
,
615 IRQF_SHARED
, pdev
->name
, pdev
);
617 dev_warn(dev
, "feed control interrupt is not available.\n");
622 platform_set_drvdata(pdev
, pdata
);
625 tasklet_init(&pdata
->tasklet
, s5p_tasklet_cb
, (unsigned long)pdata
);
626 crypto_init_queue(&pdata
->queue
, CRYPTO_QUEUE_LEN
);
628 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++) {
629 err
= crypto_register_alg(&algs
[i
]);
634 pr_info("s5p-sss driver registered\n");
639 dev_err(dev
, "can't register '%s': %d\n", algs
[i
].cra_name
, err
);
641 for (j
= 0; j
< i
; j
++)
642 crypto_unregister_alg(&algs
[j
]);
644 tasklet_kill(&pdata
->tasklet
);
647 clk_disable(pdata
->clk
);
651 platform_set_drvdata(pdev
, NULL
);
656 static int s5p_aes_remove(struct platform_device
*pdev
)
658 struct s5p_aes_dev
*pdata
= platform_get_drvdata(pdev
);
664 for (i
= 0; i
< ARRAY_SIZE(algs
); i
++)
665 crypto_unregister_alg(&algs
[i
]);
667 tasklet_kill(&pdata
->tasklet
);
669 clk_disable(pdata
->clk
);
673 platform_set_drvdata(pdev
, NULL
);
678 static struct platform_driver s5p_aes_crypto
= {
679 .probe
= s5p_aes_probe
,
680 .remove
= s5p_aes_remove
,
682 .owner
= THIS_MODULE
,
687 module_platform_driver(s5p_aes_crypto
);
689 MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");
690 MODULE_LICENSE("GPL v2");
691 MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");