2 * sun4i-ss-cipher.c - hardware cryptographic accelerator for Allwinner A20 SoC
4 * Copyright (C) 2013-2015 Corentin LABBE <clabbe.montjoie@gmail.com>
6 * This file add support for AES cipher with 128,192,256 bits
7 * keysize in CBC and ECB mode.
8 * Add support also for DES and 3DES in CBC and ECB mode.
10 * You could find the datasheet in Documentation/arm/sunxi/README
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License as published by
14 * the Free Software Foundation; either version 2 of the License, or
15 * (at your option) any later version.
19 static int sun4i_ss_opti_poll(struct ablkcipher_request
*areq
)
21 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
22 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
23 struct sun4i_ss_ctx
*ss
= op
->ss
;
24 unsigned int ivsize
= crypto_ablkcipher_ivsize(tfm
);
25 struct sun4i_cipher_req_ctx
*ctx
= ablkcipher_request_ctx(areq
);
27 /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
28 u32 rx_cnt
= SS_RX_DEFAULT
;
33 unsigned int ileft
= areq
->nbytes
;
34 unsigned int oleft
= areq
->nbytes
;
36 struct sg_mapping_iter mi
, mo
;
37 unsigned int oi
, oo
; /* offset for in and out */
39 if (areq
->nbytes
== 0)
43 dev_err_ratelimited(ss
->dev
, "ERROR: Empty IV\n");
47 if (!areq
->src
|| !areq
->dst
) {
48 dev_err_ratelimited(ss
->dev
, "ERROR: Some SGs are NULL\n");
52 spin_lock_bh(&ss
->slock
);
54 for (i
= 0; i
< op
->keylen
; i
+= 4)
55 writel(*(op
->key
+ i
/ 4), ss
->base
+ SS_KEY0
+ i
);
58 for (i
= 0; i
< 4 && i
< ivsize
/ 4; i
++) {
59 v
= *(u32
*)(areq
->info
+ i
* 4);
60 writel(v
, ss
->base
+ SS_IV0
+ i
* 4);
63 writel(mode
, ss
->base
+ SS_CTL
);
65 sg_miter_start(&mi
, areq
->src
, sg_nents(areq
->src
),
66 SG_MITER_FROM_SG
| SG_MITER_ATOMIC
);
67 sg_miter_start(&mo
, areq
->dst
, sg_nents(areq
->dst
),
68 SG_MITER_TO_SG
| SG_MITER_ATOMIC
);
71 if (!mi
.addr
|| !mo
.addr
) {
72 dev_err_ratelimited(ss
->dev
, "ERROR: sg_miter return null\n");
77 ileft
= areq
->nbytes
/ 4;
78 oleft
= areq
->nbytes
/ 4;
82 todo
= min3(rx_cnt
, ileft
, (mi
.length
- oi
) / 4);
85 writesl(ss
->base
+ SS_RXFIFO
, mi
.addr
+ oi
, todo
);
88 if (oi
== mi
.length
) {
93 spaces
= readl(ss
->base
+ SS_FCSR
);
94 rx_cnt
= SS_RXFIFO_SPACES(spaces
);
95 tx_cnt
= SS_TXFIFO_SPACES(spaces
);
97 todo
= min3(tx_cnt
, oleft
, (mo
.length
- oo
) / 4);
100 readsl(ss
->base
+ SS_TXFIFO
, mo
.addr
+ oo
, todo
);
103 if (oo
== mo
.length
) {
110 for (i
= 0; i
< 4 && i
< ivsize
/ 4; i
++) {
111 v
= readl(ss
->base
+ SS_IV0
+ i
* 4);
112 *(u32
*)(areq
->info
+ i
* 4) = v
;
119 writel(0, ss
->base
+ SS_CTL
);
120 spin_unlock_bh(&ss
->slock
);
124 /* Generic function that support SG with size not multiple of 4 */
125 static int sun4i_ss_cipher_poll(struct ablkcipher_request
*areq
)
127 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
128 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
129 struct sun4i_ss_ctx
*ss
= op
->ss
;
131 struct scatterlist
*in_sg
= areq
->src
;
132 struct scatterlist
*out_sg
= areq
->dst
;
133 unsigned int ivsize
= crypto_ablkcipher_ivsize(tfm
);
134 struct sun4i_cipher_req_ctx
*ctx
= ablkcipher_request_ctx(areq
);
135 u32 mode
= ctx
->mode
;
136 /* when activating SS, the default FIFO space is SS_RX_DEFAULT(32) */
137 u32 rx_cnt
= SS_RX_DEFAULT
;
142 unsigned int ileft
= areq
->nbytes
;
143 unsigned int oleft
= areq
->nbytes
;
145 struct sg_mapping_iter mi
, mo
;
146 unsigned int oi
, oo
; /* offset for in and out */
147 char buf
[4 * SS_RX_MAX
];/* buffer for linearize SG src */
148 char bufo
[4 * SS_TX_MAX
]; /* buffer for linearize SG dst */
149 unsigned int ob
= 0; /* offset in buf */
150 unsigned int obo
= 0; /* offset in bufo*/
151 unsigned int obl
= 0; /* length of data in bufo */
153 if (areq
->nbytes
== 0)
157 dev_err_ratelimited(ss
->dev
, "ERROR: Empty IV\n");
161 if (!areq
->src
|| !areq
->dst
) {
162 dev_err_ratelimited(ss
->dev
, "ERROR: Some SGs are NULL\n");
167 * if we have only SGs with size multiple of 4,
168 * we can use the SS optimized function
170 while (in_sg
&& no_chunk
== 1) {
171 if ((in_sg
->length
% 4) != 0)
173 in_sg
= sg_next(in_sg
);
175 while (out_sg
&& no_chunk
== 1) {
176 if ((out_sg
->length
% 4) != 0)
178 out_sg
= sg_next(out_sg
);
182 return sun4i_ss_opti_poll(areq
);
184 spin_lock_bh(&ss
->slock
);
186 for (i
= 0; i
< op
->keylen
; i
+= 4)
187 writel(*(op
->key
+ i
/ 4), ss
->base
+ SS_KEY0
+ i
);
190 for (i
= 0; i
< 4 && i
< ivsize
/ 4; i
++) {
191 v
= *(u32
*)(areq
->info
+ i
* 4);
192 writel(v
, ss
->base
+ SS_IV0
+ i
* 4);
195 writel(mode
, ss
->base
+ SS_CTL
);
197 sg_miter_start(&mi
, areq
->src
, sg_nents(areq
->src
),
198 SG_MITER_FROM_SG
| SG_MITER_ATOMIC
);
199 sg_miter_start(&mo
, areq
->dst
, sg_nents(areq
->dst
),
200 SG_MITER_TO_SG
| SG_MITER_ATOMIC
);
203 if (!mi
.addr
|| !mo
.addr
) {
204 dev_err_ratelimited(ss
->dev
, "ERROR: sg_miter return null\n");
208 ileft
= areq
->nbytes
;
209 oleft
= areq
->nbytes
;
216 * todo is the number of consecutive 4byte word that we
217 * can read from current SG
219 todo
= min3(rx_cnt
, ileft
/ 4, (mi
.length
- oi
) / 4);
220 if (todo
> 0 && ob
== 0) {
221 writesl(ss
->base
+ SS_RXFIFO
, mi
.addr
+ oi
,
227 * not enough consecutive bytes, so we need to
228 * linearize in buf. todo is in bytes
229 * After that copy, if we have a multiple of 4
230 * we need to be able to write all buf in one
231 * pass, so it is why we min() with rx_cnt
233 todo
= min3(rx_cnt
* 4 - ob
, ileft
,
235 memcpy(buf
+ ob
, mi
.addr
+ oi
, todo
);
240 writesl(ss
->base
+ SS_RXFIFO
, buf
,
245 if (oi
== mi
.length
) {
251 spaces
= readl(ss
->base
+ SS_FCSR
);
252 rx_cnt
= SS_RXFIFO_SPACES(spaces
);
253 tx_cnt
= SS_TXFIFO_SPACES(spaces
);
254 dev_dbg(ss
->dev
, "%x %u/%u %u/%u cnt=%u %u/%u %u/%u cnt=%u %u\n",
256 oi
, mi
.length
, ileft
, areq
->nbytes
, rx_cnt
,
257 oo
, mo
.length
, oleft
, areq
->nbytes
, tx_cnt
, ob
);
261 /* todo in 4bytes word */
262 todo
= min3(tx_cnt
, oleft
/ 4, (mo
.length
- oo
) / 4);
264 readsl(ss
->base
+ SS_TXFIFO
, mo
.addr
+ oo
, todo
);
267 if (oo
== mo
.length
) {
273 * read obl bytes in bufo, we read at maximum for
274 * emptying the device
276 readsl(ss
->base
+ SS_TXFIFO
, bufo
, tx_cnt
);
281 * how many bytes we can copy ?
282 * no more than remaining SG size
283 * no more than remaining buffer
284 * no need to test against oleft
286 todo
= min(mo
.length
- oo
, obl
- obo
);
287 memcpy(mo
.addr
+ oo
, bufo
+ obo
, todo
);
291 if (oo
== mo
.length
) {
296 /* bufo must be fully used here */
300 for (i
= 0; i
< 4 && i
< ivsize
/ 4; i
++) {
301 v
= readl(ss
->base
+ SS_IV0
+ i
* 4);
302 *(u32
*)(areq
->info
+ i
* 4) = v
;
309 writel(0, ss
->base
+ SS_CTL
);
310 spin_unlock_bh(&ss
->slock
);
316 int sun4i_ss_cbc_aes_encrypt(struct ablkcipher_request
*areq
)
318 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
319 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
320 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
322 rctx
->mode
= SS_OP_AES
| SS_CBC
| SS_ENABLED
| SS_ENCRYPTION
|
324 return sun4i_ss_cipher_poll(areq
);
327 int sun4i_ss_cbc_aes_decrypt(struct ablkcipher_request
*areq
)
329 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
330 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
331 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
333 rctx
->mode
= SS_OP_AES
| SS_CBC
| SS_ENABLED
| SS_DECRYPTION
|
335 return sun4i_ss_cipher_poll(areq
);
339 int sun4i_ss_ecb_aes_encrypt(struct ablkcipher_request
*areq
)
341 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
342 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
343 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
345 rctx
->mode
= SS_OP_AES
| SS_ECB
| SS_ENABLED
| SS_ENCRYPTION
|
347 return sun4i_ss_cipher_poll(areq
);
350 int sun4i_ss_ecb_aes_decrypt(struct ablkcipher_request
*areq
)
352 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
353 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
354 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
356 rctx
->mode
= SS_OP_AES
| SS_ECB
| SS_ENABLED
| SS_DECRYPTION
|
358 return sun4i_ss_cipher_poll(areq
);
362 int sun4i_ss_cbc_des_encrypt(struct ablkcipher_request
*areq
)
364 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
365 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
366 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
368 rctx
->mode
= SS_OP_DES
| SS_CBC
| SS_ENABLED
| SS_ENCRYPTION
|
370 return sun4i_ss_cipher_poll(areq
);
373 int sun4i_ss_cbc_des_decrypt(struct ablkcipher_request
*areq
)
375 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
376 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
377 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
379 rctx
->mode
= SS_OP_DES
| SS_CBC
| SS_ENABLED
| SS_DECRYPTION
|
381 return sun4i_ss_cipher_poll(areq
);
385 int sun4i_ss_ecb_des_encrypt(struct ablkcipher_request
*areq
)
387 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
388 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
389 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
391 rctx
->mode
= SS_OP_DES
| SS_ECB
| SS_ENABLED
| SS_ENCRYPTION
|
393 return sun4i_ss_cipher_poll(areq
);
396 int sun4i_ss_ecb_des_decrypt(struct ablkcipher_request
*areq
)
398 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
399 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
400 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
402 rctx
->mode
= SS_OP_DES
| SS_ECB
| SS_ENABLED
| SS_DECRYPTION
|
404 return sun4i_ss_cipher_poll(areq
);
408 int sun4i_ss_cbc_des3_encrypt(struct ablkcipher_request
*areq
)
410 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
411 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
412 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
414 rctx
->mode
= SS_OP_3DES
| SS_CBC
| SS_ENABLED
| SS_ENCRYPTION
|
416 return sun4i_ss_cipher_poll(areq
);
419 int sun4i_ss_cbc_des3_decrypt(struct ablkcipher_request
*areq
)
421 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
422 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
423 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
425 rctx
->mode
= SS_OP_3DES
| SS_CBC
| SS_ENABLED
| SS_DECRYPTION
|
427 return sun4i_ss_cipher_poll(areq
);
431 int sun4i_ss_ecb_des3_encrypt(struct ablkcipher_request
*areq
)
433 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
434 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
435 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
437 rctx
->mode
= SS_OP_3DES
| SS_ECB
| SS_ENABLED
| SS_ENCRYPTION
|
439 return sun4i_ss_cipher_poll(areq
);
442 int sun4i_ss_ecb_des3_decrypt(struct ablkcipher_request
*areq
)
444 struct crypto_ablkcipher
*tfm
= crypto_ablkcipher_reqtfm(areq
);
445 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
446 struct sun4i_cipher_req_ctx
*rctx
= ablkcipher_request_ctx(areq
);
448 rctx
->mode
= SS_OP_3DES
| SS_ECB
| SS_ENABLED
| SS_DECRYPTION
|
450 return sun4i_ss_cipher_poll(areq
);
453 int sun4i_ss_cipher_init(struct crypto_tfm
*tfm
)
455 struct sun4i_tfm_ctx
*op
= crypto_tfm_ctx(tfm
);
456 struct crypto_alg
*alg
= tfm
->__crt_alg
;
457 struct sun4i_ss_alg_template
*algt
;
459 memset(op
, 0, sizeof(struct sun4i_tfm_ctx
));
461 algt
= container_of(alg
, struct sun4i_ss_alg_template
, alg
.crypto
);
464 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct sun4i_cipher_req_ctx
);
469 /* check and set the AES key, prepare the mode to be used */
470 int sun4i_ss_aes_setkey(struct crypto_ablkcipher
*tfm
, const u8
*key
,
473 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
474 struct sun4i_ss_ctx
*ss
= op
->ss
;
478 op
->keymode
= SS_AES_128BITS
;
481 op
->keymode
= SS_AES_192BITS
;
484 op
->keymode
= SS_AES_256BITS
;
487 dev_err(ss
->dev
, "ERROR: Invalid keylen %u\n", keylen
);
488 crypto_ablkcipher_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
492 memcpy(op
->key
, key
, keylen
);
496 /* check and set the DES key, prepare the mode to be used */
497 int sun4i_ss_des_setkey(struct crypto_ablkcipher
*tfm
, const u8
*key
,
500 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
501 struct sun4i_ss_ctx
*ss
= op
->ss
;
503 u32 tmp
[DES_EXPKEY_WORDS
];
506 if (unlikely(keylen
!= DES_KEY_SIZE
)) {
507 dev_err(ss
->dev
, "Invalid keylen %u\n", keylen
);
508 crypto_ablkcipher_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
512 flags
= crypto_ablkcipher_get_flags(tfm
);
514 ret
= des_ekey(tmp
, key
);
515 if (unlikely(ret
== 0) && (flags
& CRYPTO_TFM_REQ_WEAK_KEY
)) {
516 crypto_ablkcipher_set_flags(tfm
, CRYPTO_TFM_RES_WEAK_KEY
);
517 dev_dbg(ss
->dev
, "Weak key %u\n", keylen
);
522 memcpy(op
->key
, key
, keylen
);
526 /* check and set the 3DES key, prepare the mode to be used */
527 int sun4i_ss_des3_setkey(struct crypto_ablkcipher
*tfm
, const u8
*key
,
530 struct sun4i_tfm_ctx
*op
= crypto_ablkcipher_ctx(tfm
);
531 struct sun4i_ss_ctx
*ss
= op
->ss
;
533 if (unlikely(keylen
!= 3 * DES_KEY_SIZE
)) {
534 dev_err(ss
->dev
, "Invalid keylen %u\n", keylen
);
535 crypto_ablkcipher_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
539 memcpy(op
->key
, key
, keylen
);