4 * Support for SAHARA cryptographic accelerator.
6 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
7 * Copyright (c) 2013 Vista Silicon S.L.
8 * Author: Javier Martin <javier.martin@vista-silicon.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as published
12 * by the Free Software Foundation.
14 * Based on omap-aes.c and tegra-aes.c
17 #include <crypto/algapi.h>
18 #include <crypto/aes.h>
19 #include <crypto/hash.h>
20 #include <crypto/internal/hash.h>
21 #include <crypto/scatterwalk.h>
22 #include <crypto/sha.h>
24 #include <linux/clk.h>
25 #include <linux/crypto.h>
26 #include <linux/interrupt.h>
28 #include <linux/irq.h>
29 #include <linux/kernel.h>
30 #include <linux/kthread.h>
31 #include <linux/module.h>
32 #include <linux/mutex.h>
34 #include <linux/of_device.h>
35 #include <linux/platform_device.h>
37 #define SHA_BUFFER_LEN PAGE_SIZE
38 #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
40 #define SAHARA_NAME "sahara"
41 #define SAHARA_VERSION_3 3
42 #define SAHARA_VERSION_4 4
43 #define SAHARA_TIMEOUT_MS 1000
44 #define SAHARA_MAX_HW_DESC 2
45 #define SAHARA_MAX_HW_LINK 20
47 #define FLAGS_MODE_MASK 0x000f
48 #define FLAGS_ENCRYPT BIT(0)
49 #define FLAGS_CBC BIT(1)
50 #define FLAGS_NEW_KEY BIT(3)
52 #define SAHARA_HDR_BASE 0x00800000
53 #define SAHARA_HDR_SKHA_ALG_AES 0
54 #define SAHARA_HDR_SKHA_OP_ENC (1 << 2)
55 #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3)
56 #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3)
57 #define SAHARA_HDR_FORM_DATA (5 << 16)
58 #define SAHARA_HDR_FORM_KEY (8 << 16)
59 #define SAHARA_HDR_LLO (1 << 24)
60 #define SAHARA_HDR_CHA_SKHA (1 << 28)
61 #define SAHARA_HDR_CHA_MDHA (2 << 28)
62 #define SAHARA_HDR_PARITY_BIT (1 << 31)
64 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
65 #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000
66 #define SAHARA_HDR_MDHA_HASH 0xA0850000
67 #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000
68 #define SAHARA_HDR_MDHA_ALG_SHA1 0
69 #define SAHARA_HDR_MDHA_ALG_MD5 1
70 #define SAHARA_HDR_MDHA_ALG_SHA256 2
71 #define SAHARA_HDR_MDHA_ALG_SHA224 3
72 #define SAHARA_HDR_MDHA_PDATA (1 << 2)
73 #define SAHARA_HDR_MDHA_HMAC (1 << 3)
74 #define SAHARA_HDR_MDHA_INIT (1 << 5)
75 #define SAHARA_HDR_MDHA_IPAD (1 << 6)
76 #define SAHARA_HDR_MDHA_OPAD (1 << 7)
77 #define SAHARA_HDR_MDHA_SWAP (1 << 8)
78 #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9)
79 #define SAHARA_HDR_MDHA_SSL (1 << 10)
81 /* SAHARA can only process one request at a time */
82 #define SAHARA_QUEUE_LENGTH 1
84 #define SAHARA_REG_VERSION 0x00
85 #define SAHARA_REG_DAR 0x04
86 #define SAHARA_REG_CONTROL 0x08
87 #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24)
88 #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16)
89 #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7)
90 #define SAHARA_CONTROL_ENABLE_INT (1 << 4)
91 #define SAHARA_REG_CMD 0x0C
92 #define SAHARA_CMD_RESET (1 << 0)
93 #define SAHARA_CMD_CLEAR_INT (1 << 8)
94 #define SAHARA_CMD_CLEAR_ERR (1 << 9)
95 #define SAHARA_CMD_SINGLE_STEP (1 << 10)
96 #define SAHARA_CMD_MODE_BATCH (1 << 16)
97 #define SAHARA_CMD_MODE_DEBUG (1 << 18)
98 #define SAHARA_REG_STATUS 0x10
99 #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7)
100 #define SAHARA_STATE_IDLE 0
101 #define SAHARA_STATE_BUSY 1
102 #define SAHARA_STATE_ERR 2
103 #define SAHARA_STATE_FAULT 3
104 #define SAHARA_STATE_COMPLETE 4
105 #define SAHARA_STATE_COMP_FLAG (1 << 2)
106 #define SAHARA_STATUS_DAR_FULL (1 << 3)
107 #define SAHARA_STATUS_ERROR (1 << 4)
108 #define SAHARA_STATUS_SECURE (1 << 5)
109 #define SAHARA_STATUS_FAIL (1 << 6)
110 #define SAHARA_STATUS_INIT (1 << 7)
111 #define SAHARA_STATUS_RNG_RESEED (1 << 8)
112 #define SAHARA_STATUS_ACTIVE_RNG (1 << 9)
113 #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10)
114 #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11)
115 #define SAHARA_STATUS_MODE_BATCH (1 << 16)
116 #define SAHARA_STATUS_MODE_DEDICATED (1 << 17)
117 #define SAHARA_STATUS_MODE_DEBUG (1 << 18)
118 #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff)
119 #define SAHARA_REG_ERRSTATUS 0x14
120 #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf)
121 #define SAHARA_ERRSOURCE_CHA 14
122 #define SAHARA_ERRSOURCE_DMA 15
123 #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8)
124 #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
125 #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
126 #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff)
127 #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3)
128 #define SAHARA_REG_FADDR 0x18
129 #define SAHARA_REG_CDAR 0x1C
130 #define SAHARA_REG_IDAR 0x20
132 struct sahara_hw_desc
{
141 struct sahara_hw_link
{
150 /* AES-specific context */
152 u8 key
[AES_KEYSIZE_128
];
153 struct crypto_ablkcipher
*fallback
;
155 /* SHA-specific context */
156 struct crypto_shash
*shash_fallback
;
159 struct sahara_aes_reqctx
{
164 * struct sahara_sha_reqctx - private data per request
165 * @buf: holds data for requests smaller than block_size
166 * @rembuf: used to prepare one block_size-aligned request
167 * @context: hw-specific context for request. Digest is extracted from this
168 * @mode: specifies what type of hw-descriptor needs to be built
169 * @digest_size: length of digest for this request
170 * @context_size: length of hw-context for this request.
171 * Always digest_size + 4
172 * @buf_cnt: number of bytes saved in buf
173 * @sg_in_idx: number of hw links
174 * @in_sg: scatterlist for input data
175 * @in_sg_chain: scatterlists for chained input data
176 * @total: total number of bytes for transfer
177 * @last: is this the last block
178 * @first: is this the first block
179 * @active: inside a transfer
181 struct sahara_sha_reqctx
{
182 u8 buf
[SAHARA_MAX_SHA_BLOCK_SIZE
];
183 u8 rembuf
[SAHARA_MAX_SHA_BLOCK_SIZE
];
184 u8 context
[SHA256_DIGEST_SIZE
+ 4];
187 unsigned int digest_size
;
188 unsigned int context_size
;
189 unsigned int buf_cnt
;
190 unsigned int sg_in_idx
;
191 struct scatterlist
*in_sg
;
192 struct scatterlist in_sg_chain
[2];
200 struct device
*device
;
201 unsigned int version
;
202 void __iomem
*regs_base
;
205 struct mutex queue_mutex
;
206 struct task_struct
*kthread
;
207 struct completion dma_completion
;
209 struct sahara_ctx
*ctx
;
211 struct crypto_queue queue
;
214 struct sahara_hw_desc
*hw_desc
[SAHARA_MAX_HW_DESC
];
215 dma_addr_t hw_phys_desc
[SAHARA_MAX_HW_DESC
];
218 dma_addr_t key_phys_base
;
221 dma_addr_t iv_phys_base
;
224 dma_addr_t context_phys_base
;
226 struct sahara_hw_link
*hw_link
[SAHARA_MAX_HW_LINK
];
227 dma_addr_t hw_phys_link
[SAHARA_MAX_HW_LINK
];
230 struct scatterlist
*in_sg
;
231 unsigned int nb_in_sg
;
232 struct scatterlist
*out_sg
;
233 unsigned int nb_out_sg
;
238 static struct sahara_dev
*dev_ptr
;
240 static inline void sahara_write(struct sahara_dev
*dev
, u32 data
, u32 reg
)
242 writel(data
, dev
->regs_base
+ reg
);
245 static inline unsigned int sahara_read(struct sahara_dev
*dev
, u32 reg
)
247 return readl(dev
->regs_base
+ reg
);
250 static u32
sahara_aes_key_hdr(struct sahara_dev
*dev
)
252 u32 hdr
= SAHARA_HDR_BASE
| SAHARA_HDR_SKHA_ALG_AES
|
253 SAHARA_HDR_FORM_KEY
| SAHARA_HDR_LLO
|
254 SAHARA_HDR_CHA_SKHA
| SAHARA_HDR_PARITY_BIT
;
256 if (dev
->flags
& FLAGS_CBC
) {
257 hdr
|= SAHARA_HDR_SKHA_MODE_CBC
;
258 hdr
^= SAHARA_HDR_PARITY_BIT
;
261 if (dev
->flags
& FLAGS_ENCRYPT
) {
262 hdr
|= SAHARA_HDR_SKHA_OP_ENC
;
263 hdr
^= SAHARA_HDR_PARITY_BIT
;
269 static u32
sahara_aes_data_link_hdr(struct sahara_dev
*dev
)
271 return SAHARA_HDR_BASE
| SAHARA_HDR_FORM_DATA
|
272 SAHARA_HDR_CHA_SKHA
| SAHARA_HDR_PARITY_BIT
;
275 static const char *sahara_err_src
[16] = {
278 "Descriptor length error",
279 "Descriptor length or pointer error",
281 "Link pointer error",
282 "Input buffer error",
283 "Output buffer error",
284 "Output buffer starvation",
285 "Internal state fault",
286 "General descriptor problem",
288 "Descriptor address error",
289 "Link address error",
294 static const char *sahara_err_dmasize
[4] = {
296 "Half-word transfer",
301 static const char *sahara_err_dmasrc
[8] = {
304 "Internal IP bus error",
306 "DMA crosses 256 byte boundary",
312 static const char *sahara_cha_errsrc
[12] = {
313 "Input buffer non-empty",
318 "Write during processing",
319 "CTX read during processing",
321 "Input buffer disabled/underflow",
322 "Output buffer disabled/overflow",
323 "DES key parity error",
327 static const char *sahara_cha_err
[4] = { "No error", "SKHA", "MDHA", "RNG" };
329 static void sahara_decode_error(struct sahara_dev
*dev
, unsigned int error
)
331 u8 source
= SAHARA_ERRSTATUS_GET_SOURCE(error
);
332 u16 chasrc
= ffs(SAHARA_ERRSTATUS_GET_CHASRC(error
));
334 dev_err(dev
->device
, "%s: Error Register = 0x%08x\n", __func__
, error
);
336 dev_err(dev
->device
, " - %s.\n", sahara_err_src
[source
]);
338 if (source
== SAHARA_ERRSOURCE_DMA
) {
339 if (error
& SAHARA_ERRSTATUS_DMA_DIR
)
340 dev_err(dev
->device
, " * DMA read.\n");
342 dev_err(dev
->device
, " * DMA write.\n");
344 dev_err(dev
->device
, " * %s.\n",
345 sahara_err_dmasize
[SAHARA_ERRSTATUS_GET_DMASZ(error
)]);
346 dev_err(dev
->device
, " * %s.\n",
347 sahara_err_dmasrc
[SAHARA_ERRSTATUS_GET_DMASRC(error
)]);
348 } else if (source
== SAHARA_ERRSOURCE_CHA
) {
349 dev_err(dev
->device
, " * %s.\n",
350 sahara_cha_errsrc
[chasrc
]);
351 dev_err(dev
->device
, " * %s.\n",
352 sahara_cha_err
[SAHARA_ERRSTATUS_GET_CHAERR(error
)]);
354 dev_err(dev
->device
, "\n");
357 static const char *sahara_state
[4] = { "Idle", "Busy", "Error", "HW Fault" };
359 static void sahara_decode_status(struct sahara_dev
*dev
, unsigned int status
)
363 if (!IS_ENABLED(DEBUG
))
366 state
= SAHARA_STATUS_GET_STATE(status
);
368 dev_dbg(dev
->device
, "%s: Status Register = 0x%08x\n",
371 dev_dbg(dev
->device
, " - State = %d:\n", state
);
372 if (state
& SAHARA_STATE_COMP_FLAG
)
373 dev_dbg(dev
->device
, " * Descriptor completed. IRQ pending.\n");
375 dev_dbg(dev
->device
, " * %s.\n",
376 sahara_state
[state
& ~SAHARA_STATE_COMP_FLAG
]);
378 if (status
& SAHARA_STATUS_DAR_FULL
)
379 dev_dbg(dev
->device
, " - DAR Full.\n");
380 if (status
& SAHARA_STATUS_ERROR
)
381 dev_dbg(dev
->device
, " - Error.\n");
382 if (status
& SAHARA_STATUS_SECURE
)
383 dev_dbg(dev
->device
, " - Secure.\n");
384 if (status
& SAHARA_STATUS_FAIL
)
385 dev_dbg(dev
->device
, " - Fail.\n");
386 if (status
& SAHARA_STATUS_RNG_RESEED
)
387 dev_dbg(dev
->device
, " - RNG Reseed Request.\n");
388 if (status
& SAHARA_STATUS_ACTIVE_RNG
)
389 dev_dbg(dev
->device
, " - RNG Active.\n");
390 if (status
& SAHARA_STATUS_ACTIVE_MDHA
)
391 dev_dbg(dev
->device
, " - MDHA Active.\n");
392 if (status
& SAHARA_STATUS_ACTIVE_SKHA
)
393 dev_dbg(dev
->device
, " - SKHA Active.\n");
395 if (status
& SAHARA_STATUS_MODE_BATCH
)
396 dev_dbg(dev
->device
, " - Batch Mode.\n");
397 else if (status
& SAHARA_STATUS_MODE_DEDICATED
)
398 dev_dbg(dev
->device
, " - Decidated Mode.\n");
399 else if (status
& SAHARA_STATUS_MODE_DEBUG
)
400 dev_dbg(dev
->device
, " - Debug Mode.\n");
402 dev_dbg(dev
->device
, " - Internal state = 0x%02x\n",
403 SAHARA_STATUS_GET_ISTATE(status
));
405 dev_dbg(dev
->device
, "Current DAR: 0x%08x\n",
406 sahara_read(dev
, SAHARA_REG_CDAR
));
407 dev_dbg(dev
->device
, "Initial DAR: 0x%08x\n\n",
408 sahara_read(dev
, SAHARA_REG_IDAR
));
411 static void sahara_dump_descriptors(struct sahara_dev
*dev
)
415 if (!IS_ENABLED(DEBUG
))
418 for (i
= 0; i
< SAHARA_MAX_HW_DESC
; i
++) {
419 dev_dbg(dev
->device
, "Descriptor (%d) (0x%08x):\n",
420 i
, dev
->hw_phys_desc
[i
]);
421 dev_dbg(dev
->device
, "\thdr = 0x%08x\n", dev
->hw_desc
[i
]->hdr
);
422 dev_dbg(dev
->device
, "\tlen1 = %u\n", dev
->hw_desc
[i
]->len1
);
423 dev_dbg(dev
->device
, "\tp1 = 0x%08x\n", dev
->hw_desc
[i
]->p1
);
424 dev_dbg(dev
->device
, "\tlen2 = %u\n", dev
->hw_desc
[i
]->len2
);
425 dev_dbg(dev
->device
, "\tp2 = 0x%08x\n", dev
->hw_desc
[i
]->p2
);
426 dev_dbg(dev
->device
, "\tnext = 0x%08x\n",
427 dev
->hw_desc
[i
]->next
);
429 dev_dbg(dev
->device
, "\n");
432 static void sahara_dump_links(struct sahara_dev
*dev
)
436 if (!IS_ENABLED(DEBUG
))
439 for (i
= 0; i
< SAHARA_MAX_HW_LINK
; i
++) {
440 dev_dbg(dev
->device
, "Link (%d) (0x%08x):\n",
441 i
, dev
->hw_phys_link
[i
]);
442 dev_dbg(dev
->device
, "\tlen = %u\n", dev
->hw_link
[i
]->len
);
443 dev_dbg(dev
->device
, "\tp = 0x%08x\n", dev
->hw_link
[i
]->p
);
444 dev_dbg(dev
->device
, "\tnext = 0x%08x\n",
445 dev
->hw_link
[i
]->next
);
447 dev_dbg(dev
->device
, "\n");
450 static int sahara_hw_descriptor_create(struct sahara_dev
*dev
)
452 struct sahara_ctx
*ctx
= dev
->ctx
;
453 struct scatterlist
*sg
;
458 /* Copy new key if necessary */
459 if (ctx
->flags
& FLAGS_NEW_KEY
) {
460 memcpy(dev
->key_base
, ctx
->key
, ctx
->keylen
);
461 ctx
->flags
&= ~FLAGS_NEW_KEY
;
463 if (dev
->flags
& FLAGS_CBC
) {
464 dev
->hw_desc
[idx
]->len1
= AES_BLOCK_SIZE
;
465 dev
->hw_desc
[idx
]->p1
= dev
->iv_phys_base
;
467 dev
->hw_desc
[idx
]->len1
= 0;
468 dev
->hw_desc
[idx
]->p1
= 0;
470 dev
->hw_desc
[idx
]->len2
= ctx
->keylen
;
471 dev
->hw_desc
[idx
]->p2
= dev
->key_phys_base
;
472 dev
->hw_desc
[idx
]->next
= dev
->hw_phys_desc
[1];
474 dev
->hw_desc
[idx
]->hdr
= sahara_aes_key_hdr(dev
);
479 dev
->nb_in_sg
= sg_nents_for_len(dev
->in_sg
, dev
->total
);
480 dev
->nb_out_sg
= sg_nents_for_len(dev
->out_sg
, dev
->total
);
481 if ((dev
->nb_in_sg
+ dev
->nb_out_sg
) > SAHARA_MAX_HW_LINK
) {
482 dev_err(dev
->device
, "not enough hw links (%d)\n",
483 dev
->nb_in_sg
+ dev
->nb_out_sg
);
487 ret
= dma_map_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
489 if (ret
!= dev
->nb_in_sg
) {
490 dev_err(dev
->device
, "couldn't map in sg\n");
493 ret
= dma_map_sg(dev
->device
, dev
->out_sg
, dev
->nb_out_sg
,
495 if (ret
!= dev
->nb_out_sg
) {
496 dev_err(dev
->device
, "couldn't map out sg\n");
500 /* Create input links */
501 dev
->hw_desc
[idx
]->p1
= dev
->hw_phys_link
[0];
503 for (i
= 0; i
< dev
->nb_in_sg
; i
++) {
504 dev
->hw_link
[i
]->len
= sg
->length
;
505 dev
->hw_link
[i
]->p
= sg
->dma_address
;
506 if (i
== (dev
->nb_in_sg
- 1)) {
507 dev
->hw_link
[i
]->next
= 0;
509 dev
->hw_link
[i
]->next
= dev
->hw_phys_link
[i
+ 1];
514 /* Create output links */
515 dev
->hw_desc
[idx
]->p2
= dev
->hw_phys_link
[i
];
517 for (j
= i
; j
< dev
->nb_out_sg
+ i
; j
++) {
518 dev
->hw_link
[j
]->len
= sg
->length
;
519 dev
->hw_link
[j
]->p
= sg
->dma_address
;
520 if (j
== (dev
->nb_out_sg
+ i
- 1)) {
521 dev
->hw_link
[j
]->next
= 0;
523 dev
->hw_link
[j
]->next
= dev
->hw_phys_link
[j
+ 1];
528 /* Fill remaining fields of hw_desc[1] */
529 dev
->hw_desc
[idx
]->hdr
= sahara_aes_data_link_hdr(dev
);
530 dev
->hw_desc
[idx
]->len1
= dev
->total
;
531 dev
->hw_desc
[idx
]->len2
= dev
->total
;
532 dev
->hw_desc
[idx
]->next
= 0;
534 sahara_dump_descriptors(dev
);
535 sahara_dump_links(dev
);
537 sahara_write(dev
, dev
->hw_phys_desc
[0], SAHARA_REG_DAR
);
542 dma_unmap_sg(dev
->device
, dev
->out_sg
, dev
->nb_out_sg
,
545 dma_unmap_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
551 static int sahara_aes_process(struct ablkcipher_request
*req
)
553 struct sahara_dev
*dev
= dev_ptr
;
554 struct sahara_ctx
*ctx
;
555 struct sahara_aes_reqctx
*rctx
;
557 unsigned long timeout
;
559 /* Request is ready to be dispatched by the device */
561 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
562 req
->nbytes
, req
->src
, req
->dst
);
564 /* assign new request to device */
565 dev
->total
= req
->nbytes
;
566 dev
->in_sg
= req
->src
;
567 dev
->out_sg
= req
->dst
;
569 rctx
= ablkcipher_request_ctx(req
);
570 ctx
= crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req
));
571 rctx
->mode
&= FLAGS_MODE_MASK
;
572 dev
->flags
= (dev
->flags
& ~FLAGS_MODE_MASK
) | rctx
->mode
;
574 if ((dev
->flags
& FLAGS_CBC
) && req
->info
)
575 memcpy(dev
->iv_base
, req
->info
, AES_KEYSIZE_128
);
577 /* assign new context to device */
580 reinit_completion(&dev
->dma_completion
);
582 ret
= sahara_hw_descriptor_create(dev
);
586 timeout
= wait_for_completion_timeout(&dev
->dma_completion
,
587 msecs_to_jiffies(SAHARA_TIMEOUT_MS
));
589 dev_err(dev
->device
, "AES timeout\n");
593 dma_unmap_sg(dev
->device
, dev
->out_sg
, dev
->nb_out_sg
,
595 dma_unmap_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
601 static int sahara_aes_setkey(struct crypto_ablkcipher
*tfm
, const u8
*key
,
604 struct sahara_ctx
*ctx
= crypto_ablkcipher_ctx(tfm
);
607 ctx
->keylen
= keylen
;
609 /* SAHARA only supports 128bit keys */
610 if (keylen
== AES_KEYSIZE_128
) {
611 memcpy(ctx
->key
, key
, keylen
);
612 ctx
->flags
|= FLAGS_NEW_KEY
;
616 if (keylen
!= AES_KEYSIZE_128
&&
617 keylen
!= AES_KEYSIZE_192
&& keylen
!= AES_KEYSIZE_256
)
621 * The requested key size is not supported by HW, do a fallback.
623 ctx
->fallback
->base
.crt_flags
&= ~CRYPTO_TFM_REQ_MASK
;
624 ctx
->fallback
->base
.crt_flags
|=
625 (tfm
->base
.crt_flags
& CRYPTO_TFM_REQ_MASK
);
627 ret
= crypto_ablkcipher_setkey(ctx
->fallback
, key
, keylen
);
629 struct crypto_tfm
*tfm_aux
= crypto_ablkcipher_tfm(tfm
);
631 tfm_aux
->crt_flags
&= ~CRYPTO_TFM_RES_MASK
;
632 tfm_aux
->crt_flags
|=
633 (ctx
->fallback
->base
.crt_flags
& CRYPTO_TFM_RES_MASK
);
638 static int sahara_aes_crypt(struct ablkcipher_request
*req
, unsigned long mode
)
640 struct sahara_aes_reqctx
*rctx
= ablkcipher_request_ctx(req
);
641 struct sahara_dev
*dev
= dev_ptr
;
644 dev_dbg(dev
->device
, "nbytes: %d, enc: %d, cbc: %d\n",
645 req
->nbytes
, !!(mode
& FLAGS_ENCRYPT
), !!(mode
& FLAGS_CBC
));
647 if (!IS_ALIGNED(req
->nbytes
, AES_BLOCK_SIZE
)) {
649 "request size is not exact amount of AES blocks\n");
655 mutex_lock(&dev
->queue_mutex
);
656 err
= ablkcipher_enqueue_request(&dev
->queue
, req
);
657 mutex_unlock(&dev
->queue_mutex
);
659 wake_up_process(dev
->kthread
);
664 static int sahara_aes_ecb_encrypt(struct ablkcipher_request
*req
)
666 struct crypto_tfm
*tfm
=
667 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req
));
668 struct sahara_ctx
*ctx
= crypto_ablkcipher_ctx(
669 crypto_ablkcipher_reqtfm(req
));
672 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
673 ablkcipher_request_set_tfm(req
, ctx
->fallback
);
674 err
= crypto_ablkcipher_encrypt(req
);
675 ablkcipher_request_set_tfm(req
, __crypto_ablkcipher_cast(tfm
));
679 return sahara_aes_crypt(req
, FLAGS_ENCRYPT
);
682 static int sahara_aes_ecb_decrypt(struct ablkcipher_request
*req
)
684 struct crypto_tfm
*tfm
=
685 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req
));
686 struct sahara_ctx
*ctx
= crypto_ablkcipher_ctx(
687 crypto_ablkcipher_reqtfm(req
));
690 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
691 ablkcipher_request_set_tfm(req
, ctx
->fallback
);
692 err
= crypto_ablkcipher_decrypt(req
);
693 ablkcipher_request_set_tfm(req
, __crypto_ablkcipher_cast(tfm
));
697 return sahara_aes_crypt(req
, 0);
700 static int sahara_aes_cbc_encrypt(struct ablkcipher_request
*req
)
702 struct crypto_tfm
*tfm
=
703 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req
));
704 struct sahara_ctx
*ctx
= crypto_ablkcipher_ctx(
705 crypto_ablkcipher_reqtfm(req
));
708 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
709 ablkcipher_request_set_tfm(req
, ctx
->fallback
);
710 err
= crypto_ablkcipher_encrypt(req
);
711 ablkcipher_request_set_tfm(req
, __crypto_ablkcipher_cast(tfm
));
715 return sahara_aes_crypt(req
, FLAGS_ENCRYPT
| FLAGS_CBC
);
718 static int sahara_aes_cbc_decrypt(struct ablkcipher_request
*req
)
720 struct crypto_tfm
*tfm
=
721 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req
));
722 struct sahara_ctx
*ctx
= crypto_ablkcipher_ctx(
723 crypto_ablkcipher_reqtfm(req
));
726 if (unlikely(ctx
->keylen
!= AES_KEYSIZE_128
)) {
727 ablkcipher_request_set_tfm(req
, ctx
->fallback
);
728 err
= crypto_ablkcipher_decrypt(req
);
729 ablkcipher_request_set_tfm(req
, __crypto_ablkcipher_cast(tfm
));
733 return sahara_aes_crypt(req
, FLAGS_CBC
);
736 static int sahara_aes_cra_init(struct crypto_tfm
*tfm
)
738 const char *name
= crypto_tfm_alg_name(tfm
);
739 struct sahara_ctx
*ctx
= crypto_tfm_ctx(tfm
);
741 ctx
->fallback
= crypto_alloc_ablkcipher(name
, 0,
742 CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
);
743 if (IS_ERR(ctx
->fallback
)) {
744 pr_err("Error allocating fallback algo %s\n", name
);
745 return PTR_ERR(ctx
->fallback
);
748 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct sahara_aes_reqctx
);
753 static void sahara_aes_cra_exit(struct crypto_tfm
*tfm
)
755 struct sahara_ctx
*ctx
= crypto_tfm_ctx(tfm
);
758 crypto_free_ablkcipher(ctx
->fallback
);
759 ctx
->fallback
= NULL
;
762 static u32
sahara_sha_init_hdr(struct sahara_dev
*dev
,
763 struct sahara_sha_reqctx
*rctx
)
770 hdr
|= SAHARA_HDR_MDHA_SET_MODE_HASH
;
771 hdr
|= SAHARA_HDR_MDHA_INIT
;
773 hdr
|= SAHARA_HDR_MDHA_SET_MODE_MD_KEY
;
777 hdr
|= SAHARA_HDR_MDHA_PDATA
;
779 if (hweight_long(hdr
) % 2 == 0)
780 hdr
|= SAHARA_HDR_PARITY_BIT
;
785 static int sahara_sha_hw_links_create(struct sahara_dev
*dev
,
786 struct sahara_sha_reqctx
*rctx
,
789 struct scatterlist
*sg
;
793 dev
->in_sg
= rctx
->in_sg
;
795 dev
->nb_in_sg
= sg_nents_for_len(dev
->in_sg
, rctx
->total
);
796 if ((dev
->nb_in_sg
) > SAHARA_MAX_HW_LINK
) {
797 dev_err(dev
->device
, "not enough hw links (%d)\n",
798 dev
->nb_in_sg
+ dev
->nb_out_sg
);
803 ret
= dma_map_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
, DMA_TO_DEVICE
);
807 for (i
= start
; i
< dev
->nb_in_sg
+ start
; i
++) {
808 dev
->hw_link
[i
]->len
= sg
->length
;
809 dev
->hw_link
[i
]->p
= sg
->dma_address
;
810 if (i
== (dev
->nb_in_sg
+ start
- 1)) {
811 dev
->hw_link
[i
]->next
= 0;
813 dev
->hw_link
[i
]->next
= dev
->hw_phys_link
[i
+ 1];
821 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev
*dev
,
822 struct sahara_sha_reqctx
*rctx
,
823 struct ahash_request
*req
,
830 /* Create initial descriptor: #8*/
831 dev
->hw_desc
[index
]->hdr
= sahara_sha_init_hdr(dev
, rctx
);
833 /* Create hash descriptor: #10. Must follow #6. */
834 dev
->hw_desc
[index
]->hdr
= SAHARA_HDR_MDHA_HASH
;
836 dev
->hw_desc
[index
]->len1
= rctx
->total
;
837 if (dev
->hw_desc
[index
]->len1
== 0) {
838 /* if len1 is 0, p1 must be 0, too */
839 dev
->hw_desc
[index
]->p1
= 0;
842 /* Create input links */
843 dev
->hw_desc
[index
]->p1
= dev
->hw_phys_link
[index
];
844 i
= sahara_sha_hw_links_create(dev
, rctx
, index
);
846 rctx
->sg_in_idx
= index
;
851 dev
->hw_desc
[index
]->p2
= dev
->hw_phys_link
[i
];
853 /* Save the context for the next operation */
854 result_len
= rctx
->context_size
;
855 dev
->hw_link
[i
]->p
= dev
->context_phys_base
;
857 dev
->hw_link
[i
]->len
= result_len
;
858 dev
->hw_desc
[index
]->len2
= result_len
;
860 dev
->hw_link
[i
]->next
= 0;
866 * Load descriptor aka #6
868 * To load a previously saved context back to the MDHA unit
874 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev
*dev
,
875 struct sahara_sha_reqctx
*rctx
,
876 struct ahash_request
*req
,
879 dev
->hw_desc
[index
]->hdr
= sahara_sha_init_hdr(dev
, rctx
);
881 dev
->hw_desc
[index
]->len1
= rctx
->context_size
;
882 dev
->hw_desc
[index
]->p1
= dev
->hw_phys_link
[index
];
883 dev
->hw_desc
[index
]->len2
= 0;
884 dev
->hw_desc
[index
]->p2
= 0;
886 dev
->hw_link
[index
]->len
= rctx
->context_size
;
887 dev
->hw_link
[index
]->p
= dev
->context_phys_base
;
888 dev
->hw_link
[index
]->next
= 0;
893 static int sahara_walk_and_recalc(struct scatterlist
*sg
, unsigned int nbytes
)
895 if (!sg
|| !sg
->length
)
898 while (nbytes
&& sg
) {
899 if (nbytes
<= sg
->length
) {
904 nbytes
-= sg
->length
;
911 static int sahara_sha_prepare_request(struct ahash_request
*req
)
913 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
914 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
915 unsigned int hash_later
;
916 unsigned int block_size
;
919 block_size
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
921 /* append bytes from previous operation */
922 len
= rctx
->buf_cnt
+ req
->nbytes
;
924 /* only the last transfer can be padded in hardware */
925 if (!rctx
->last
&& (len
< block_size
)) {
926 /* to few data, save for next operation */
927 scatterwalk_map_and_copy(rctx
->buf
+ rctx
->buf_cnt
, req
->src
,
929 rctx
->buf_cnt
+= req
->nbytes
;
934 /* add data from previous operation first */
936 memcpy(rctx
->rembuf
, rctx
->buf
, rctx
->buf_cnt
);
938 /* data must always be a multiple of block_size */
939 hash_later
= rctx
->last
? 0 : len
& (block_size
- 1);
941 unsigned int offset
= req
->nbytes
- hash_later
;
942 /* Save remaining bytes for later use */
943 scatterwalk_map_and_copy(rctx
->buf
, req
->src
, offset
,
947 /* nbytes should now be multiple of blocksize */
948 req
->nbytes
= req
->nbytes
- hash_later
;
950 sahara_walk_and_recalc(req
->src
, req
->nbytes
);
952 /* have data from previous operation and current */
953 if (rctx
->buf_cnt
&& req
->nbytes
) {
954 sg_init_table(rctx
->in_sg_chain
, 2);
955 sg_set_buf(rctx
->in_sg_chain
, rctx
->rembuf
, rctx
->buf_cnt
);
957 sg_chain(rctx
->in_sg_chain
, 2, req
->src
);
959 rctx
->total
= req
->nbytes
+ rctx
->buf_cnt
;
960 rctx
->in_sg
= rctx
->in_sg_chain
;
962 req
->src
= rctx
->in_sg_chain
;
963 /* only data from previous operation */
964 } else if (rctx
->buf_cnt
) {
966 rctx
->in_sg
= req
->src
;
968 rctx
->in_sg
= rctx
->in_sg_chain
;
969 /* buf was copied into rembuf above */
970 sg_init_one(rctx
->in_sg
, rctx
->rembuf
, rctx
->buf_cnt
);
971 rctx
->total
= rctx
->buf_cnt
;
972 /* no data from previous operation */
974 rctx
->in_sg
= req
->src
;
975 rctx
->total
= req
->nbytes
;
976 req
->src
= rctx
->in_sg
;
979 /* on next call, we only have the remaining data in the buffer */
980 rctx
->buf_cnt
= hash_later
;
985 static int sahara_sha_process(struct ahash_request
*req
)
987 struct sahara_dev
*dev
= dev_ptr
;
988 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
990 unsigned long timeout
;
992 ret
= sahara_sha_prepare_request(req
);
997 sahara_sha_hw_data_descriptor_create(dev
, rctx
, req
, 0);
998 dev
->hw_desc
[0]->next
= 0;
1001 memcpy(dev
->context_base
, rctx
->context
, rctx
->context_size
);
1003 sahara_sha_hw_context_descriptor_create(dev
, rctx
, req
, 0);
1004 dev
->hw_desc
[0]->next
= dev
->hw_phys_desc
[1];
1005 sahara_sha_hw_data_descriptor_create(dev
, rctx
, req
, 1);
1006 dev
->hw_desc
[1]->next
= 0;
1009 sahara_dump_descriptors(dev
);
1010 sahara_dump_links(dev
);
1012 reinit_completion(&dev
->dma_completion
);
1014 sahara_write(dev
, dev
->hw_phys_desc
[0], SAHARA_REG_DAR
);
1016 timeout
= wait_for_completion_timeout(&dev
->dma_completion
,
1017 msecs_to_jiffies(SAHARA_TIMEOUT_MS
));
1019 dev_err(dev
->device
, "SHA timeout\n");
1023 if (rctx
->sg_in_idx
)
1024 dma_unmap_sg(dev
->device
, dev
->in_sg
, dev
->nb_in_sg
,
1027 memcpy(rctx
->context
, dev
->context_base
, rctx
->context_size
);
1030 memcpy(req
->result
, rctx
->context
, rctx
->digest_size
);
1035 static int sahara_queue_manage(void *data
)
1037 struct sahara_dev
*dev
= (struct sahara_dev
*)data
;
1038 struct crypto_async_request
*async_req
;
1039 struct crypto_async_request
*backlog
;
1043 __set_current_state(TASK_INTERRUPTIBLE
);
1045 mutex_lock(&dev
->queue_mutex
);
1046 backlog
= crypto_get_backlog(&dev
->queue
);
1047 async_req
= crypto_dequeue_request(&dev
->queue
);
1048 mutex_unlock(&dev
->queue_mutex
);
1051 backlog
->complete(backlog
, -EINPROGRESS
);
1054 if (crypto_tfm_alg_type(async_req
->tfm
) ==
1055 CRYPTO_ALG_TYPE_AHASH
) {
1056 struct ahash_request
*req
=
1057 ahash_request_cast(async_req
);
1059 ret
= sahara_sha_process(req
);
1061 struct ablkcipher_request
*req
=
1062 ablkcipher_request_cast(async_req
);
1064 ret
= sahara_aes_process(req
);
1067 async_req
->complete(async_req
, ret
);
1073 } while (!kthread_should_stop());
1078 static int sahara_sha_enqueue(struct ahash_request
*req
, int last
)
1080 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1081 struct sahara_dev
*dev
= dev_ptr
;
1084 if (!req
->nbytes
&& !last
)
1087 mutex_lock(&rctx
->mutex
);
1090 if (!rctx
->active
) {
1095 mutex_lock(&dev
->queue_mutex
);
1096 ret
= crypto_enqueue_request(&dev
->queue
, &req
->base
);
1097 mutex_unlock(&dev
->queue_mutex
);
1099 wake_up_process(dev
->kthread
);
1100 mutex_unlock(&rctx
->mutex
);
1105 static int sahara_sha_init(struct ahash_request
*req
)
1107 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
1108 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1110 memset(rctx
, 0, sizeof(*rctx
));
1112 switch (crypto_ahash_digestsize(tfm
)) {
1113 case SHA1_DIGEST_SIZE
:
1114 rctx
->mode
|= SAHARA_HDR_MDHA_ALG_SHA1
;
1115 rctx
->digest_size
= SHA1_DIGEST_SIZE
;
1117 case SHA256_DIGEST_SIZE
:
1118 rctx
->mode
|= SAHARA_HDR_MDHA_ALG_SHA256
;
1119 rctx
->digest_size
= SHA256_DIGEST_SIZE
;
1125 rctx
->context_size
= rctx
->digest_size
+ 4;
1128 mutex_init(&rctx
->mutex
);
1133 static int sahara_sha_update(struct ahash_request
*req
)
1135 return sahara_sha_enqueue(req
, 0);
1138 static int sahara_sha_final(struct ahash_request
*req
)
1141 return sahara_sha_enqueue(req
, 1);
1144 static int sahara_sha_finup(struct ahash_request
*req
)
1146 return sahara_sha_enqueue(req
, 1);
1149 static int sahara_sha_digest(struct ahash_request
*req
)
1151 sahara_sha_init(req
);
1153 return sahara_sha_finup(req
);
1156 static int sahara_sha_export(struct ahash_request
*req
, void *out
)
1158 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(req
);
1159 struct sahara_ctx
*ctx
= crypto_ahash_ctx(ahash
);
1160 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1162 memcpy(out
, ctx
, sizeof(struct sahara_ctx
));
1163 memcpy(out
+ sizeof(struct sahara_sha_reqctx
), rctx
,
1164 sizeof(struct sahara_sha_reqctx
));
1169 static int sahara_sha_import(struct ahash_request
*req
, const void *in
)
1171 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(req
);
1172 struct sahara_ctx
*ctx
= crypto_ahash_ctx(ahash
);
1173 struct sahara_sha_reqctx
*rctx
= ahash_request_ctx(req
);
1175 memcpy(ctx
, in
, sizeof(struct sahara_ctx
));
1176 memcpy(rctx
, in
+ sizeof(struct sahara_sha_reqctx
),
1177 sizeof(struct sahara_sha_reqctx
));
1182 static int sahara_sha_cra_init(struct crypto_tfm
*tfm
)
1184 const char *name
= crypto_tfm_alg_name(tfm
);
1185 struct sahara_ctx
*ctx
= crypto_tfm_ctx(tfm
);
1187 ctx
->shash_fallback
= crypto_alloc_shash(name
, 0,
1188 CRYPTO_ALG_NEED_FALLBACK
);
1189 if (IS_ERR(ctx
->shash_fallback
)) {
1190 pr_err("Error allocating fallback algo %s\n", name
);
1191 return PTR_ERR(ctx
->shash_fallback
);
1193 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
1194 sizeof(struct sahara_sha_reqctx
) +
1195 SHA_BUFFER_LEN
+ SHA256_BLOCK_SIZE
);
1200 static void sahara_sha_cra_exit(struct crypto_tfm
*tfm
)
1202 struct sahara_ctx
*ctx
= crypto_tfm_ctx(tfm
);
1204 crypto_free_shash(ctx
->shash_fallback
);
1205 ctx
->shash_fallback
= NULL
;
1208 static struct crypto_alg aes_algs
[] = {
1210 .cra_name
= "ecb(aes)",
1211 .cra_driver_name
= "sahara-ecb-aes",
1212 .cra_priority
= 300,
1213 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1214 CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
,
1215 .cra_blocksize
= AES_BLOCK_SIZE
,
1216 .cra_ctxsize
= sizeof(struct sahara_ctx
),
1217 .cra_alignmask
= 0x0,
1218 .cra_type
= &crypto_ablkcipher_type
,
1219 .cra_module
= THIS_MODULE
,
1220 .cra_init
= sahara_aes_cra_init
,
1221 .cra_exit
= sahara_aes_cra_exit
,
1222 .cra_u
.ablkcipher
= {
1223 .min_keysize
= AES_MIN_KEY_SIZE
,
1224 .max_keysize
= AES_MAX_KEY_SIZE
,
1225 .setkey
= sahara_aes_setkey
,
1226 .encrypt
= sahara_aes_ecb_encrypt
,
1227 .decrypt
= sahara_aes_ecb_decrypt
,
1230 .cra_name
= "cbc(aes)",
1231 .cra_driver_name
= "sahara-cbc-aes",
1232 .cra_priority
= 300,
1233 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1234 CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
,
1235 .cra_blocksize
= AES_BLOCK_SIZE
,
1236 .cra_ctxsize
= sizeof(struct sahara_ctx
),
1237 .cra_alignmask
= 0x0,
1238 .cra_type
= &crypto_ablkcipher_type
,
1239 .cra_module
= THIS_MODULE
,
1240 .cra_init
= sahara_aes_cra_init
,
1241 .cra_exit
= sahara_aes_cra_exit
,
1242 .cra_u
.ablkcipher
= {
1243 .min_keysize
= AES_MIN_KEY_SIZE
,
1244 .max_keysize
= AES_MAX_KEY_SIZE
,
1245 .ivsize
= AES_BLOCK_SIZE
,
1246 .setkey
= sahara_aes_setkey
,
1247 .encrypt
= sahara_aes_cbc_encrypt
,
1248 .decrypt
= sahara_aes_cbc_decrypt
,
1253 static struct ahash_alg sha_v3_algs
[] = {
1255 .init
= sahara_sha_init
,
1256 .update
= sahara_sha_update
,
1257 .final
= sahara_sha_final
,
1258 .finup
= sahara_sha_finup
,
1259 .digest
= sahara_sha_digest
,
1260 .export
= sahara_sha_export
,
1261 .import
= sahara_sha_import
,
1262 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
1265 .cra_driver_name
= "sahara-sha1",
1266 .cra_priority
= 300,
1267 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1269 CRYPTO_ALG_NEED_FALLBACK
,
1270 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1271 .cra_ctxsize
= sizeof(struct sahara_ctx
),
1273 .cra_module
= THIS_MODULE
,
1274 .cra_init
= sahara_sha_cra_init
,
1275 .cra_exit
= sahara_sha_cra_exit
,
1280 static struct ahash_alg sha_v4_algs
[] = {
1282 .init
= sahara_sha_init
,
1283 .update
= sahara_sha_update
,
1284 .final
= sahara_sha_final
,
1285 .finup
= sahara_sha_finup
,
1286 .digest
= sahara_sha_digest
,
1287 .export
= sahara_sha_export
,
1288 .import
= sahara_sha_import
,
1289 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
1291 .cra_name
= "sha256",
1292 .cra_driver_name
= "sahara-sha256",
1293 .cra_priority
= 300,
1294 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
1296 CRYPTO_ALG_NEED_FALLBACK
,
1297 .cra_blocksize
= SHA256_BLOCK_SIZE
,
1298 .cra_ctxsize
= sizeof(struct sahara_ctx
),
1300 .cra_module
= THIS_MODULE
,
1301 .cra_init
= sahara_sha_cra_init
,
1302 .cra_exit
= sahara_sha_cra_exit
,
1307 static irqreturn_t
sahara_irq_handler(int irq
, void *data
)
1309 struct sahara_dev
*dev
= (struct sahara_dev
*)data
;
1310 unsigned int stat
= sahara_read(dev
, SAHARA_REG_STATUS
);
1311 unsigned int err
= sahara_read(dev
, SAHARA_REG_ERRSTATUS
);
1313 sahara_write(dev
, SAHARA_CMD_CLEAR_INT
| SAHARA_CMD_CLEAR_ERR
,
1316 sahara_decode_status(dev
, stat
);
1318 if (SAHARA_STATUS_GET_STATE(stat
) == SAHARA_STATE_BUSY
) {
1320 } else if (SAHARA_STATUS_GET_STATE(stat
) == SAHARA_STATE_COMPLETE
) {
1323 sahara_decode_error(dev
, err
);
1324 dev
->error
= -EINVAL
;
1327 complete(&dev
->dma_completion
);
1333 static int sahara_register_algs(struct sahara_dev
*dev
)
1336 unsigned int i
, j
, k
, l
;
1338 for (i
= 0; i
< ARRAY_SIZE(aes_algs
); i
++) {
1339 INIT_LIST_HEAD(&aes_algs
[i
].cra_list
);
1340 err
= crypto_register_alg(&aes_algs
[i
]);
1345 for (k
= 0; k
< ARRAY_SIZE(sha_v3_algs
); k
++) {
1346 err
= crypto_register_ahash(&sha_v3_algs
[k
]);
1348 goto err_sha_v3_algs
;
1351 if (dev
->version
> SAHARA_VERSION_3
)
1352 for (l
= 0; l
< ARRAY_SIZE(sha_v4_algs
); l
++) {
1353 err
= crypto_register_ahash(&sha_v4_algs
[l
]);
1355 goto err_sha_v4_algs
;
1361 for (j
= 0; j
< l
; j
++)
1362 crypto_unregister_ahash(&sha_v4_algs
[j
]);
1365 for (j
= 0; j
< k
; j
++)
1366 crypto_unregister_ahash(&sha_v3_algs
[j
]);
1369 for (j
= 0; j
< i
; j
++)
1370 crypto_unregister_alg(&aes_algs
[j
]);
1375 static void sahara_unregister_algs(struct sahara_dev
*dev
)
1379 for (i
= 0; i
< ARRAY_SIZE(aes_algs
); i
++)
1380 crypto_unregister_alg(&aes_algs
[i
]);
1382 for (i
= 0; i
< ARRAY_SIZE(sha_v3_algs
); i
++)
1383 crypto_unregister_ahash(&sha_v3_algs
[i
]);
1385 if (dev
->version
> SAHARA_VERSION_3
)
1386 for (i
= 0; i
< ARRAY_SIZE(sha_v4_algs
); i
++)
1387 crypto_unregister_ahash(&sha_v4_algs
[i
]);
1390 static struct platform_device_id sahara_platform_ids
[] = {
1391 { .name
= "sahara-imx27" },
1394 MODULE_DEVICE_TABLE(platform
, sahara_platform_ids
);
1396 static struct of_device_id sahara_dt_ids
[] = {
1397 { .compatible
= "fsl,imx53-sahara" },
1398 { .compatible
= "fsl,imx27-sahara" },
1401 MODULE_DEVICE_TABLE(of
, sahara_dt_ids
);
1403 static int sahara_probe(struct platform_device
*pdev
)
1405 struct sahara_dev
*dev
;
1406 struct resource
*res
;
1412 dev
= devm_kzalloc(&pdev
->dev
, sizeof(struct sahara_dev
), GFP_KERNEL
);
1414 dev_err(&pdev
->dev
, "unable to alloc data struct.\n");
1418 dev
->device
= &pdev
->dev
;
1419 platform_set_drvdata(pdev
, dev
);
1421 /* Get the base address */
1422 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1423 dev
->regs_base
= devm_ioremap_resource(&pdev
->dev
, res
);
1424 if (IS_ERR(dev
->regs_base
))
1425 return PTR_ERR(dev
->regs_base
);
1428 irq
= platform_get_irq(pdev
, 0);
1430 dev_err(&pdev
->dev
, "failed to get irq resource\n");
1434 err
= devm_request_irq(&pdev
->dev
, irq
, sahara_irq_handler
,
1435 0, dev_name(&pdev
->dev
), dev
);
1437 dev_err(&pdev
->dev
, "failed to request irq\n");
1442 dev
->clk_ipg
= devm_clk_get(&pdev
->dev
, "ipg");
1443 if (IS_ERR(dev
->clk_ipg
)) {
1444 dev_err(&pdev
->dev
, "Could not get ipg clock\n");
1445 return PTR_ERR(dev
->clk_ipg
);
1448 dev
->clk_ahb
= devm_clk_get(&pdev
->dev
, "ahb");
1449 if (IS_ERR(dev
->clk_ahb
)) {
1450 dev_err(&pdev
->dev
, "Could not get ahb clock\n");
1451 return PTR_ERR(dev
->clk_ahb
);
1454 /* Allocate HW descriptors */
1455 dev
->hw_desc
[0] = dmam_alloc_coherent(&pdev
->dev
,
1456 SAHARA_MAX_HW_DESC
* sizeof(struct sahara_hw_desc
),
1457 &dev
->hw_phys_desc
[0], GFP_KERNEL
);
1458 if (!dev
->hw_desc
[0]) {
1459 dev_err(&pdev
->dev
, "Could not allocate hw descriptors\n");
1462 dev
->hw_desc
[1] = dev
->hw_desc
[0] + 1;
1463 dev
->hw_phys_desc
[1] = dev
->hw_phys_desc
[0] +
1464 sizeof(struct sahara_hw_desc
);
1466 /* Allocate space for iv and key */
1467 dev
->key_base
= dmam_alloc_coherent(&pdev
->dev
, 2 * AES_KEYSIZE_128
,
1468 &dev
->key_phys_base
, GFP_KERNEL
);
1469 if (!dev
->key_base
) {
1470 dev_err(&pdev
->dev
, "Could not allocate memory for key\n");
1473 dev
->iv_base
= dev
->key_base
+ AES_KEYSIZE_128
;
1474 dev
->iv_phys_base
= dev
->key_phys_base
+ AES_KEYSIZE_128
;
1476 /* Allocate space for context: largest digest + message length field */
1477 dev
->context_base
= dmam_alloc_coherent(&pdev
->dev
,
1478 SHA256_DIGEST_SIZE
+ 4,
1479 &dev
->context_phys_base
, GFP_KERNEL
);
1480 if (!dev
->context_base
) {
1481 dev_err(&pdev
->dev
, "Could not allocate memory for MDHA context\n");
1485 /* Allocate space for HW links */
1486 dev
->hw_link
[0] = dmam_alloc_coherent(&pdev
->dev
,
1487 SAHARA_MAX_HW_LINK
* sizeof(struct sahara_hw_link
),
1488 &dev
->hw_phys_link
[0], GFP_KERNEL
);
1489 if (!dev
->hw_link
[0]) {
1490 dev_err(&pdev
->dev
, "Could not allocate hw links\n");
1493 for (i
= 1; i
< SAHARA_MAX_HW_LINK
; i
++) {
1494 dev
->hw_phys_link
[i
] = dev
->hw_phys_link
[i
- 1] +
1495 sizeof(struct sahara_hw_link
);
1496 dev
->hw_link
[i
] = dev
->hw_link
[i
- 1] + 1;
1499 crypto_init_queue(&dev
->queue
, SAHARA_QUEUE_LENGTH
);
1501 spin_lock_init(&dev
->lock
);
1502 mutex_init(&dev
->queue_mutex
);
1506 dev
->kthread
= kthread_run(sahara_queue_manage
, dev
, "sahara_crypto");
1507 if (IS_ERR(dev
->kthread
)) {
1508 return PTR_ERR(dev
->kthread
);
1511 init_completion(&dev
->dma_completion
);
1513 err
= clk_prepare_enable(dev
->clk_ipg
);
1516 err
= clk_prepare_enable(dev
->clk_ahb
);
1518 goto clk_ipg_disable
;
1520 version
= sahara_read(dev
, SAHARA_REG_VERSION
);
1521 if (of_device_is_compatible(pdev
->dev
.of_node
, "fsl,imx27-sahara")) {
1522 if (version
!= SAHARA_VERSION_3
)
1524 } else if (of_device_is_compatible(pdev
->dev
.of_node
,
1525 "fsl,imx53-sahara")) {
1526 if (((version
>> 8) & 0xff) != SAHARA_VERSION_4
)
1528 version
= (version
>> 8) & 0xff;
1530 if (err
== -ENODEV
) {
1531 dev_err(&pdev
->dev
, "SAHARA version %d not supported\n",
1536 dev
->version
= version
;
1538 sahara_write(dev
, SAHARA_CMD_RESET
| SAHARA_CMD_MODE_BATCH
,
1540 sahara_write(dev
, SAHARA_CONTROL_SET_THROTTLE(0) |
1541 SAHARA_CONTROL_SET_MAXBURST(8) |
1542 SAHARA_CONTROL_RNG_AUTORSD
|
1543 SAHARA_CONTROL_ENABLE_INT
,
1544 SAHARA_REG_CONTROL
);
1546 err
= sahara_register_algs(dev
);
1550 dev_info(&pdev
->dev
, "SAHARA version %d initialized\n", version
);
1555 kthread_stop(dev
->kthread
);
1557 clk_disable_unprepare(dev
->clk_ahb
);
1559 clk_disable_unprepare(dev
->clk_ipg
);
1564 static int sahara_remove(struct platform_device
*pdev
)
1566 struct sahara_dev
*dev
= platform_get_drvdata(pdev
);
1568 kthread_stop(dev
->kthread
);
1570 sahara_unregister_algs(dev
);
1572 clk_disable_unprepare(dev
->clk_ipg
);
1573 clk_disable_unprepare(dev
->clk_ahb
);
1580 static struct platform_driver sahara_driver
= {
1581 .probe
= sahara_probe
,
1582 .remove
= sahara_remove
,
1584 .name
= SAHARA_NAME
,
1585 .of_match_table
= sahara_dt_ids
,
1587 .id_table
= sahara_platform_ids
,
1590 module_platform_driver(sahara_driver
);
1592 MODULE_LICENSE("GPL");
1593 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1594 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1595 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");