1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Freescale i.MX23/i.MX28 Data Co-Processor driver
5 * Copyright (C) 2013 Marek Vasut <marex@denx.de>
8 #include <linux/dma-mapping.h>
9 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/kthread.h>
13 #include <linux/module.h>
15 #include <linux/platform_device.h>
16 #include <linux/stmp_device.h>
17 #include <linux/clk.h>
18 #include <soc/fsl/dcp.h>
20 #include <crypto/aes.h>
21 #include <crypto/sha1.h>
22 #include <crypto/sha2.h>
23 #include <crypto/internal/hash.h>
24 #include <crypto/internal/skcipher.h>
25 #include <crypto/scatterwalk.h>
27 #define DCP_MAX_CHANS 4
28 #define DCP_BUF_SZ PAGE_SIZE
29 #define DCP_SHA_PAY_SZ 64
31 #define DCP_ALIGNMENT 64
34 * Null hashes to align with hw behavior on imx6sl and ull
35 * these are flipped for consistency with hw output
37 static const uint8_t sha1_null_hash
[] =
38 "\x09\x07\xd8\xaf\x90\x18\x60\x95\xef\xbf"
39 "\x55\x32\x0d\x4b\x6b\x5e\xee\xa3\x39\xda";
41 static const uint8_t sha256_null_hash
[] =
42 "\x55\xb8\x52\x78\x1b\x99\x95\xa4"
43 "\x4c\x93\x9b\x64\xe4\x41\xae\x27"
44 "\x24\xb9\x6f\x99\xc8\xf4\xfb\x9a"
45 "\x14\x1c\xfc\x98\x42\xc4\xb0\xe3";
47 /* DCP DMA descriptor. */
49 uint32_t next_cmd_addr
;
59 /* Coherent aligned block for bounce buffering. */
60 struct dcp_coherent_block
{
61 uint8_t aes_in_buf
[DCP_BUF_SZ
];
62 uint8_t aes_out_buf
[DCP_BUF_SZ
];
63 uint8_t sha_in_buf
[DCP_BUF_SZ
];
64 uint8_t sha_out_buf
[DCP_SHA_PAY_SZ
];
66 uint8_t aes_key
[2 * AES_KEYSIZE_128
];
68 struct dcp_dma_desc desc
[DCP_MAX_CHANS
];
77 struct dcp_coherent_block
*coh
;
79 struct completion completion
[DCP_MAX_CHANS
];
80 spinlock_t lock
[DCP_MAX_CHANS
];
81 struct task_struct
*thread
[DCP_MAX_CHANS
];
82 struct crypto_queue queue
[DCP_MAX_CHANS
];
87 DCP_CHAN_HASH_SHA
= 0,
91 struct dcp_async_ctx
{
96 /* SHA Hash-specific context */
101 /* Crypto-specific context */
102 struct crypto_skcipher
*fallback
;
103 unsigned int key_len
;
104 uint8_t key
[AES_KEYSIZE_128
];
108 struct dcp_aes_req_ctx
{
111 struct skcipher_request fallback_req
; // keep at the end
114 struct dcp_sha_req_ctx
{
119 struct dcp_export_state
{
120 struct dcp_sha_req_ctx req_ctx
;
121 struct dcp_async_ctx async_ctx
;
125 * There can even be only one instance of the MXS DCP due to the
126 * design of Linux Crypto API.
128 static struct dcp
*global_sdcp
;
130 /* DCP register layout. */
131 #define MXS_DCP_CTRL 0x00
132 #define MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES (1 << 23)
133 #define MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING (1 << 22)
135 #define MXS_DCP_STAT 0x10
136 #define MXS_DCP_STAT_CLR 0x18
137 #define MXS_DCP_STAT_IRQ_MASK 0xf
139 #define MXS_DCP_CHANNELCTRL 0x20
140 #define MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK 0xff
142 #define MXS_DCP_CAPABILITY1 0x40
143 #define MXS_DCP_CAPABILITY1_SHA256 (4 << 16)
144 #define MXS_DCP_CAPABILITY1_SHA1 (1 << 16)
145 #define MXS_DCP_CAPABILITY1_AES128 (1 << 0)
147 #define MXS_DCP_CONTEXT 0x50
149 #define MXS_DCP_CH_N_CMDPTR(n) (0x100 + ((n) * 0x40))
151 #define MXS_DCP_CH_N_SEMA(n) (0x110 + ((n) * 0x40))
153 #define MXS_DCP_CH_N_STAT(n) (0x120 + ((n) * 0x40))
154 #define MXS_DCP_CH_N_STAT_CLR(n) (0x128 + ((n) * 0x40))
156 /* DMA descriptor bits. */
157 #define MXS_DCP_CONTROL0_HASH_TERM (1 << 13)
158 #define MXS_DCP_CONTROL0_HASH_INIT (1 << 12)
159 #define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11)
160 #define MXS_DCP_CONTROL0_OTP_KEY (1 << 10)
161 #define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8)
162 #define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9)
163 #define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6)
164 #define MXS_DCP_CONTROL0_ENABLE_CIPHER (1 << 5)
165 #define MXS_DCP_CONTROL0_DECR_SEMAPHORE (1 << 1)
166 #define MXS_DCP_CONTROL0_INTERRUPT (1 << 0)
168 #define MXS_DCP_CONTROL1_HASH_SELECT_SHA256 (2 << 16)
169 #define MXS_DCP_CONTROL1_HASH_SELECT_SHA1 (0 << 16)
170 #define MXS_DCP_CONTROL1_CIPHER_MODE_CBC (1 << 4)
171 #define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4)
172 #define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0)
174 #define MXS_DCP_CONTROL1_KEY_SELECT_SHIFT 8
176 static int mxs_dcp_start_dma(struct dcp_async_ctx
*actx
)
179 struct dcp
*sdcp
= global_sdcp
;
180 const int chan
= actx
->chan
;
183 struct dcp_dma_desc
*desc
= &sdcp
->coh
->desc
[actx
->chan
];
184 dma_addr_t desc_phys
= dma_map_single(sdcp
->dev
, desc
, sizeof(*desc
),
187 dma_err
= dma_mapping_error(sdcp
->dev
, desc_phys
);
191 reinit_completion(&sdcp
->completion
[chan
]);
193 /* Clear status register. */
194 writel(0xffffffff, sdcp
->base
+ MXS_DCP_CH_N_STAT_CLR(chan
));
196 /* Load the DMA descriptor. */
197 writel(desc_phys
, sdcp
->base
+ MXS_DCP_CH_N_CMDPTR(chan
));
199 /* Increment the semaphore to start the DMA transfer. */
200 writel(1, sdcp
->base
+ MXS_DCP_CH_N_SEMA(chan
));
202 ret
= wait_for_completion_timeout(&sdcp
->completion
[chan
],
203 msecs_to_jiffies(1000));
205 dev_err(sdcp
->dev
, "Channel %i timeout (DCP_STAT=0x%08x)\n",
206 chan
, readl(sdcp
->base
+ MXS_DCP_STAT
));
210 stat
= readl(sdcp
->base
+ MXS_DCP_CH_N_STAT(chan
));
212 dev_err(sdcp
->dev
, "Channel %i error (CH_STAT=0x%08x)\n",
217 dma_unmap_single(sdcp
->dev
, desc_phys
, sizeof(*desc
), DMA_TO_DEVICE
);
223 * Encryption (AES128)
225 static int mxs_dcp_run_aes(struct dcp_async_ctx
*actx
,
226 struct skcipher_request
*req
, int init
)
228 dma_addr_t key_phys
, src_phys
, dst_phys
;
229 struct dcp
*sdcp
= global_sdcp
;
230 struct dcp_dma_desc
*desc
= &sdcp
->coh
->desc
[actx
->chan
];
231 struct dcp_aes_req_ctx
*rctx
= skcipher_request_ctx(req
);
232 bool key_referenced
= actx
->key_referenced
;
236 key_phys
= dma_map_single(sdcp
->dev
, sdcp
->coh
->aes_key
+ AES_KEYSIZE_128
,
237 AES_KEYSIZE_128
, DMA_TO_DEVICE
);
239 key_phys
= dma_map_single(sdcp
->dev
, sdcp
->coh
->aes_key
,
240 2 * AES_KEYSIZE_128
, DMA_TO_DEVICE
);
241 ret
= dma_mapping_error(sdcp
->dev
, key_phys
);
245 src_phys
= dma_map_single(sdcp
->dev
, sdcp
->coh
->aes_in_buf
,
246 DCP_BUF_SZ
, DMA_TO_DEVICE
);
247 ret
= dma_mapping_error(sdcp
->dev
, src_phys
);
251 dst_phys
= dma_map_single(sdcp
->dev
, sdcp
->coh
->aes_out_buf
,
252 DCP_BUF_SZ
, DMA_FROM_DEVICE
);
253 ret
= dma_mapping_error(sdcp
->dev
, dst_phys
);
257 if (actx
->fill
% AES_BLOCK_SIZE
) {
258 dev_err(sdcp
->dev
, "Invalid block size!\n");
263 /* Fill in the DMA descriptor. */
264 desc
->control0
= MXS_DCP_CONTROL0_DECR_SEMAPHORE
|
265 MXS_DCP_CONTROL0_INTERRUPT
|
266 MXS_DCP_CONTROL0_ENABLE_CIPHER
;
269 /* Set OTP key bit to select the key via KEY_SELECT. */
270 desc
->control0
|= MXS_DCP_CONTROL0_OTP_KEY
;
272 /* Payload contains the key. */
273 desc
->control0
|= MXS_DCP_CONTROL0_PAYLOAD_KEY
;
276 desc
->control0
|= MXS_DCP_CONTROL0_CIPHER_ENCRYPT
;
278 desc
->control0
|= MXS_DCP_CONTROL0_CIPHER_INIT
;
280 desc
->control1
= MXS_DCP_CONTROL1_CIPHER_SELECT_AES128
;
283 desc
->control1
|= MXS_DCP_CONTROL1_CIPHER_MODE_ECB
;
285 desc
->control1
|= MXS_DCP_CONTROL1_CIPHER_MODE_CBC
;
288 desc
->control1
|= sdcp
->coh
->aes_key
[0] << MXS_DCP_CONTROL1_KEY_SELECT_SHIFT
;
290 desc
->next_cmd_addr
= 0;
291 desc
->source
= src_phys
;
292 desc
->destination
= dst_phys
;
293 desc
->size
= actx
->fill
;
294 desc
->payload
= key_phys
;
297 ret
= mxs_dcp_start_dma(actx
);
300 dma_unmap_single(sdcp
->dev
, dst_phys
, DCP_BUF_SZ
, DMA_FROM_DEVICE
);
302 dma_unmap_single(sdcp
->dev
, src_phys
, DCP_BUF_SZ
, DMA_TO_DEVICE
);
305 dma_unmap_single(sdcp
->dev
, key_phys
, AES_KEYSIZE_128
,
308 dma_unmap_single(sdcp
->dev
, key_phys
, 2 * AES_KEYSIZE_128
,
313 static int mxs_dcp_aes_block_crypt(struct crypto_async_request
*arq
)
315 struct dcp
*sdcp
= global_sdcp
;
317 struct skcipher_request
*req
= skcipher_request_cast(arq
);
318 struct dcp_async_ctx
*actx
= crypto_tfm_ctx(arq
->tfm
);
319 struct dcp_aes_req_ctx
*rctx
= skcipher_request_ctx(req
);
321 struct scatterlist
*dst
= req
->dst
;
322 struct scatterlist
*src
= req
->src
;
323 int dst_nents
= sg_nents(dst
);
325 const int out_off
= DCP_BUF_SZ
;
326 uint8_t *in_buf
= sdcp
->coh
->aes_in_buf
;
327 uint8_t *out_buf
= sdcp
->coh
->aes_out_buf
;
329 uint32_t dst_off
= 0;
330 uint8_t *src_buf
= NULL
;
331 uint32_t last_out_len
= 0;
333 uint8_t *key
= sdcp
->coh
->aes_key
;
336 unsigned int i
, len
, clen
, tlen
= 0;
338 bool limit_hit
= false;
342 /* Copy the key from the temporary location. */
343 memcpy(key
, actx
->key
, actx
->key_len
);
346 /* Copy the CBC IV just past the key. */
347 memcpy(key
+ AES_KEYSIZE_128
, req
->iv
, AES_KEYSIZE_128
);
348 /* CBC needs the INIT set. */
351 memset(key
+ AES_KEYSIZE_128
, 0, AES_KEYSIZE_128
);
354 for_each_sg(req
->src
, src
, sg_nents(req
->src
), i
) {
355 src_buf
= sg_virt(src
);
356 len
= sg_dma_len(src
);
358 limit_hit
= tlen
> req
->cryptlen
;
361 len
= req
->cryptlen
- (tlen
- len
);
364 if (actx
->fill
+ len
> out_off
)
365 clen
= out_off
- actx
->fill
;
369 memcpy(in_buf
+ actx
->fill
, src_buf
, clen
);
375 * If we filled the buffer or this is the last SG,
378 if (actx
->fill
== out_off
|| sg_is_last(src
) ||
380 ret
= mxs_dcp_run_aes(actx
, req
, init
);
385 sg_pcopy_from_buffer(dst
, dst_nents
, out_buf
,
386 actx
->fill
, dst_off
);
387 dst_off
+= actx
->fill
;
388 last_out_len
= actx
->fill
;
397 /* Copy the IV for CBC for chaining */
400 memcpy(req
->iv
, out_buf
+(last_out_len
-AES_BLOCK_SIZE
),
403 memcpy(req
->iv
, in_buf
+(last_out_len
-AES_BLOCK_SIZE
),
410 static int dcp_chan_thread_aes(void *data
)
412 struct dcp
*sdcp
= global_sdcp
;
413 const int chan
= DCP_CHAN_CRYPTO
;
415 struct crypto_async_request
*backlog
;
416 struct crypto_async_request
*arq
;
420 while (!kthread_should_stop()) {
421 set_current_state(TASK_INTERRUPTIBLE
);
423 spin_lock(&sdcp
->lock
[chan
]);
424 backlog
= crypto_get_backlog(&sdcp
->queue
[chan
]);
425 arq
= crypto_dequeue_request(&sdcp
->queue
[chan
]);
426 spin_unlock(&sdcp
->lock
[chan
]);
428 if (!backlog
&& !arq
) {
433 set_current_state(TASK_RUNNING
);
436 crypto_request_complete(backlog
, -EINPROGRESS
);
439 ret
= mxs_dcp_aes_block_crypt(arq
);
440 crypto_request_complete(arq
, ret
);
447 static int mxs_dcp_block_fallback(struct skcipher_request
*req
, int enc
)
449 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
450 struct dcp_aes_req_ctx
*rctx
= skcipher_request_ctx(req
);
451 struct dcp_async_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
454 skcipher_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback
);
455 skcipher_request_set_callback(&rctx
->fallback_req
, req
->base
.flags
,
456 req
->base
.complete
, req
->base
.data
);
457 skcipher_request_set_crypt(&rctx
->fallback_req
, req
->src
, req
->dst
,
458 req
->cryptlen
, req
->iv
);
461 ret
= crypto_skcipher_encrypt(&rctx
->fallback_req
);
463 ret
= crypto_skcipher_decrypt(&rctx
->fallback_req
);
468 static int mxs_dcp_aes_enqueue(struct skcipher_request
*req
, int enc
, int ecb
)
470 struct dcp
*sdcp
= global_sdcp
;
471 struct crypto_async_request
*arq
= &req
->base
;
472 struct dcp_async_ctx
*actx
= crypto_tfm_ctx(arq
->tfm
);
473 struct dcp_aes_req_ctx
*rctx
= skcipher_request_ctx(req
);
476 if (unlikely(actx
->key_len
!= AES_KEYSIZE_128
&& !actx
->key_referenced
))
477 return mxs_dcp_block_fallback(req
, enc
);
481 actx
->chan
= DCP_CHAN_CRYPTO
;
483 spin_lock(&sdcp
->lock
[actx
->chan
]);
484 ret
= crypto_enqueue_request(&sdcp
->queue
[actx
->chan
], &req
->base
);
485 spin_unlock(&sdcp
->lock
[actx
->chan
]);
487 wake_up_process(sdcp
->thread
[actx
->chan
]);
492 static int mxs_dcp_aes_ecb_decrypt(struct skcipher_request
*req
)
494 return mxs_dcp_aes_enqueue(req
, 0, 1);
497 static int mxs_dcp_aes_ecb_encrypt(struct skcipher_request
*req
)
499 return mxs_dcp_aes_enqueue(req
, 1, 1);
502 static int mxs_dcp_aes_cbc_decrypt(struct skcipher_request
*req
)
504 return mxs_dcp_aes_enqueue(req
, 0, 0);
507 static int mxs_dcp_aes_cbc_encrypt(struct skcipher_request
*req
)
509 return mxs_dcp_aes_enqueue(req
, 1, 0);
512 static int mxs_dcp_aes_setkey(struct crypto_skcipher
*tfm
, const u8
*key
,
515 struct dcp_async_ctx
*actx
= crypto_skcipher_ctx(tfm
);
518 * AES 128 is supposed by the hardware, store key into temporary
519 * buffer and exit. We must use the temporary buffer here, since
520 * there can still be an operation in progress.
523 actx
->key_referenced
= false;
524 if (len
== AES_KEYSIZE_128
) {
525 memcpy(actx
->key
, key
, len
);
530 * If the requested AES key size is not supported by the hardware,
531 * but is supported by in-kernel software implementation, we use
534 crypto_skcipher_clear_flags(actx
->fallback
, CRYPTO_TFM_REQ_MASK
);
535 crypto_skcipher_set_flags(actx
->fallback
,
536 tfm
->base
.crt_flags
& CRYPTO_TFM_REQ_MASK
);
537 return crypto_skcipher_setkey(actx
->fallback
, key
, len
);
540 static int mxs_dcp_aes_setrefkey(struct crypto_skcipher
*tfm
, const u8
*key
,
543 struct dcp_async_ctx
*actx
= crypto_skcipher_ctx(tfm
);
545 if (len
!= DCP_PAES_KEYSIZE
)
549 case DCP_PAES_KEY_SLOT0
:
550 case DCP_PAES_KEY_SLOT1
:
551 case DCP_PAES_KEY_SLOT2
:
552 case DCP_PAES_KEY_SLOT3
:
553 case DCP_PAES_KEY_UNIQUE
:
554 case DCP_PAES_KEY_OTP
:
555 memcpy(actx
->key
, key
, len
);
557 actx
->key_referenced
= true;
566 static int mxs_dcp_aes_fallback_init_tfm(struct crypto_skcipher
*tfm
)
568 const char *name
= crypto_tfm_alg_name(crypto_skcipher_tfm(tfm
));
569 struct dcp_async_ctx
*actx
= crypto_skcipher_ctx(tfm
);
570 struct crypto_skcipher
*blk
;
572 blk
= crypto_alloc_skcipher(name
, 0, CRYPTO_ALG_NEED_FALLBACK
);
576 actx
->fallback
= blk
;
577 crypto_skcipher_set_reqsize(tfm
, sizeof(struct dcp_aes_req_ctx
) +
578 crypto_skcipher_reqsize(blk
));
582 static void mxs_dcp_aes_fallback_exit_tfm(struct crypto_skcipher
*tfm
)
584 struct dcp_async_ctx
*actx
= crypto_skcipher_ctx(tfm
);
586 crypto_free_skcipher(actx
->fallback
);
589 static int mxs_dcp_paes_init_tfm(struct crypto_skcipher
*tfm
)
591 crypto_skcipher_set_reqsize(tfm
, sizeof(struct dcp_aes_req_ctx
));
597 * Hashing (SHA1/SHA256)
599 static int mxs_dcp_run_sha(struct ahash_request
*req
)
601 struct dcp
*sdcp
= global_sdcp
;
604 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
605 struct dcp_async_ctx
*actx
= crypto_ahash_ctx(tfm
);
606 struct dcp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
607 struct dcp_dma_desc
*desc
= &sdcp
->coh
->desc
[actx
->chan
];
609 dma_addr_t digest_phys
= 0;
610 dma_addr_t buf_phys
= dma_map_single(sdcp
->dev
, sdcp
->coh
->sha_in_buf
,
611 DCP_BUF_SZ
, DMA_TO_DEVICE
);
613 ret
= dma_mapping_error(sdcp
->dev
, buf_phys
);
617 /* Fill in the DMA descriptor. */
618 desc
->control0
= MXS_DCP_CONTROL0_DECR_SEMAPHORE
|
619 MXS_DCP_CONTROL0_INTERRUPT
|
620 MXS_DCP_CONTROL0_ENABLE_HASH
;
622 desc
->control0
|= MXS_DCP_CONTROL0_HASH_INIT
;
624 desc
->control1
= actx
->alg
;
625 desc
->next_cmd_addr
= 0;
626 desc
->source
= buf_phys
;
627 desc
->destination
= 0;
628 desc
->size
= actx
->fill
;
633 * Align driver with hw behavior when generating null hashes
635 if (rctx
->init
&& rctx
->fini
&& desc
->size
== 0) {
636 struct hash_alg_common
*halg
= crypto_hash_alg_common(tfm
);
637 const uint8_t *sha_buf
=
638 (actx
->alg
== MXS_DCP_CONTROL1_HASH_SELECT_SHA1
) ?
639 sha1_null_hash
: sha256_null_hash
;
640 memcpy(sdcp
->coh
->sha_out_buf
, sha_buf
, halg
->digestsize
);
645 /* Set HASH_TERM bit for last transfer block. */
647 digest_phys
= dma_map_single(sdcp
->dev
, sdcp
->coh
->sha_out_buf
,
648 DCP_SHA_PAY_SZ
, DMA_FROM_DEVICE
);
649 ret
= dma_mapping_error(sdcp
->dev
, digest_phys
);
653 desc
->control0
|= MXS_DCP_CONTROL0_HASH_TERM
;
654 desc
->payload
= digest_phys
;
657 ret
= mxs_dcp_start_dma(actx
);
660 dma_unmap_single(sdcp
->dev
, digest_phys
, DCP_SHA_PAY_SZ
,
664 dma_unmap_single(sdcp
->dev
, buf_phys
, DCP_BUF_SZ
, DMA_TO_DEVICE
);
669 static int dcp_sha_req_to_buf(struct crypto_async_request
*arq
)
671 struct dcp
*sdcp
= global_sdcp
;
673 struct ahash_request
*req
= ahash_request_cast(arq
);
674 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
675 struct dcp_async_ctx
*actx
= crypto_ahash_ctx(tfm
);
676 struct dcp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
677 struct hash_alg_common
*halg
= crypto_hash_alg_common(tfm
);
679 uint8_t *in_buf
= sdcp
->coh
->sha_in_buf
;
680 uint8_t *out_buf
= sdcp
->coh
->sha_out_buf
;
682 struct scatterlist
*src
;
684 unsigned int i
, len
, clen
, oft
= 0;
687 int fin
= rctx
->fini
;
695 if (actx
->fill
+ len
> DCP_BUF_SZ
)
696 clen
= DCP_BUF_SZ
- actx
->fill
;
700 scatterwalk_map_and_copy(in_buf
+ actx
->fill
, src
, oft
, clen
,
708 * If we filled the buffer and still have some
709 * more data, submit the buffer.
711 if (len
&& actx
->fill
== DCP_BUF_SZ
) {
712 ret
= mxs_dcp_run_sha(req
);
723 /* Submit whatever is left. */
727 ret
= mxs_dcp_run_sha(req
);
733 /* For some reason the result is flipped */
734 for (i
= 0; i
< halg
->digestsize
; i
++)
735 req
->result
[i
] = out_buf
[halg
->digestsize
- i
- 1];
741 static int dcp_chan_thread_sha(void *data
)
743 struct dcp
*sdcp
= global_sdcp
;
744 const int chan
= DCP_CHAN_HASH_SHA
;
746 struct crypto_async_request
*backlog
;
747 struct crypto_async_request
*arq
;
750 while (!kthread_should_stop()) {
751 set_current_state(TASK_INTERRUPTIBLE
);
753 spin_lock(&sdcp
->lock
[chan
]);
754 backlog
= crypto_get_backlog(&sdcp
->queue
[chan
]);
755 arq
= crypto_dequeue_request(&sdcp
->queue
[chan
]);
756 spin_unlock(&sdcp
->lock
[chan
]);
758 if (!backlog
&& !arq
) {
763 set_current_state(TASK_RUNNING
);
766 crypto_request_complete(backlog
, -EINPROGRESS
);
769 ret
= dcp_sha_req_to_buf(arq
);
770 crypto_request_complete(arq
, ret
);
777 static int dcp_sha_init(struct ahash_request
*req
)
779 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
780 struct dcp_async_ctx
*actx
= crypto_ahash_ctx(tfm
);
782 struct hash_alg_common
*halg
= crypto_hash_alg_common(tfm
);
785 * Start hashing session. The code below only inits the
786 * hashing session context, nothing more.
788 memset(actx
, 0, sizeof(*actx
));
790 if (strcmp(halg
->base
.cra_name
, "sha1") == 0)
791 actx
->alg
= MXS_DCP_CONTROL1_HASH_SELECT_SHA1
;
793 actx
->alg
= MXS_DCP_CONTROL1_HASH_SELECT_SHA256
;
797 actx
->chan
= DCP_CHAN_HASH_SHA
;
799 mutex_init(&actx
->mutex
);
804 static int dcp_sha_update_fx(struct ahash_request
*req
, int fini
)
806 struct dcp
*sdcp
= global_sdcp
;
808 struct dcp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
809 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
810 struct dcp_async_ctx
*actx
= crypto_ahash_ctx(tfm
);
815 * Ignore requests that have no data in them and are not
816 * the trailing requests in the stream of requests.
818 if (!req
->nbytes
&& !fini
)
821 mutex_lock(&actx
->mutex
);
830 spin_lock(&sdcp
->lock
[actx
->chan
]);
831 ret
= crypto_enqueue_request(&sdcp
->queue
[actx
->chan
], &req
->base
);
832 spin_unlock(&sdcp
->lock
[actx
->chan
]);
834 wake_up_process(sdcp
->thread
[actx
->chan
]);
835 mutex_unlock(&actx
->mutex
);
840 static int dcp_sha_update(struct ahash_request
*req
)
842 return dcp_sha_update_fx(req
, 0);
845 static int dcp_sha_final(struct ahash_request
*req
)
847 ahash_request_set_crypt(req
, NULL
, req
->result
, 0);
849 return dcp_sha_update_fx(req
, 1);
852 static int dcp_sha_finup(struct ahash_request
*req
)
854 return dcp_sha_update_fx(req
, 1);
857 static int dcp_sha_digest(struct ahash_request
*req
)
861 ret
= dcp_sha_init(req
);
865 return dcp_sha_finup(req
);
868 static int dcp_sha_import(struct ahash_request
*req
, const void *in
)
870 struct dcp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
871 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
872 struct dcp_async_ctx
*actx
= crypto_ahash_ctx(tfm
);
873 const struct dcp_export_state
*export
= in
;
875 memset(rctx
, 0, sizeof(struct dcp_sha_req_ctx
));
876 memset(actx
, 0, sizeof(struct dcp_async_ctx
));
877 memcpy(rctx
, &export
->req_ctx
, sizeof(struct dcp_sha_req_ctx
));
878 memcpy(actx
, &export
->async_ctx
, sizeof(struct dcp_async_ctx
));
883 static int dcp_sha_export(struct ahash_request
*req
, void *out
)
885 struct dcp_sha_req_ctx
*rctx_state
= ahash_request_ctx(req
);
886 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
887 struct dcp_async_ctx
*actx_state
= crypto_ahash_ctx(tfm
);
888 struct dcp_export_state
*export
= out
;
890 memcpy(&export
->req_ctx
, rctx_state
, sizeof(struct dcp_sha_req_ctx
));
891 memcpy(&export
->async_ctx
, actx_state
, sizeof(struct dcp_async_ctx
));
896 static int dcp_sha_cra_init(struct crypto_tfm
*tfm
)
898 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
899 sizeof(struct dcp_sha_req_ctx
));
903 static void dcp_sha_cra_exit(struct crypto_tfm
*tfm
)
907 /* AES 128 ECB and AES 128 CBC */
908 static struct skcipher_alg dcp_aes_algs
[] = {
910 .base
.cra_name
= "ecb(aes)",
911 .base
.cra_driver_name
= "ecb-aes-dcp",
912 .base
.cra_priority
= 400,
913 .base
.cra_alignmask
= 15,
914 .base
.cra_flags
= CRYPTO_ALG_ASYNC
|
915 CRYPTO_ALG_NEED_FALLBACK
,
916 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
917 .base
.cra_ctxsize
= sizeof(struct dcp_async_ctx
),
918 .base
.cra_module
= THIS_MODULE
,
920 .min_keysize
= AES_MIN_KEY_SIZE
,
921 .max_keysize
= AES_MAX_KEY_SIZE
,
922 .setkey
= mxs_dcp_aes_setkey
,
923 .encrypt
= mxs_dcp_aes_ecb_encrypt
,
924 .decrypt
= mxs_dcp_aes_ecb_decrypt
,
925 .init
= mxs_dcp_aes_fallback_init_tfm
,
926 .exit
= mxs_dcp_aes_fallback_exit_tfm
,
928 .base
.cra_name
= "cbc(aes)",
929 .base
.cra_driver_name
= "cbc-aes-dcp",
930 .base
.cra_priority
= 400,
931 .base
.cra_alignmask
= 15,
932 .base
.cra_flags
= CRYPTO_ALG_ASYNC
|
933 CRYPTO_ALG_NEED_FALLBACK
,
934 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
935 .base
.cra_ctxsize
= sizeof(struct dcp_async_ctx
),
936 .base
.cra_module
= THIS_MODULE
,
938 .min_keysize
= AES_MIN_KEY_SIZE
,
939 .max_keysize
= AES_MAX_KEY_SIZE
,
940 .setkey
= mxs_dcp_aes_setkey
,
941 .encrypt
= mxs_dcp_aes_cbc_encrypt
,
942 .decrypt
= mxs_dcp_aes_cbc_decrypt
,
943 .ivsize
= AES_BLOCK_SIZE
,
944 .init
= mxs_dcp_aes_fallback_init_tfm
,
945 .exit
= mxs_dcp_aes_fallback_exit_tfm
,
947 .base
.cra_name
= "ecb(paes)",
948 .base
.cra_driver_name
= "ecb-paes-dcp",
949 .base
.cra_priority
= 401,
950 .base
.cra_alignmask
= 15,
951 .base
.cra_flags
= CRYPTO_ALG_ASYNC
| CRYPTO_ALG_INTERNAL
,
952 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
953 .base
.cra_ctxsize
= sizeof(struct dcp_async_ctx
),
954 .base
.cra_module
= THIS_MODULE
,
956 .min_keysize
= DCP_PAES_KEYSIZE
,
957 .max_keysize
= DCP_PAES_KEYSIZE
,
958 .setkey
= mxs_dcp_aes_setrefkey
,
959 .encrypt
= mxs_dcp_aes_ecb_encrypt
,
960 .decrypt
= mxs_dcp_aes_ecb_decrypt
,
961 .init
= mxs_dcp_paes_init_tfm
,
963 .base
.cra_name
= "cbc(paes)",
964 .base
.cra_driver_name
= "cbc-paes-dcp",
965 .base
.cra_priority
= 401,
966 .base
.cra_alignmask
= 15,
967 .base
.cra_flags
= CRYPTO_ALG_ASYNC
| CRYPTO_ALG_INTERNAL
,
968 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
969 .base
.cra_ctxsize
= sizeof(struct dcp_async_ctx
),
970 .base
.cra_module
= THIS_MODULE
,
972 .min_keysize
= DCP_PAES_KEYSIZE
,
973 .max_keysize
= DCP_PAES_KEYSIZE
,
974 .setkey
= mxs_dcp_aes_setrefkey
,
975 .encrypt
= mxs_dcp_aes_cbc_encrypt
,
976 .decrypt
= mxs_dcp_aes_cbc_decrypt
,
977 .ivsize
= AES_BLOCK_SIZE
,
978 .init
= mxs_dcp_paes_init_tfm
,
983 static struct ahash_alg dcp_sha1_alg
= {
984 .init
= dcp_sha_init
,
985 .update
= dcp_sha_update
,
986 .final
= dcp_sha_final
,
987 .finup
= dcp_sha_finup
,
988 .digest
= dcp_sha_digest
,
989 .import
= dcp_sha_import
,
990 .export
= dcp_sha_export
,
992 .digestsize
= SHA1_DIGEST_SIZE
,
993 .statesize
= sizeof(struct dcp_export_state
),
996 .cra_driver_name
= "sha1-dcp",
998 .cra_flags
= CRYPTO_ALG_ASYNC
,
999 .cra_blocksize
= SHA1_BLOCK_SIZE
,
1000 .cra_ctxsize
= sizeof(struct dcp_async_ctx
),
1001 .cra_module
= THIS_MODULE
,
1002 .cra_init
= dcp_sha_cra_init
,
1003 .cra_exit
= dcp_sha_cra_exit
,
1009 static struct ahash_alg dcp_sha256_alg
= {
1010 .init
= dcp_sha_init
,
1011 .update
= dcp_sha_update
,
1012 .final
= dcp_sha_final
,
1013 .finup
= dcp_sha_finup
,
1014 .digest
= dcp_sha_digest
,
1015 .import
= dcp_sha_import
,
1016 .export
= dcp_sha_export
,
1018 .digestsize
= SHA256_DIGEST_SIZE
,
1019 .statesize
= sizeof(struct dcp_export_state
),
1021 .cra_name
= "sha256",
1022 .cra_driver_name
= "sha256-dcp",
1023 .cra_priority
= 400,
1024 .cra_flags
= CRYPTO_ALG_ASYNC
,
1025 .cra_blocksize
= SHA256_BLOCK_SIZE
,
1026 .cra_ctxsize
= sizeof(struct dcp_async_ctx
),
1027 .cra_module
= THIS_MODULE
,
1028 .cra_init
= dcp_sha_cra_init
,
1029 .cra_exit
= dcp_sha_cra_exit
,
1034 static irqreturn_t
mxs_dcp_irq(int irq
, void *context
)
1036 struct dcp
*sdcp
= context
;
1040 stat
= readl(sdcp
->base
+ MXS_DCP_STAT
);
1041 stat
&= MXS_DCP_STAT_IRQ_MASK
;
1045 /* Clear the interrupts. */
1046 writel(stat
, sdcp
->base
+ MXS_DCP_STAT_CLR
);
1048 /* Complete the DMA requests that finished. */
1049 for (i
= 0; i
< DCP_MAX_CHANS
; i
++)
1050 if (stat
& (1 << i
))
1051 complete(&sdcp
->completion
[i
]);
1056 static int mxs_dcp_probe(struct platform_device
*pdev
)
1058 struct device
*dev
= &pdev
->dev
;
1059 struct dcp
*sdcp
= NULL
;
1061 int dcp_vmi_irq
, dcp_irq
;
1064 dev_err(dev
, "Only one DCP instance allowed!\n");
1068 dcp_vmi_irq
= platform_get_irq(pdev
, 0);
1069 if (dcp_vmi_irq
< 0)
1072 dcp_irq
= platform_get_irq(pdev
, 1);
1076 sdcp
= devm_kzalloc(dev
, sizeof(*sdcp
), GFP_KERNEL
);
1081 sdcp
->base
= devm_platform_ioremap_resource(pdev
, 0);
1082 if (IS_ERR(sdcp
->base
))
1083 return PTR_ERR(sdcp
->base
);
1086 ret
= devm_request_irq(dev
, dcp_vmi_irq
, mxs_dcp_irq
, 0,
1087 "dcp-vmi-irq", sdcp
);
1089 dev_err(dev
, "Failed to claim DCP VMI IRQ!\n");
1093 ret
= devm_request_irq(dev
, dcp_irq
, mxs_dcp_irq
, 0,
1096 dev_err(dev
, "Failed to claim DCP IRQ!\n");
1100 /* Allocate coherent helper block. */
1101 sdcp
->coh
= devm_kzalloc(dev
, sizeof(*sdcp
->coh
) + DCP_ALIGNMENT
,
1106 /* Re-align the structure so it fits the DCP constraints. */
1107 sdcp
->coh
= PTR_ALIGN(sdcp
->coh
, DCP_ALIGNMENT
);
1109 /* DCP clock is optional, only used on some SOCs */
1110 sdcp
->dcp_clk
= devm_clk_get_optional_enabled(dev
, "dcp");
1111 if (IS_ERR(sdcp
->dcp_clk
))
1112 return PTR_ERR(sdcp
->dcp_clk
);
1114 /* Restart the DCP block. */
1115 ret
= stmp_reset_block(sdcp
->base
);
1117 dev_err(dev
, "Failed reset\n");
1121 /* Initialize control register. */
1122 writel(MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES
|
1123 MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING
| 0xf,
1124 sdcp
->base
+ MXS_DCP_CTRL
);
1126 /* Enable all DCP DMA channels. */
1127 writel(MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK
,
1128 sdcp
->base
+ MXS_DCP_CHANNELCTRL
);
1131 * We do not enable context switching. Give the context buffer a
1132 * pointer to an illegal address so if context switching is
1133 * inadvertantly enabled, the DCP will return an error instead of
1134 * trashing good memory. The DCP DMA cannot access ROM, so any ROM
1137 writel(0xffff0000, sdcp
->base
+ MXS_DCP_CONTEXT
);
1138 for (i
= 0; i
< DCP_MAX_CHANS
; i
++)
1139 writel(0xffffffff, sdcp
->base
+ MXS_DCP_CH_N_STAT_CLR(i
));
1140 writel(0xffffffff, sdcp
->base
+ MXS_DCP_STAT_CLR
);
1144 platform_set_drvdata(pdev
, sdcp
);
1146 for (i
= 0; i
< DCP_MAX_CHANS
; i
++) {
1147 spin_lock_init(&sdcp
->lock
[i
]);
1148 init_completion(&sdcp
->completion
[i
]);
1149 crypto_init_queue(&sdcp
->queue
[i
], 50);
1152 /* Create the SHA and AES handler threads. */
1153 sdcp
->thread
[DCP_CHAN_HASH_SHA
] = kthread_run(dcp_chan_thread_sha
,
1154 NULL
, "mxs_dcp_chan/sha");
1155 if (IS_ERR(sdcp
->thread
[DCP_CHAN_HASH_SHA
])) {
1156 dev_err(dev
, "Error starting SHA thread!\n");
1157 ret
= PTR_ERR(sdcp
->thread
[DCP_CHAN_HASH_SHA
]);
1161 sdcp
->thread
[DCP_CHAN_CRYPTO
] = kthread_run(dcp_chan_thread_aes
,
1162 NULL
, "mxs_dcp_chan/aes");
1163 if (IS_ERR(sdcp
->thread
[DCP_CHAN_CRYPTO
])) {
1164 dev_err(dev
, "Error starting SHA thread!\n");
1165 ret
= PTR_ERR(sdcp
->thread
[DCP_CHAN_CRYPTO
]);
1166 goto err_destroy_sha_thread
;
1169 /* Register the various crypto algorithms. */
1170 sdcp
->caps
= readl(sdcp
->base
+ MXS_DCP_CAPABILITY1
);
1172 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_AES128
) {
1173 ret
= crypto_register_skciphers(dcp_aes_algs
,
1174 ARRAY_SIZE(dcp_aes_algs
));
1176 /* Failed to register algorithm. */
1177 dev_err(dev
, "Failed to register AES crypto!\n");
1178 goto err_destroy_aes_thread
;
1182 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_SHA1
) {
1183 ret
= crypto_register_ahash(&dcp_sha1_alg
);
1185 dev_err(dev
, "Failed to register %s hash!\n",
1186 dcp_sha1_alg
.halg
.base
.cra_name
);
1187 goto err_unregister_aes
;
1191 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_SHA256
) {
1192 ret
= crypto_register_ahash(&dcp_sha256_alg
);
1194 dev_err(dev
, "Failed to register %s hash!\n",
1195 dcp_sha256_alg
.halg
.base
.cra_name
);
1196 goto err_unregister_sha1
;
1202 err_unregister_sha1
:
1203 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_SHA1
)
1204 crypto_unregister_ahash(&dcp_sha1_alg
);
1207 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_AES128
)
1208 crypto_unregister_skciphers(dcp_aes_algs
, ARRAY_SIZE(dcp_aes_algs
));
1210 err_destroy_aes_thread
:
1211 kthread_stop(sdcp
->thread
[DCP_CHAN_CRYPTO
]);
1213 err_destroy_sha_thread
:
1214 kthread_stop(sdcp
->thread
[DCP_CHAN_HASH_SHA
]);
1219 static void mxs_dcp_remove(struct platform_device
*pdev
)
1221 struct dcp
*sdcp
= platform_get_drvdata(pdev
);
1223 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_SHA256
)
1224 crypto_unregister_ahash(&dcp_sha256_alg
);
1226 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_SHA1
)
1227 crypto_unregister_ahash(&dcp_sha1_alg
);
1229 if (sdcp
->caps
& MXS_DCP_CAPABILITY1_AES128
)
1230 crypto_unregister_skciphers(dcp_aes_algs
, ARRAY_SIZE(dcp_aes_algs
));
1232 kthread_stop(sdcp
->thread
[DCP_CHAN_HASH_SHA
]);
1233 kthread_stop(sdcp
->thread
[DCP_CHAN_CRYPTO
]);
1235 platform_set_drvdata(pdev
, NULL
);
1240 static const struct of_device_id mxs_dcp_dt_ids
[] = {
1241 { .compatible
= "fsl,imx23-dcp", .data
= NULL
, },
1242 { .compatible
= "fsl,imx28-dcp", .data
= NULL
, },
1246 MODULE_DEVICE_TABLE(of
, mxs_dcp_dt_ids
);
1248 static struct platform_driver mxs_dcp_driver
= {
1249 .probe
= mxs_dcp_probe
,
1250 .remove
= mxs_dcp_remove
,
1253 .of_match_table
= mxs_dcp_dt_ids
,
1257 module_platform_driver(mxs_dcp_driver
);
1259 MODULE_AUTHOR("Marek Vasut <marex@denx.de>");
1260 MODULE_DESCRIPTION("Freescale MXS DCP Driver");
1261 MODULE_LICENSE("GPL");
1262 MODULE_ALIAS("platform:mxs-dcp");