?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
PM / sleep: Asynchronous threads for suspend_noirq
[linux/fpc-iii.git] / drivers / crypto / mxs-dcp.c
bloba6db7fa6f891acb89c0d2c3d06e8ae389aa903f1
1 /*
2 * Freescale i.MX23/i.MX28 Data Co-Processor driver
3 *
4 * Copyright (C) 2013 Marek Vasut <marex@denx.de>
5 *
6 * The code contained herein is licensed under the GNU General Public
7 * License. You may obtain a copy of the GNU General Public License
8 * Version 2 or later at the following locations:
9 *
10 * http://www.opensource.org/licenses/gpl-license.html
11 * http://www.gnu.org/copyleft/gpl.html
12 */
13
14 #include <linux/crypto.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/interrupt.h>
17 #include <linux/io.h>
18 #include <linux/kernel.h>
19 #include <linux/kthread.h>
20 #include <linux/module.h>
21 #include <linux/of.h>
22 #include <linux/platform_device.h>
23 #include <linux/stmp_device.h>
24
25 #include <crypto/aes.h>
26 #include <crypto/sha.h>
27 #include <crypto/internal/hash.h>
28
29 #define DCP_MAX_CHANS 4
30 #define DCP_BUF_SZ PAGE_SIZE
31
32 /* DCP DMA descriptor. */
33 struct dcp_dma_desc {
34 uint32_t next_cmd_addr;
35 uint32_t control0;
36 uint32_t control1;
37 uint32_t source;
38 uint32_t destination;
39 uint32_t size;
40 uint32_t payload;
41 uint32_t status;
42 };
43
44 /* Coherent aligned block for bounce buffering. */
45 struct dcp_coherent_block {
46 uint8_t aes_in_buf[DCP_BUF_SZ];
47 uint8_t aes_out_buf[DCP_BUF_SZ];
48 uint8_t sha_in_buf[DCP_BUF_SZ];
49
50 uint8_t aes_key[2 * AES_KEYSIZE_128];
51 uint8_t sha_digest[SHA256_DIGEST_SIZE];
52
53 struct dcp_dma_desc desc[DCP_MAX_CHANS];
54 };
55
56 struct dcp {
57 struct device *dev;
58 void __iomem *base;
59
60 uint32_t caps;
61
62 struct dcp_coherent_block *coh;
63
64 struct completion completion[DCP_MAX_CHANS];
65 struct mutex mutex[DCP_MAX_CHANS];
66 struct task_struct *thread[DCP_MAX_CHANS];
67 struct crypto_queue queue[DCP_MAX_CHANS];
68 };
69
70 enum dcp_chan {
71 DCP_CHAN_HASH_SHA = 0,
72 DCP_CHAN_CRYPTO = 2,
73 };
74
75 struct dcp_async_ctx {
76 /* Common context */
77 enum dcp_chan chan;
78 uint32_t fill;
79
80 /* SHA Hash-specific context */
81 struct mutex mutex;
82 uint32_t alg;
83 unsigned int hot:1;
84
85 /* Crypto-specific context */
86 unsigned int enc:1;
87 unsigned int ecb:1;
88 struct crypto_ablkcipher *fallback;
89 unsigned int key_len;
90 uint8_t key[AES_KEYSIZE_128];
91 };
92
93 struct dcp_sha_req_ctx {
94 unsigned int init:1;
95 unsigned int fini:1;
96 };
97
98 /*
99 * There can even be only one instance of the MXS DCP due to the
100 * design of Linux Crypto API.
101 */
102 static struct dcp *global_sdcp;
103 static DEFINE_MUTEX(global_mutex);
104
105 /* DCP register layout. */
106 #define MXS_DCP_CTRL 0x00
107 #define MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES (1 << 23)
108 #define MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING (1 << 22)
109
110 #define MXS_DCP_STAT 0x10
111 #define MXS_DCP_STAT_CLR 0x18
112 #define MXS_DCP_STAT_IRQ_MASK 0xf
113
114 #define MXS_DCP_CHANNELCTRL 0x20
115 #define MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK 0xff
116
117 #define MXS_DCP_CAPABILITY1 0x40
118 #define MXS_DCP_CAPABILITY1_SHA256 (4 << 16)
119 #define MXS_DCP_CAPABILITY1_SHA1 (1 << 16)
120 #define MXS_DCP_CAPABILITY1_AES128 (1 << 0)
121
122 #define MXS_DCP_CONTEXT 0x50
123
124 #define MXS_DCP_CH_N_CMDPTR(n) (0x100 + ((n) * 0x40))
125
126 #define MXS_DCP_CH_N_SEMA(n) (0x110 + ((n) * 0x40))
127
128 #define MXS_DCP_CH_N_STAT(n) (0x120 + ((n) * 0x40))
129 #define MXS_DCP_CH_N_STAT_CLR(n) (0x128 + ((n) * 0x40))
130
131 /* DMA descriptor bits. */
132 #define MXS_DCP_CONTROL0_HASH_TERM (1 << 13)
133 #define MXS_DCP_CONTROL0_HASH_INIT (1 << 12)
134 #define MXS_DCP_CONTROL0_PAYLOAD_KEY (1 << 11)
135 #define MXS_DCP_CONTROL0_CIPHER_ENCRYPT (1 << 8)
136 #define MXS_DCP_CONTROL0_CIPHER_INIT (1 << 9)
137 #define MXS_DCP_CONTROL0_ENABLE_HASH (1 << 6)
138 #define MXS_DCP_CONTROL0_ENABLE_CIPHER (1 << 5)
139 #define MXS_DCP_CONTROL0_DECR_SEMAPHORE (1 << 1)
140 #define MXS_DCP_CONTROL0_INTERRUPT (1 << 0)
141
142 #define MXS_DCP_CONTROL1_HASH_SELECT_SHA256 (2 << 16)
143 #define MXS_DCP_CONTROL1_HASH_SELECT_SHA1 (0 << 16)
144 #define MXS_DCP_CONTROL1_CIPHER_MODE_CBC (1 << 4)
145 #define MXS_DCP_CONTROL1_CIPHER_MODE_ECB (0 << 4)
146 #define MXS_DCP_CONTROL1_CIPHER_SELECT_AES128 (0 << 0)
147
148 static int mxs_dcp_start_dma(struct dcp_async_ctx *actx)
149 {
150 struct dcp *sdcp = global_sdcp;
151 const int chan = actx->chan;
152 uint32_t stat;
153 int ret;
154 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan];
155
156 dma_addr_t desc_phys = dma_map_single(sdcp->dev, desc, sizeof(*desc),
157 DMA_TO_DEVICE);
158
159 reinit_completion(&sdcp->completion[chan]);
160
161 /* Clear status register. */
162 writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(chan));
163
164 /* Load the DMA descriptor. */
165 writel(desc_phys, sdcp->base + MXS_DCP_CH_N_CMDPTR(chan));
166
167 /* Increment the semaphore to start the DMA transfer. */
168 writel(1, sdcp->base + MXS_DCP_CH_N_SEMA(chan));
169
170 ret = wait_for_completion_timeout(&sdcp->completion[chan],
171 msecs_to_jiffies(1000));
172 if (!ret) {
173 dev_err(sdcp->dev, "Channel %i timeout (DCP_STAT=0x%08x)\n",
174 chan, readl(sdcp->base + MXS_DCP_STAT));
175 return -ETIMEDOUT;
176 }
177
178 stat = readl(sdcp->base + MXS_DCP_CH_N_STAT(chan));
179 if (stat & 0xff) {
180 dev_err(sdcp->dev, "Channel %i error (CH_STAT=0x%08x)\n",
181 chan, stat);
182 return -EINVAL;
183 }
184
185 dma_unmap_single(sdcp->dev, desc_phys, sizeof(*desc), DMA_TO_DEVICE);
186
187 return 0;
188 }
189
190 /*
191 * Encryption (AES128)
192 */
193 static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, int init)
194 {
195 struct dcp *sdcp = global_sdcp;
196 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan];
197 int ret;
198
199 dma_addr_t key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key,
200 2 * AES_KEYSIZE_128,
201 DMA_TO_DEVICE);
202 dma_addr_t src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf,
203 DCP_BUF_SZ, DMA_TO_DEVICE);
204 dma_addr_t dst_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_out_buf,
205 DCP_BUF_SZ, DMA_FROM_DEVICE);
206
207 /* Fill in the DMA descriptor. */
208 desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE |
209 MXS_DCP_CONTROL0_INTERRUPT |
210 MXS_DCP_CONTROL0_ENABLE_CIPHER;
211
212 /* Payload contains the key. */
213 desc->control0 |= MXS_DCP_CONTROL0_PAYLOAD_KEY;
214
215 if (actx->enc)
216 desc->control0 |= MXS_DCP_CONTROL0_CIPHER_ENCRYPT;
217 if (init)
218 desc->control0 |= MXS_DCP_CONTROL0_CIPHER_INIT;
219
220 desc->control1 = MXS_DCP_CONTROL1_CIPHER_SELECT_AES128;
221
222 if (actx->ecb)
223 desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_ECB;
224 else
225 desc->control1 |= MXS_DCP_CONTROL1_CIPHER_MODE_CBC;
226
227 desc->next_cmd_addr = 0;
228 desc->source = src_phys;
229 desc->destination = dst_phys;
230 desc->size = actx->fill;
231 desc->payload = key_phys;
232 desc->status = 0;
233
234 ret = mxs_dcp_start_dma(actx);
235
236 dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128,
237 DMA_TO_DEVICE);
238 dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE);
239 dma_unmap_single(sdcp->dev, dst_phys, DCP_BUF_SZ, DMA_FROM_DEVICE);
240
241 return ret;
242 }
243
244 static int mxs_dcp_aes_block_crypt(struct crypto_async_request *arq)
245 {
246 struct dcp *sdcp = global_sdcp;
247
248 struct ablkcipher_request *req = ablkcipher_request_cast(arq);
249 struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm);
250
251 struct scatterlist *dst = req->dst;
252 struct scatterlist *src = req->src;
253 const int nents = sg_nents(req->src);
254
255 const int out_off = DCP_BUF_SZ;
256 uint8_t *in_buf = sdcp->coh->aes_in_buf;
257 uint8_t *out_buf = sdcp->coh->aes_out_buf;
258
259 uint8_t *out_tmp, *src_buf, *dst_buf = NULL;
260 uint32_t dst_off = 0;
261
262 uint8_t *key = sdcp->coh->aes_key;
263
264 int ret = 0;
265 int split = 0;
266 unsigned int i, len, clen, rem = 0;
267 int init = 0;
268
269 actx->fill = 0;
270
271 /* Copy the key from the temporary location. */
272 memcpy(key, actx->key, actx->key_len);
273
274 if (!actx->ecb) {
275 /* Copy the CBC IV just past the key. */
276 memcpy(key + AES_KEYSIZE_128, req->info, AES_KEYSIZE_128);
277 /* CBC needs the INIT set. */
278 init = 1;
279 } else {
280 memset(key + AES_KEYSIZE_128, 0, AES_KEYSIZE_128);
281 }
282
283 for_each_sg(req->src, src, nents, i) {
284 src_buf = sg_virt(src);
285 len = sg_dma_len(src);
286
287 do {
288 if (actx->fill + len > out_off)
289 clen = out_off - actx->fill;
290 else
291 clen = len;
292
293 memcpy(in_buf + actx->fill, src_buf, clen);
294 len -= clen;
295 src_buf += clen;
296 actx->fill += clen;
297
298 /*
299 * If we filled the buffer or this is the last SG,
300 * submit the buffer.
301 */
302 if (actx->fill == out_off || sg_is_last(src)) {
303 ret = mxs_dcp_run_aes(actx, init);
304 if (ret)
305 return ret;
306 init = 0;
307
308 out_tmp = out_buf;
309 while (dst && actx->fill) {
310 if (!split) {
311 dst_buf = sg_virt(dst);
312 dst_off = 0;
313 }
314 rem = min(sg_dma_len(dst) - dst_off,
315 actx->fill);
316
317 memcpy(dst_buf + dst_off, out_tmp, rem);
318 out_tmp += rem;
319 dst_off += rem;
320 actx->fill -= rem;
321
322 if (dst_off == sg_dma_len(dst)) {
323 dst = sg_next(dst);
324 split = 0;
325 } else {
326 split = 1;
327 }
328 }
329 }
330 } while (len);
331 }
332
333 return ret;
334 }
335
336 static int dcp_chan_thread_aes(void *data)
337 {
338 struct dcp *sdcp = global_sdcp;
339 const int chan = DCP_CHAN_CRYPTO;
340
341 struct crypto_async_request *backlog;
342 struct crypto_async_request *arq;
343
344 int ret;
345
346 do {
347 __set_current_state(TASK_INTERRUPTIBLE);
348
349 mutex_lock(&sdcp->mutex[chan]);
350 backlog = crypto_get_backlog(&sdcp->queue[chan]);
351 arq = crypto_dequeue_request(&sdcp->queue[chan]);
352 mutex_unlock(&sdcp->mutex[chan]);
353
354 if (backlog)
355 backlog->complete(backlog, -EINPROGRESS);
356
357 if (arq) {
358 ret = mxs_dcp_aes_block_crypt(arq);
359 arq->complete(arq, ret);
360 continue;
361 }
362
363 schedule();
364 } while (!kthread_should_stop());
365
366 return 0;
367 }
368
369 static int mxs_dcp_block_fallback(struct ablkcipher_request *req, int enc)
370 {
371 struct crypto_tfm *tfm =
372 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
373 struct dcp_async_ctx *ctx = crypto_ablkcipher_ctx(
374 crypto_ablkcipher_reqtfm(req));
375 int ret;
376
377 ablkcipher_request_set_tfm(req, ctx->fallback);
378
379 if (enc)
380 ret = crypto_ablkcipher_encrypt(req);
381 else
382 ret = crypto_ablkcipher_decrypt(req);
383
384 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
385
386 return ret;
387 }
388
389 static int mxs_dcp_aes_enqueue(struct ablkcipher_request *req, int enc, int ecb)
390 {
391 struct dcp *sdcp = global_sdcp;
392 struct crypto_async_request *arq = &req->base;
393 struct dcp_async_ctx *actx = crypto_tfm_ctx(arq->tfm);
394 int ret;
395
396 if (unlikely(actx->key_len != AES_KEYSIZE_128))
397 return mxs_dcp_block_fallback(req, enc);
398
399 actx->enc = enc;
400 actx->ecb = ecb;
401 actx->chan = DCP_CHAN_CRYPTO;
402
403 mutex_lock(&sdcp->mutex[actx->chan]);
404 ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base);
405 mutex_unlock(&sdcp->mutex[actx->chan]);
406
407 wake_up_process(sdcp->thread[actx->chan]);
408
409 return -EINPROGRESS;
410 }
411
412 static int mxs_dcp_aes_ecb_decrypt(struct ablkcipher_request *req)
413 {
414 return mxs_dcp_aes_enqueue(req, 0, 1);
415 }
416
417 static int mxs_dcp_aes_ecb_encrypt(struct ablkcipher_request *req)
418 {
419 return mxs_dcp_aes_enqueue(req, 1, 1);
420 }
421
422 static int mxs_dcp_aes_cbc_decrypt(struct ablkcipher_request *req)
423 {
424 return mxs_dcp_aes_enqueue(req, 0, 0);
425 }
426
427 static int mxs_dcp_aes_cbc_encrypt(struct ablkcipher_request *req)
428 {
429 return mxs_dcp_aes_enqueue(req, 1, 0);
430 }
431
432 static int mxs_dcp_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
433 unsigned int len)
434 {
435 struct dcp_async_ctx *actx = crypto_ablkcipher_ctx(tfm);
436 unsigned int ret;
437
438 /*
439 * AES 128 is supposed by the hardware, store key into temporary
440 * buffer and exit. We must use the temporary buffer here, since
441 * there can still be an operation in progress.
442 */
443 actx->key_len = len;
444 if (len == AES_KEYSIZE_128) {
445 memcpy(actx->key, key, len);
446 return 0;
447 }
448
449 /* Check if the key size is supported by kernel at all. */
450 if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
451 tfm->base.crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
452 return -EINVAL;
453 }
454
455 /*
456 * If the requested AES key size is not supported by the hardware,
457 * but is supported by in-kernel software implementation, we use
458 * software fallback.
459 */
460 actx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
461 actx->fallback->base.crt_flags |=
462 tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK;
463
464 ret = crypto_ablkcipher_setkey(actx->fallback, key, len);
465 if (!ret)
466 return 0;
467
468 tfm->base.crt_flags &= ~CRYPTO_TFM_RES_MASK;
469 tfm->base.crt_flags |=
470 actx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK;
471
472 return ret;
473 }
474
475 static int mxs_dcp_aes_fallback_init(struct crypto_tfm *tfm)
476 {
477 const char *name = tfm->__crt_alg->cra_name;
478 const uint32_t flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
479 struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm);
480 struct crypto_ablkcipher *blk;
481
482 blk = crypto_alloc_ablkcipher(name, 0, flags);
483 if (IS_ERR(blk))
484 return PTR_ERR(blk);
485
486 actx->fallback = blk;
487 tfm->crt_ablkcipher.reqsize = sizeof(struct dcp_async_ctx);
488 return 0;
489 }
490
491 static void mxs_dcp_aes_fallback_exit(struct crypto_tfm *tfm)
492 {
493 struct dcp_async_ctx *actx = crypto_tfm_ctx(tfm);
494
495 crypto_free_ablkcipher(actx->fallback);
496 actx->fallback = NULL;
497 }
498
499 /*
500 * Hashing (SHA1/SHA256)
501 */
502 static int mxs_dcp_run_sha(struct ahash_request *req)
503 {
504 struct dcp *sdcp = global_sdcp;
505 int ret;
506
507 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
508 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm);
509 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req);
510
511 struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan];
512 dma_addr_t digest_phys = dma_map_single(sdcp->dev,
513 sdcp->coh->sha_digest,
514 SHA256_DIGEST_SIZE,
515 DMA_FROM_DEVICE);
516
517 dma_addr_t buf_phys = dma_map_single(sdcp->dev, sdcp->coh->sha_in_buf,
518 DCP_BUF_SZ, DMA_TO_DEVICE);
519
520 /* Fill in the DMA descriptor. */
521 desc->control0 = MXS_DCP_CONTROL0_DECR_SEMAPHORE |
522 MXS_DCP_CONTROL0_INTERRUPT |
523 MXS_DCP_CONTROL0_ENABLE_HASH;
524 if (rctx->init)
525 desc->control0 |= MXS_DCP_CONTROL0_HASH_INIT;
526
527 desc->control1 = actx->alg;
528 desc->next_cmd_addr = 0;
529 desc->source = buf_phys;
530 desc->destination = 0;
531 desc->size = actx->fill;
532 desc->payload = 0;
533 desc->status = 0;
534
535 /* Set HASH_TERM bit for last transfer block. */
536 if (rctx->fini) {
537 desc->control0 |= MXS_DCP_CONTROL0_HASH_TERM;
538 desc->payload = digest_phys;
539 }
540
541 ret = mxs_dcp_start_dma(actx);
542
543 dma_unmap_single(sdcp->dev, digest_phys, SHA256_DIGEST_SIZE,
544 DMA_FROM_DEVICE);
545 dma_unmap_single(sdcp->dev, buf_phys, DCP_BUF_SZ, DMA_TO_DEVICE);
546
547 return ret;
548 }
549
550 static int dcp_sha_req_to_buf(struct crypto_async_request *arq)
551 {
552 struct dcp *sdcp = global_sdcp;
553
554 struct ahash_request *req = ahash_request_cast(arq);
555 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
556 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm);
557 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req);
558 struct hash_alg_common *halg = crypto_hash_alg_common(tfm);
559 const int nents = sg_nents(req->src);
560
561 uint8_t *digest = sdcp->coh->sha_digest;
562 uint8_t *in_buf = sdcp->coh->sha_in_buf;
563
564 uint8_t *src_buf;
565
566 struct scatterlist *src;
567
568 unsigned int i, len, clen;
569 int ret;
570
571 int fin = rctx->fini;
572 if (fin)
573 rctx->fini = 0;
574
575 for_each_sg(req->src, src, nents, i) {
576 src_buf = sg_virt(src);
577 len = sg_dma_len(src);
578
579 do {
580 if (actx->fill + len > DCP_BUF_SZ)
581 clen = DCP_BUF_SZ - actx->fill;
582 else
583 clen = len;
584
585 memcpy(in_buf + actx->fill, src_buf, clen);
586 len -= clen;
587 src_buf += clen;
588 actx->fill += clen;
589
590 /*
591 * If we filled the buffer and still have some
592 * more data, submit the buffer.
593 */
594 if (len && actx->fill == DCP_BUF_SZ) {
595 ret = mxs_dcp_run_sha(req);
596 if (ret)
597 return ret;
598 actx->fill = 0;
599 rctx->init = 0;
600 }
601 } while (len);
602 }
603
604 if (fin) {
605 rctx->fini = 1;
606
607 /* Submit whatever is left. */
608 ret = mxs_dcp_run_sha(req);
609 if (ret || !req->result)
610 return ret;
611 actx->fill = 0;
612
613 /* For some reason, the result is flipped. */
614 for (i = 0; i < halg->digestsize; i++)
615 req->result[i] = digest[halg->digestsize - i - 1];
616 }
617
618 return 0;
619 }
620
621 static int dcp_chan_thread_sha(void *data)
622 {
623 struct dcp *sdcp = global_sdcp;
624 const int chan = DCP_CHAN_HASH_SHA;
625
626 struct crypto_async_request *backlog;
627 struct crypto_async_request *arq;
628
629 struct dcp_sha_req_ctx *rctx;
630
631 struct ahash_request *req;
632 int ret, fini;
633
634 do {
635 __set_current_state(TASK_INTERRUPTIBLE);
636
637 mutex_lock(&sdcp->mutex[chan]);
638 backlog = crypto_get_backlog(&sdcp->queue[chan]);
639 arq = crypto_dequeue_request(&sdcp->queue[chan]);
640 mutex_unlock(&sdcp->mutex[chan]);
641
642 if (backlog)
643 backlog->complete(backlog, -EINPROGRESS);
644
645 if (arq) {
646 req = ahash_request_cast(arq);
647 rctx = ahash_request_ctx(req);
648
649 ret = dcp_sha_req_to_buf(arq);
650 fini = rctx->fini;
651 arq->complete(arq, ret);
652 if (!fini)
653 continue;
654 }
655
656 schedule();
657 } while (!kthread_should_stop());
658
659 return 0;
660 }
661
662 static int dcp_sha_init(struct ahash_request *req)
663 {
664 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
665 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm);
666
667 struct hash_alg_common *halg = crypto_hash_alg_common(tfm);
668
669 /*
670 * Start hashing session. The code below only inits the
671 * hashing session context, nothing more.
672 */
673 memset(actx, 0, sizeof(*actx));
674
675 if (strcmp(halg->base.cra_name, "sha1") == 0)
676 actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA1;
677 else
678 actx->alg = MXS_DCP_CONTROL1_HASH_SELECT_SHA256;
679
680 actx->fill = 0;
681 actx->hot = 0;
682 actx->chan = DCP_CHAN_HASH_SHA;
683
684 mutex_init(&actx->mutex);
685
686 return 0;
687 }
688
689 static int dcp_sha_update_fx(struct ahash_request *req, int fini)
690 {
691 struct dcp *sdcp = global_sdcp;
692
693 struct dcp_sha_req_ctx *rctx = ahash_request_ctx(req);
694 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
695 struct dcp_async_ctx *actx = crypto_ahash_ctx(tfm);
696
697 int ret;
698
699 /*
700 * Ignore requests that have no data in them and are not
701 * the trailing requests in the stream of requests.
702 */
703 if (!req->nbytes && !fini)
704 return 0;
705
706 mutex_lock(&actx->mutex);
707
708 rctx->fini = fini;
709
710 if (!actx->hot) {
711 actx->hot = 1;
712 rctx->init = 1;
713 }
714
715 mutex_lock(&sdcp->mutex[actx->chan]);
716 ret = crypto_enqueue_request(&sdcp->queue[actx->chan], &req->base);
717 mutex_unlock(&sdcp->mutex[actx->chan]);
718
719 wake_up_process(sdcp->thread[actx->chan]);
720 mutex_unlock(&actx->mutex);
721
722 return -EINPROGRESS;
723 }
724
725 static int dcp_sha_update(struct ahash_request *req)
726 {
727 return dcp_sha_update_fx(req, 0);
728 }
729
730 static int dcp_sha_final(struct ahash_request *req)
731 {
732 ahash_request_set_crypt(req, NULL, req->result, 0);
733 req->nbytes = 0;
734 return dcp_sha_update_fx(req, 1);
735 }
736
737 static int dcp_sha_finup(struct ahash_request *req)
738 {
739 return dcp_sha_update_fx(req, 1);
740 }
741
742 static int dcp_sha_digest(struct ahash_request *req)
743 {
744 int ret;
745
746 ret = dcp_sha_init(req);
747 if (ret)
748 return ret;
749
750 return dcp_sha_finup(req);
751 }
752
753 static int dcp_sha_cra_init(struct crypto_tfm *tfm)
754 {
755 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
756 sizeof(struct dcp_sha_req_ctx));
757 return 0;
758 }
759
760 static void dcp_sha_cra_exit(struct crypto_tfm *tfm)
761 {
762 }
763
764 /* AES 128 ECB and AES 128 CBC */
765 static struct crypto_alg dcp_aes_algs[] = {
766 {
767 .cra_name = "ecb(aes)",
768 .cra_driver_name = "ecb-aes-dcp",
769 .cra_priority = 400,
770 .cra_alignmask = 15,
771 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
772 CRYPTO_ALG_ASYNC |
773 CRYPTO_ALG_NEED_FALLBACK,
774 .cra_init = mxs_dcp_aes_fallback_init,
775 .cra_exit = mxs_dcp_aes_fallback_exit,
776 .cra_blocksize = AES_BLOCK_SIZE,
777 .cra_ctxsize = sizeof(struct dcp_async_ctx),
778 .cra_type = &crypto_ablkcipher_type,
779 .cra_module = THIS_MODULE,
780 .cra_u = {
781 .ablkcipher = {
782 .min_keysize = AES_MIN_KEY_SIZE,
783 .max_keysize = AES_MAX_KEY_SIZE,
784 .setkey = mxs_dcp_aes_setkey,
785 .encrypt = mxs_dcp_aes_ecb_encrypt,
786 .decrypt = mxs_dcp_aes_ecb_decrypt
787 },
788 },
789 }, {
790 .cra_name = "cbc(aes)",
791 .cra_driver_name = "cbc-aes-dcp",
792 .cra_priority = 400,
793 .cra_alignmask = 15,
794 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
795 CRYPTO_ALG_ASYNC |
796 CRYPTO_ALG_NEED_FALLBACK,
797 .cra_init = mxs_dcp_aes_fallback_init,
798 .cra_exit = mxs_dcp_aes_fallback_exit,
799 .cra_blocksize = AES_BLOCK_SIZE,
800 .cra_ctxsize = sizeof(struct dcp_async_ctx),
801 .cra_type = &crypto_ablkcipher_type,
802 .cra_module = THIS_MODULE,
803 .cra_u = {
804 .ablkcipher = {
805 .min_keysize = AES_MIN_KEY_SIZE,
806 .max_keysize = AES_MAX_KEY_SIZE,
807 .setkey = mxs_dcp_aes_setkey,
808 .encrypt = mxs_dcp_aes_cbc_encrypt,
809 .decrypt = mxs_dcp_aes_cbc_decrypt,
810 .ivsize = AES_BLOCK_SIZE,
811 },
812 },
813 },
814 };
815
816 /* SHA1 */
817 static struct ahash_alg dcp_sha1_alg = {
818 .init = dcp_sha_init,
819 .update = dcp_sha_update,
820 .final = dcp_sha_final,
821 .finup = dcp_sha_finup,
822 .digest = dcp_sha_digest,
823 .halg = {
824 .digestsize = SHA1_DIGEST_SIZE,
825 .base = {
826 .cra_name = "sha1",
827 .cra_driver_name = "sha1-dcp",
828 .cra_priority = 400,
829 .cra_alignmask = 63,
830 .cra_flags = CRYPTO_ALG_ASYNC,
831 .cra_blocksize = SHA1_BLOCK_SIZE,
832 .cra_ctxsize = sizeof(struct dcp_async_ctx),
833 .cra_module = THIS_MODULE,
834 .cra_init = dcp_sha_cra_init,
835 .cra_exit = dcp_sha_cra_exit,
836 },
837 },
838 };
839
840 /* SHA256 */
841 static struct ahash_alg dcp_sha256_alg = {
842 .init = dcp_sha_init,
843 .update = dcp_sha_update,
844 .final = dcp_sha_final,
845 .finup = dcp_sha_finup,
846 .digest = dcp_sha_digest,
847 .halg = {
848 .digestsize = SHA256_DIGEST_SIZE,
849 .base = {
850 .cra_name = "sha256",
851 .cra_driver_name = "sha256-dcp",
852 .cra_priority = 400,
853 .cra_alignmask = 63,
854 .cra_flags = CRYPTO_ALG_ASYNC,
855 .cra_blocksize = SHA256_BLOCK_SIZE,
856 .cra_ctxsize = sizeof(struct dcp_async_ctx),
857 .cra_module = THIS_MODULE,
858 .cra_init = dcp_sha_cra_init,
859 .cra_exit = dcp_sha_cra_exit,
860 },
861 },
862 };
863
864 static irqreturn_t mxs_dcp_irq(int irq, void *context)
865 {
866 struct dcp *sdcp = context;
867 uint32_t stat;
868 int i;
869
870 stat = readl(sdcp->base + MXS_DCP_STAT);
871 stat &= MXS_DCP_STAT_IRQ_MASK;
872 if (!stat)
873 return IRQ_NONE;
874
875 /* Clear the interrupts. */
876 writel(stat, sdcp->base + MXS_DCP_STAT_CLR);
877
878 /* Complete the DMA requests that finished. */
879 for (i = 0; i < DCP_MAX_CHANS; i++)
880 if (stat & (1 << i))
881 complete(&sdcp->completion[i]);
882
883 return IRQ_HANDLED;
884 }
885
886 static int mxs_dcp_probe(struct platform_device *pdev)
887 {
888 struct device *dev = &pdev->dev;
889 struct dcp *sdcp = NULL;
890 int i, ret;
891
892 struct resource *iores;
893 int dcp_vmi_irq, dcp_irq;
894
895 mutex_lock(&global_mutex);
896 if (global_sdcp) {
897 dev_err(dev, "Only one DCP instance allowed!\n");
898 ret = -ENODEV;
899 goto err_mutex;
900 }
901
902 iores = platform_get_resource(pdev, IORESOURCE_MEM, 0);
903 dcp_vmi_irq = platform_get_irq(pdev, 0);
904 dcp_irq = platform_get_irq(pdev, 1);
905 if (dcp_vmi_irq < 0 || dcp_irq < 0) {
906 ret = -EINVAL;
907 goto err_mutex;
908 }
909
910 sdcp = devm_kzalloc(dev, sizeof(*sdcp), GFP_KERNEL);
911 if (!sdcp) {
912 ret = -ENOMEM;
913 goto err_mutex;
914 }
915
916 sdcp->dev = dev;
917 sdcp->base = devm_ioremap_resource(dev, iores);
918 if (IS_ERR(sdcp->base)) {
919 ret = PTR_ERR(sdcp->base);
920 goto err_mutex;
921 }
922
923 ret = devm_request_irq(dev, dcp_vmi_irq, mxs_dcp_irq, 0,
924 "dcp-vmi-irq", sdcp);
925 if (ret) {
926 dev_err(dev, "Failed to claim DCP VMI IRQ!\n");
927 goto err_mutex;
928 }
929
930 ret = devm_request_irq(dev, dcp_irq, mxs_dcp_irq, 0,
931 "dcp-irq", sdcp);
932 if (ret) {
933 dev_err(dev, "Failed to claim DCP IRQ!\n");
934 goto err_mutex;
935 }
936
937 /* Allocate coherent helper block. */
938 sdcp->coh = kzalloc(sizeof(struct dcp_coherent_block), GFP_KERNEL);
939 if (!sdcp->coh) {
940 dev_err(dev, "Error allocating coherent block\n");
941 ret = -ENOMEM;
942 goto err_mutex;
943 }
944
945 /* Restart the DCP block. */
946 stmp_reset_block(sdcp->base);
947
948 /* Initialize control register. */
949 writel(MXS_DCP_CTRL_GATHER_RESIDUAL_WRITES |
950 MXS_DCP_CTRL_ENABLE_CONTEXT_CACHING | 0xf,
951 sdcp->base + MXS_DCP_CTRL);
952
953 /* Enable all DCP DMA channels. */
954 writel(MXS_DCP_CHANNELCTRL_ENABLE_CHANNEL_MASK,
955 sdcp->base + MXS_DCP_CHANNELCTRL);
956
957 /*
958 * We do not enable context switching. Give the context buffer a
959 * pointer to an illegal address so if context switching is
960 * inadvertantly enabled, the DCP will return an error instead of
961 * trashing good memory. The DCP DMA cannot access ROM, so any ROM
962 * address will do.
963 */
964 writel(0xffff0000, sdcp->base + MXS_DCP_CONTEXT);
965 for (i = 0; i < DCP_MAX_CHANS; i++)
966 writel(0xffffffff, sdcp->base + MXS_DCP_CH_N_STAT_CLR(i));
967 writel(0xffffffff, sdcp->base + MXS_DCP_STAT_CLR);
968
969 global_sdcp = sdcp;
970
971 platform_set_drvdata(pdev, sdcp);
972
973 for (i = 0; i < DCP_MAX_CHANS; i++) {
974 mutex_init(&sdcp->mutex[i]);
975 init_completion(&sdcp->completion[i]);
976 crypto_init_queue(&sdcp->queue[i], 50);
977 }
978
979 /* Create the SHA and AES handler threads. */
980 sdcp->thread[DCP_CHAN_HASH_SHA] = kthread_run(dcp_chan_thread_sha,
981 NULL, "mxs_dcp_chan/sha");
982 if (IS_ERR(sdcp->thread[DCP_CHAN_HASH_SHA])) {
983 dev_err(dev, "Error starting SHA thread!\n");
984 ret = PTR_ERR(sdcp->thread[DCP_CHAN_HASH_SHA]);
985 goto err_free_coherent;
986 }
987
988 sdcp->thread[DCP_CHAN_CRYPTO] = kthread_run(dcp_chan_thread_aes,
989 NULL, "mxs_dcp_chan/aes");
990 if (IS_ERR(sdcp->thread[DCP_CHAN_CRYPTO])) {
991 dev_err(dev, "Error starting SHA thread!\n");
992 ret = PTR_ERR(sdcp->thread[DCP_CHAN_CRYPTO]);
993 goto err_destroy_sha_thread;
994 }
995
996 /* Register the various crypto algorithms. */
997 sdcp->caps = readl(sdcp->base + MXS_DCP_CAPABILITY1);
998
999 if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128) {
1000 ret = crypto_register_algs(dcp_aes_algs,
1001 ARRAY_SIZE(dcp_aes_algs));
1002 if (ret) {
1003 /* Failed to register algorithm. */
1004 dev_err(dev, "Failed to register AES crypto!\n");
1005 goto err_destroy_aes_thread;
1006 }
1007 }
1008
1009 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1) {
1010 ret = crypto_register_ahash(&dcp_sha1_alg);
1011 if (ret) {
1012 dev_err(dev, "Failed to register %s hash!\n",
1013 dcp_sha1_alg.halg.base.cra_name);
1014 goto err_unregister_aes;
1015 }
1016 }
1017
1018 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256) {
1019 ret = crypto_register_ahash(&dcp_sha256_alg);
1020 if (ret) {
1021 dev_err(dev, "Failed to register %s hash!\n",
1022 dcp_sha256_alg.halg.base.cra_name);
1023 goto err_unregister_sha1;
1024 }
1025 }
1026
1027 return 0;
1028
1029 err_unregister_sha1:
1030 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1)
1031 crypto_unregister_ahash(&dcp_sha1_alg);
1032
1033 err_unregister_aes:
1034 if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128)
1035 crypto_unregister_algs(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs));
1036
1037 err_destroy_aes_thread:
1038 kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]);
1039
1040 err_destroy_sha_thread:
1041 kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]);
1042
1043 err_free_coherent:
1044 kfree(sdcp->coh);
1045 err_mutex:
1046 mutex_unlock(&global_mutex);
1047 return ret;
1048 }
1049
1050 static int mxs_dcp_remove(struct platform_device *pdev)
1051 {
1052 struct dcp *sdcp = platform_get_drvdata(pdev);
1053
1054 kfree(sdcp->coh);
1055
1056 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA256)
1057 crypto_unregister_ahash(&dcp_sha256_alg);
1058
1059 if (sdcp->caps & MXS_DCP_CAPABILITY1_SHA1)
1060 crypto_unregister_ahash(&dcp_sha1_alg);
1061
1062 if (sdcp->caps & MXS_DCP_CAPABILITY1_AES128)
1063 crypto_unregister_algs(dcp_aes_algs, ARRAY_SIZE(dcp_aes_algs));
1064
1065 kthread_stop(sdcp->thread[DCP_CHAN_HASH_SHA]);
1066 kthread_stop(sdcp->thread[DCP_CHAN_CRYPTO]);
1067
1068 platform_set_drvdata(pdev, NULL);
1069
1070 mutex_lock(&global_mutex);
1071 global_sdcp = NULL;
1072 mutex_unlock(&global_mutex);
1073
1074 return 0;
1075 }
1076
1077 static const struct of_device_id mxs_dcp_dt_ids[] = {
1078 { .compatible = "fsl,imx23-dcp", .data = NULL, },
1079 { .compatible = "fsl,imx28-dcp", .data = NULL, },
1080 { /* sentinel */ }
1081 };
1082
1083 MODULE_DEVICE_TABLE(of, mxs_dcp_dt_ids);
1084
1085 static struct platform_driver mxs_dcp_driver = {
1086 .probe = mxs_dcp_probe,
1087 .remove = mxs_dcp_remove,
1088 .driver = {
1089 .name = "mxs-dcp",
1090 .owner = THIS_MODULE,
1091 .of_match_table = mxs_dcp_dt_ids,
1092 },
1093 };
1094
1095 module_platform_driver(mxs_dcp_driver);
1096
1097 MODULE_AUTHOR("Marek Vasut <marex@denx.de>");
1098 MODULE_DESCRIPTION("Freescale MXS DCP Driver");
1099 MODULE_LICENSE("GPL");
1100 MODULE_ALIAS("platform:mxs-dcp");
Linux with added FPC-III support