?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 4.1.16
[linux/fpc-iii.git] / drivers / crypto / sahara.c
blob6be377f6b9e7019c44ba0e5b303b1d7cdd918170
1 /*
2 * Cryptographic API.
3 *
4 * Support for SAHARA cryptographic accelerator.
5 *
6 * Copyright (c) 2014 Steffen Trumtrar <s.trumtrar@pengutronix.de>
7 * Copyright (c) 2013 Vista Silicon S.L.
8 * Author: Javier Martin <javier.martin@vista-silicon.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2 as published
12 * by the Free Software Foundation.
13 *
14 * Based on omap-aes.c and tegra-aes.c
15 */
16
17 #include <crypto/algapi.h>
18 #include <crypto/aes.h>
19 #include <crypto/hash.h>
20 #include <crypto/internal/hash.h>
21 #include <crypto/scatterwalk.h>
22 #include <crypto/sha.h>
23
24 #include <linux/clk.h>
25 #include <linux/crypto.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/irq.h>
29 #include <linux/kernel.h>
30 #include <linux/kthread.h>
31 #include <linux/module.h>
32 #include <linux/mutex.h>
33 #include <linux/of.h>
34 #include <linux/of_device.h>
35 #include <linux/platform_device.h>
36
37 #define SHA_BUFFER_LEN PAGE_SIZE
38 #define SAHARA_MAX_SHA_BLOCK_SIZE SHA256_BLOCK_SIZE
39
40 #define SAHARA_NAME "sahara"
41 #define SAHARA_VERSION_3 3
42 #define SAHARA_VERSION_4 4
43 #define SAHARA_TIMEOUT_MS 1000
44 #define SAHARA_MAX_HW_DESC 2
45 #define SAHARA_MAX_HW_LINK 20
46
47 #define FLAGS_MODE_MASK 0x000f
48 #define FLAGS_ENCRYPT BIT(0)
49 #define FLAGS_CBC BIT(1)
50 #define FLAGS_NEW_KEY BIT(3)
51
52 #define SAHARA_HDR_BASE 0x00800000
53 #define SAHARA_HDR_SKHA_ALG_AES 0
54 #define SAHARA_HDR_SKHA_OP_ENC (1 << 2)
55 #define SAHARA_HDR_SKHA_MODE_ECB (0 << 3)
56 #define SAHARA_HDR_SKHA_MODE_CBC (1 << 3)
57 #define SAHARA_HDR_FORM_DATA (5 << 16)
58 #define SAHARA_HDR_FORM_KEY (8 << 16)
59 #define SAHARA_HDR_LLO (1 << 24)
60 #define SAHARA_HDR_CHA_SKHA (1 << 28)
61 #define SAHARA_HDR_CHA_MDHA (2 << 28)
62 #define SAHARA_HDR_PARITY_BIT (1 << 31)
63
64 #define SAHARA_HDR_MDHA_SET_MODE_MD_KEY 0x20880000
65 #define SAHARA_HDR_MDHA_SET_MODE_HASH 0x208D0000
66 #define SAHARA_HDR_MDHA_HASH 0xA0850000
67 #define SAHARA_HDR_MDHA_STORE_DIGEST 0x20820000
68 #define SAHARA_HDR_MDHA_ALG_SHA1 0
69 #define SAHARA_HDR_MDHA_ALG_MD5 1
70 #define SAHARA_HDR_MDHA_ALG_SHA256 2
71 #define SAHARA_HDR_MDHA_ALG_SHA224 3
72 #define SAHARA_HDR_MDHA_PDATA (1 << 2)
73 #define SAHARA_HDR_MDHA_HMAC (1 << 3)
74 #define SAHARA_HDR_MDHA_INIT (1 << 5)
75 #define SAHARA_HDR_MDHA_IPAD (1 << 6)
76 #define SAHARA_HDR_MDHA_OPAD (1 << 7)
77 #define SAHARA_HDR_MDHA_SWAP (1 << 8)
78 #define SAHARA_HDR_MDHA_MAC_FULL (1 << 9)
79 #define SAHARA_HDR_MDHA_SSL (1 << 10)
80
81 /* SAHARA can only process one request at a time */
82 #define SAHARA_QUEUE_LENGTH 1
83
84 #define SAHARA_REG_VERSION 0x00
85 #define SAHARA_REG_DAR 0x04
86 #define SAHARA_REG_CONTROL 0x08
87 #define SAHARA_CONTROL_SET_THROTTLE(x) (((x) & 0xff) << 24)
88 #define SAHARA_CONTROL_SET_MAXBURST(x) (((x) & 0xff) << 16)
89 #define SAHARA_CONTROL_RNG_AUTORSD (1 << 7)
90 #define SAHARA_CONTROL_ENABLE_INT (1 << 4)
91 #define SAHARA_REG_CMD 0x0C
92 #define SAHARA_CMD_RESET (1 << 0)
93 #define SAHARA_CMD_CLEAR_INT (1 << 8)
94 #define SAHARA_CMD_CLEAR_ERR (1 << 9)
95 #define SAHARA_CMD_SINGLE_STEP (1 << 10)
96 #define SAHARA_CMD_MODE_BATCH (1 << 16)
97 #define SAHARA_CMD_MODE_DEBUG (1 << 18)
98 #define SAHARA_REG_STATUS 0x10
99 #define SAHARA_STATUS_GET_STATE(x) ((x) & 0x7)
100 #define SAHARA_STATE_IDLE 0
101 #define SAHARA_STATE_BUSY 1
102 #define SAHARA_STATE_ERR 2
103 #define SAHARA_STATE_FAULT 3
104 #define SAHARA_STATE_COMPLETE 4
105 #define SAHARA_STATE_COMP_FLAG (1 << 2)
106 #define SAHARA_STATUS_DAR_FULL (1 << 3)
107 #define SAHARA_STATUS_ERROR (1 << 4)
108 #define SAHARA_STATUS_SECURE (1 << 5)
109 #define SAHARA_STATUS_FAIL (1 << 6)
110 #define SAHARA_STATUS_INIT (1 << 7)
111 #define SAHARA_STATUS_RNG_RESEED (1 << 8)
112 #define SAHARA_STATUS_ACTIVE_RNG (1 << 9)
113 #define SAHARA_STATUS_ACTIVE_MDHA (1 << 10)
114 #define SAHARA_STATUS_ACTIVE_SKHA (1 << 11)
115 #define SAHARA_STATUS_MODE_BATCH (1 << 16)
116 #define SAHARA_STATUS_MODE_DEDICATED (1 << 17)
117 #define SAHARA_STATUS_MODE_DEBUG (1 << 18)
118 #define SAHARA_STATUS_GET_ISTATE(x) (((x) >> 24) & 0xff)
119 #define SAHARA_REG_ERRSTATUS 0x14
120 #define SAHARA_ERRSTATUS_GET_SOURCE(x) ((x) & 0xf)
121 #define SAHARA_ERRSOURCE_CHA 14
122 #define SAHARA_ERRSOURCE_DMA 15
123 #define SAHARA_ERRSTATUS_DMA_DIR (1 << 8)
124 #define SAHARA_ERRSTATUS_GET_DMASZ(x)(((x) >> 9) & 0x3)
125 #define SAHARA_ERRSTATUS_GET_DMASRC(x) (((x) >> 13) & 0x7)
126 #define SAHARA_ERRSTATUS_GET_CHASRC(x) (((x) >> 16) & 0xfff)
127 #define SAHARA_ERRSTATUS_GET_CHAERR(x) (((x) >> 28) & 0x3)
128 #define SAHARA_REG_FADDR 0x18
129 #define SAHARA_REG_CDAR 0x1C
130 #define SAHARA_REG_IDAR 0x20
131
132 struct sahara_hw_desc {
133 u32 hdr;
134 u32 len1;
135 dma_addr_t p1;
136 u32 len2;
137 dma_addr_t p2;
138 dma_addr_t next;
139 };
140
141 struct sahara_hw_link {
142 u32 len;
143 dma_addr_t p;
144 dma_addr_t next;
145 };
146
147 struct sahara_ctx {
148 unsigned long flags;
149
150 /* AES-specific context */
151 int keylen;
152 u8 key[AES_KEYSIZE_128];
153 struct crypto_ablkcipher *fallback;
154
155 /* SHA-specific context */
156 struct crypto_shash *shash_fallback;
157 };
158
159 struct sahara_aes_reqctx {
160 unsigned long mode;
161 };
162
163 /*
164 * struct sahara_sha_reqctx - private data per request
165 * @buf: holds data for requests smaller than block_size
166 * @rembuf: used to prepare one block_size-aligned request
167 * @context: hw-specific context for request. Digest is extracted from this
168 * @mode: specifies what type of hw-descriptor needs to be built
169 * @digest_size: length of digest for this request
170 * @context_size: length of hw-context for this request.
171 * Always digest_size + 4
172 * @buf_cnt: number of bytes saved in buf
173 * @sg_in_idx: number of hw links
174 * @in_sg: scatterlist for input data
175 * @in_sg_chain: scatterlists for chained input data
176 * @in_sg_chained: specifies if chained scatterlists are used or not
177 * @total: total number of bytes for transfer
178 * @last: is this the last block
179 * @first: is this the first block
180 * @active: inside a transfer
181 */
182 struct sahara_sha_reqctx {
183 u8 buf[SAHARA_MAX_SHA_BLOCK_SIZE];
184 u8 rembuf[SAHARA_MAX_SHA_BLOCK_SIZE];
185 u8 context[SHA256_DIGEST_SIZE + 4];
186 struct mutex mutex;
187 unsigned int mode;
188 unsigned int digest_size;
189 unsigned int context_size;
190 unsigned int buf_cnt;
191 unsigned int sg_in_idx;
192 struct scatterlist *in_sg;
193 struct scatterlist in_sg_chain[2];
194 bool in_sg_chained;
195 size_t total;
196 unsigned int last;
197 unsigned int first;
198 unsigned int active;
199 };
200
201 struct sahara_dev {
202 struct device *device;
203 unsigned int version;
204 void __iomem *regs_base;
205 struct clk *clk_ipg;
206 struct clk *clk_ahb;
207 struct mutex queue_mutex;
208 struct task_struct *kthread;
209 struct completion dma_completion;
210
211 struct sahara_ctx *ctx;
212 spinlock_t lock;
213 struct crypto_queue queue;
214 unsigned long flags;
215
216 struct sahara_hw_desc *hw_desc[SAHARA_MAX_HW_DESC];
217 dma_addr_t hw_phys_desc[SAHARA_MAX_HW_DESC];
218
219 u8 *key_base;
220 dma_addr_t key_phys_base;
221
222 u8 *iv_base;
223 dma_addr_t iv_phys_base;
224
225 u8 *context_base;
226 dma_addr_t context_phys_base;
227
228 struct sahara_hw_link *hw_link[SAHARA_MAX_HW_LINK];
229 dma_addr_t hw_phys_link[SAHARA_MAX_HW_LINK];
230
231 size_t total;
232 struct scatterlist *in_sg;
233 unsigned int nb_in_sg;
234 struct scatterlist *out_sg;
235 unsigned int nb_out_sg;
236
237 u32 error;
238 };
239
240 static struct sahara_dev *dev_ptr;
241
242 static inline void sahara_write(struct sahara_dev *dev, u32 data, u32 reg)
243 {
244 writel(data, dev->regs_base + reg);
245 }
246
247 static inline unsigned int sahara_read(struct sahara_dev *dev, u32 reg)
248 {
249 return readl(dev->regs_base + reg);
250 }
251
252 static u32 sahara_aes_key_hdr(struct sahara_dev *dev)
253 {
254 u32 hdr = SAHARA_HDR_BASE | SAHARA_HDR_SKHA_ALG_AES |
255 SAHARA_HDR_FORM_KEY | SAHARA_HDR_LLO |
256 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
257
258 if (dev->flags & FLAGS_CBC) {
259 hdr |= SAHARA_HDR_SKHA_MODE_CBC;
260 hdr ^= SAHARA_HDR_PARITY_BIT;
261 }
262
263 if (dev->flags & FLAGS_ENCRYPT) {
264 hdr |= SAHARA_HDR_SKHA_OP_ENC;
265 hdr ^= SAHARA_HDR_PARITY_BIT;
266 }
267
268 return hdr;
269 }
270
271 static u32 sahara_aes_data_link_hdr(struct sahara_dev *dev)
272 {
273 return SAHARA_HDR_BASE | SAHARA_HDR_FORM_DATA |
274 SAHARA_HDR_CHA_SKHA | SAHARA_HDR_PARITY_BIT;
275 }
276
277 static int sahara_sg_length(struct scatterlist *sg,
278 unsigned int total)
279 {
280 int sg_nb;
281 unsigned int len;
282 struct scatterlist *sg_list;
283
284 sg_nb = 0;
285 sg_list = sg;
286
287 while (total) {
288 len = min(sg_list->length, total);
289
290 sg_nb++;
291 total -= len;
292
293 sg_list = sg_next(sg_list);
294 if (!sg_list)
295 total = 0;
296 }
297
298 return sg_nb;
299 }
300
301 static char *sahara_err_src[16] = {
302 "No error",
303 "Header error",
304 "Descriptor length error",
305 "Descriptor length or pointer error",
306 "Link length error",
307 "Link pointer error",
308 "Input buffer error",
309 "Output buffer error",
310 "Output buffer starvation",
311 "Internal state fault",
312 "General descriptor problem",
313 "Reserved",
314 "Descriptor address error",
315 "Link address error",
316 "CHA error",
317 "DMA error"
318 };
319
320 static char *sahara_err_dmasize[4] = {
321 "Byte transfer",
322 "Half-word transfer",
323 "Word transfer",
324 "Reserved"
325 };
326
327 static char *sahara_err_dmasrc[8] = {
328 "No error",
329 "AHB bus error",
330 "Internal IP bus error",
331 "Parity error",
332 "DMA crosses 256 byte boundary",
333 "DMA is busy",
334 "Reserved",
335 "DMA HW error"
336 };
337
338 static char *sahara_cha_errsrc[12] = {
339 "Input buffer non-empty",
340 "Illegal address",
341 "Illegal mode",
342 "Illegal data size",
343 "Illegal key size",
344 "Write during processing",
345 "CTX read during processing",
346 "HW error",
347 "Input buffer disabled/underflow",
348 "Output buffer disabled/overflow",
349 "DES key parity error",
350 "Reserved"
351 };
352
353 static char *sahara_cha_err[4] = { "No error", "SKHA", "MDHA", "RNG" };
354
355 static void sahara_decode_error(struct sahara_dev *dev, unsigned int error)
356 {
357 u8 source = SAHARA_ERRSTATUS_GET_SOURCE(error);
358 u16 chasrc = ffs(SAHARA_ERRSTATUS_GET_CHASRC(error));
359
360 dev_err(dev->device, "%s: Error Register = 0x%08x\n", __func__, error);
361
362 dev_err(dev->device, " - %s.\n", sahara_err_src[source]);
363
364 if (source == SAHARA_ERRSOURCE_DMA) {
365 if (error & SAHARA_ERRSTATUS_DMA_DIR)
366 dev_err(dev->device, " * DMA read.\n");
367 else
368 dev_err(dev->device, " * DMA write.\n");
369
370 dev_err(dev->device, " * %s.\n",
371 sahara_err_dmasize[SAHARA_ERRSTATUS_GET_DMASZ(error)]);
372 dev_err(dev->device, " * %s.\n",
373 sahara_err_dmasrc[SAHARA_ERRSTATUS_GET_DMASRC(error)]);
374 } else if (source == SAHARA_ERRSOURCE_CHA) {
375 dev_err(dev->device, " * %s.\n",
376 sahara_cha_errsrc[chasrc]);
377 dev_err(dev->device, " * %s.\n",
378 sahara_cha_err[SAHARA_ERRSTATUS_GET_CHAERR(error)]);
379 }
380 dev_err(dev->device, "\n");
381 }
382
383 static char *sahara_state[4] = { "Idle", "Busy", "Error", "HW Fault" };
384
385 static void sahara_decode_status(struct sahara_dev *dev, unsigned int status)
386 {
387 u8 state;
388
389 if (!IS_ENABLED(DEBUG))
390 return;
391
392 state = SAHARA_STATUS_GET_STATE(status);
393
394 dev_dbg(dev->device, "%s: Status Register = 0x%08x\n",
395 __func__, status);
396
397 dev_dbg(dev->device, " - State = %d:\n", state);
398 if (state & SAHARA_STATE_COMP_FLAG)
399 dev_dbg(dev->device, " * Descriptor completed. IRQ pending.\n");
400
401 dev_dbg(dev->device, " * %s.\n",
402 sahara_state[state & ~SAHARA_STATE_COMP_FLAG]);
403
404 if (status & SAHARA_STATUS_DAR_FULL)
405 dev_dbg(dev->device, " - DAR Full.\n");
406 if (status & SAHARA_STATUS_ERROR)
407 dev_dbg(dev->device, " - Error.\n");
408 if (status & SAHARA_STATUS_SECURE)
409 dev_dbg(dev->device, " - Secure.\n");
410 if (status & SAHARA_STATUS_FAIL)
411 dev_dbg(dev->device, " - Fail.\n");
412 if (status & SAHARA_STATUS_RNG_RESEED)
413 dev_dbg(dev->device, " - RNG Reseed Request.\n");
414 if (status & SAHARA_STATUS_ACTIVE_RNG)
415 dev_dbg(dev->device, " - RNG Active.\n");
416 if (status & SAHARA_STATUS_ACTIVE_MDHA)
417 dev_dbg(dev->device, " - MDHA Active.\n");
418 if (status & SAHARA_STATUS_ACTIVE_SKHA)
419 dev_dbg(dev->device, " - SKHA Active.\n");
420
421 if (status & SAHARA_STATUS_MODE_BATCH)
422 dev_dbg(dev->device, " - Batch Mode.\n");
423 else if (status & SAHARA_STATUS_MODE_DEDICATED)
424 dev_dbg(dev->device, " - Decidated Mode.\n");
425 else if (status & SAHARA_STATUS_MODE_DEBUG)
426 dev_dbg(dev->device, " - Debug Mode.\n");
427
428 dev_dbg(dev->device, " - Internal state = 0x%02x\n",
429 SAHARA_STATUS_GET_ISTATE(status));
430
431 dev_dbg(dev->device, "Current DAR: 0x%08x\n",
432 sahara_read(dev, SAHARA_REG_CDAR));
433 dev_dbg(dev->device, "Initial DAR: 0x%08x\n\n",
434 sahara_read(dev, SAHARA_REG_IDAR));
435 }
436
437 static void sahara_dump_descriptors(struct sahara_dev *dev)
438 {
439 int i;
440
441 if (!IS_ENABLED(DEBUG))
442 return;
443
444 for (i = 0; i < SAHARA_MAX_HW_DESC; i++) {
445 dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n",
446 i, dev->hw_phys_desc[i]);
447 dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr);
448 dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1);
449 dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1);
450 dev_dbg(dev->device, "\tlen2 = %u\n", dev->hw_desc[i]->len2);
451 dev_dbg(dev->device, "\tp2 = 0x%08x\n", dev->hw_desc[i]->p2);
452 dev_dbg(dev->device, "\tnext = 0x%08x\n",
453 dev->hw_desc[i]->next);
454 }
455 dev_dbg(dev->device, "\n");
456 }
457
458 static void sahara_dump_links(struct sahara_dev *dev)
459 {
460 int i;
461
462 if (!IS_ENABLED(DEBUG))
463 return;
464
465 for (i = 0; i < SAHARA_MAX_HW_LINK; i++) {
466 dev_dbg(dev->device, "Link (%d) (0x%08x):\n",
467 i, dev->hw_phys_link[i]);
468 dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len);
469 dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p);
470 dev_dbg(dev->device, "\tnext = 0x%08x\n",
471 dev->hw_link[i]->next);
472 }
473 dev_dbg(dev->device, "\n");
474 }
475
476 static int sahara_hw_descriptor_create(struct sahara_dev *dev)
477 {
478 struct sahara_ctx *ctx = dev->ctx;
479 struct scatterlist *sg;
480 int ret;
481 int i, j;
482 int idx = 0;
483
484 /* Copy new key if necessary */
485 if (ctx->flags & FLAGS_NEW_KEY) {
486 memcpy(dev->key_base, ctx->key, ctx->keylen);
487 ctx->flags &= ~FLAGS_NEW_KEY;
488
489 if (dev->flags & FLAGS_CBC) {
490 dev->hw_desc[idx]->len1 = AES_BLOCK_SIZE;
491 dev->hw_desc[idx]->p1 = dev->iv_phys_base;
492 } else {
493 dev->hw_desc[idx]->len1 = 0;
494 dev->hw_desc[idx]->p1 = 0;
495 }
496 dev->hw_desc[idx]->len2 = ctx->keylen;
497 dev->hw_desc[idx]->p2 = dev->key_phys_base;
498 dev->hw_desc[idx]->next = dev->hw_phys_desc[1];
499
500 dev->hw_desc[idx]->hdr = sahara_aes_key_hdr(dev);
501
502 idx++;
503 }
504
505 dev->nb_in_sg = sahara_sg_length(dev->in_sg, dev->total);
506 dev->nb_out_sg = sahara_sg_length(dev->out_sg, dev->total);
507 if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) {
508 dev_err(dev->device, "not enough hw links (%d)\n",
509 dev->nb_in_sg + dev->nb_out_sg);
510 return -EINVAL;
511 }
512
513 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
514 DMA_TO_DEVICE);
515 if (ret != dev->nb_in_sg) {
516 dev_err(dev->device, "couldn't map in sg\n");
517 goto unmap_in;
518 }
519 ret = dma_map_sg(dev->device, dev->out_sg, dev->nb_out_sg,
520 DMA_FROM_DEVICE);
521 if (ret != dev->nb_out_sg) {
522 dev_err(dev->device, "couldn't map out sg\n");
523 goto unmap_out;
524 }
525
526 /* Create input links */
527 dev->hw_desc[idx]->p1 = dev->hw_phys_link[0];
528 sg = dev->in_sg;
529 for (i = 0; i < dev->nb_in_sg; i++) {
530 dev->hw_link[i]->len = sg->length;
531 dev->hw_link[i]->p = sg->dma_address;
532 if (i == (dev->nb_in_sg - 1)) {
533 dev->hw_link[i]->next = 0;
534 } else {
535 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
536 sg = sg_next(sg);
537 }
538 }
539
540 /* Create output links */
541 dev->hw_desc[idx]->p2 = dev->hw_phys_link[i];
542 sg = dev->out_sg;
543 for (j = i; j < dev->nb_out_sg + i; j++) {
544 dev->hw_link[j]->len = sg->length;
545 dev->hw_link[j]->p = sg->dma_address;
546 if (j == (dev->nb_out_sg + i - 1)) {
547 dev->hw_link[j]->next = 0;
548 } else {
549 dev->hw_link[j]->next = dev->hw_phys_link[j + 1];
550 sg = sg_next(sg);
551 }
552 }
553
554 /* Fill remaining fields of hw_desc[1] */
555 dev->hw_desc[idx]->hdr = sahara_aes_data_link_hdr(dev);
556 dev->hw_desc[idx]->len1 = dev->total;
557 dev->hw_desc[idx]->len2 = dev->total;
558 dev->hw_desc[idx]->next = 0;
559
560 sahara_dump_descriptors(dev);
561 sahara_dump_links(dev);
562
563 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
564
565 return 0;
566
567 unmap_out:
568 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
569 DMA_TO_DEVICE);
570 unmap_in:
571 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
572 DMA_FROM_DEVICE);
573
574 return -EINVAL;
575 }
576
577 static int sahara_aes_process(struct ablkcipher_request *req)
578 {
579 struct sahara_dev *dev = dev_ptr;
580 struct sahara_ctx *ctx;
581 struct sahara_aes_reqctx *rctx;
582 int ret;
583 unsigned long timeout;
584
585 /* Request is ready to be dispatched by the device */
586 dev_dbg(dev->device,
587 "dispatch request (nbytes=%d, src=%p, dst=%p)\n",
588 req->nbytes, req->src, req->dst);
589
590 /* assign new request to device */
591 dev->total = req->nbytes;
592 dev->in_sg = req->src;
593 dev->out_sg = req->dst;
594
595 rctx = ablkcipher_request_ctx(req);
596 ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req));
597 rctx->mode &= FLAGS_MODE_MASK;
598 dev->flags = (dev->flags & ~FLAGS_MODE_MASK) | rctx->mode;
599
600 if ((dev->flags & FLAGS_CBC) && req->info)
601 memcpy(dev->iv_base, req->info, AES_KEYSIZE_128);
602
603 /* assign new context to device */
604 dev->ctx = ctx;
605
606 reinit_completion(&dev->dma_completion);
607
608 ret = sahara_hw_descriptor_create(dev);
609 if (ret)
610 return -EINVAL;
611
612 timeout = wait_for_completion_timeout(&dev->dma_completion,
613 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
614 if (!timeout) {
615 dev_err(dev->device, "AES timeout\n");
616 return -ETIMEDOUT;
617 }
618
619 dma_unmap_sg(dev->device, dev->out_sg, dev->nb_out_sg,
620 DMA_TO_DEVICE);
621 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
622 DMA_FROM_DEVICE);
623
624 return 0;
625 }
626
627 static int sahara_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key,
628 unsigned int keylen)
629 {
630 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(tfm);
631 int ret;
632
633 ctx->keylen = keylen;
634
635 /* SAHARA only supports 128bit keys */
636 if (keylen == AES_KEYSIZE_128) {
637 memcpy(ctx->key, key, keylen);
638 ctx->flags |= FLAGS_NEW_KEY;
639 return 0;
640 }
641
642 if (keylen != AES_KEYSIZE_128 &&
643 keylen != AES_KEYSIZE_192 && keylen != AES_KEYSIZE_256)
644 return -EINVAL;
645
646 /*
647 * The requested key size is not supported by HW, do a fallback.
648 */
649 ctx->fallback->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
650 ctx->fallback->base.crt_flags |=
651 (tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
652
653 ret = crypto_ablkcipher_setkey(ctx->fallback, key, keylen);
654 if (ret) {
655 struct crypto_tfm *tfm_aux = crypto_ablkcipher_tfm(tfm);
656
657 tfm_aux->crt_flags &= ~CRYPTO_TFM_RES_MASK;
658 tfm_aux->crt_flags |=
659 (ctx->fallback->base.crt_flags & CRYPTO_TFM_RES_MASK);
660 }
661 return ret;
662 }
663
664 static int sahara_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
665 {
666 struct sahara_aes_reqctx *rctx = ablkcipher_request_ctx(req);
667 struct sahara_dev *dev = dev_ptr;
668 int err = 0;
669
670 dev_dbg(dev->device, "nbytes: %d, enc: %d, cbc: %d\n",
671 req->nbytes, !!(mode & FLAGS_ENCRYPT), !!(mode & FLAGS_CBC));
672
673 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
674 dev_err(dev->device,
675 "request size is not exact amount of AES blocks\n");
676 return -EINVAL;
677 }
678
679 rctx->mode = mode;
680
681 mutex_lock(&dev->queue_mutex);
682 err = ablkcipher_enqueue_request(&dev->queue, req);
683 mutex_unlock(&dev->queue_mutex);
684
685 wake_up_process(dev->kthread);
686
687 return err;
688 }
689
690 static int sahara_aes_ecb_encrypt(struct ablkcipher_request *req)
691 {
692 struct crypto_tfm *tfm =
693 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
694 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
695 crypto_ablkcipher_reqtfm(req));
696 int err;
697
698 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
699 ablkcipher_request_set_tfm(req, ctx->fallback);
700 err = crypto_ablkcipher_encrypt(req);
701 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
702 return err;
703 }
704
705 return sahara_aes_crypt(req, FLAGS_ENCRYPT);
706 }
707
708 static int sahara_aes_ecb_decrypt(struct ablkcipher_request *req)
709 {
710 struct crypto_tfm *tfm =
711 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
712 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
713 crypto_ablkcipher_reqtfm(req));
714 int err;
715
716 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
717 ablkcipher_request_set_tfm(req, ctx->fallback);
718 err = crypto_ablkcipher_decrypt(req);
719 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
720 return err;
721 }
722
723 return sahara_aes_crypt(req, 0);
724 }
725
726 static int sahara_aes_cbc_encrypt(struct ablkcipher_request *req)
727 {
728 struct crypto_tfm *tfm =
729 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
730 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
731 crypto_ablkcipher_reqtfm(req));
732 int err;
733
734 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
735 ablkcipher_request_set_tfm(req, ctx->fallback);
736 err = crypto_ablkcipher_encrypt(req);
737 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
738 return err;
739 }
740
741 return sahara_aes_crypt(req, FLAGS_ENCRYPT | FLAGS_CBC);
742 }
743
744 static int sahara_aes_cbc_decrypt(struct ablkcipher_request *req)
745 {
746 struct crypto_tfm *tfm =
747 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req));
748 struct sahara_ctx *ctx = crypto_ablkcipher_ctx(
749 crypto_ablkcipher_reqtfm(req));
750 int err;
751
752 if (unlikely(ctx->keylen != AES_KEYSIZE_128)) {
753 ablkcipher_request_set_tfm(req, ctx->fallback);
754 err = crypto_ablkcipher_decrypt(req);
755 ablkcipher_request_set_tfm(req, __crypto_ablkcipher_cast(tfm));
756 return err;
757 }
758
759 return sahara_aes_crypt(req, FLAGS_CBC);
760 }
761
762 static int sahara_aes_cra_init(struct crypto_tfm *tfm)
763 {
764 const char *name = crypto_tfm_alg_name(tfm);
765 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
766
767 ctx->fallback = crypto_alloc_ablkcipher(name, 0,
768 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
769 if (IS_ERR(ctx->fallback)) {
770 pr_err("Error allocating fallback algo %s\n", name);
771 return PTR_ERR(ctx->fallback);
772 }
773
774 tfm->crt_ablkcipher.reqsize = sizeof(struct sahara_aes_reqctx);
775
776 return 0;
777 }
778
779 static void sahara_aes_cra_exit(struct crypto_tfm *tfm)
780 {
781 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
782
783 if (ctx->fallback)
784 crypto_free_ablkcipher(ctx->fallback);
785 ctx->fallback = NULL;
786 }
787
788 static u32 sahara_sha_init_hdr(struct sahara_dev *dev,
789 struct sahara_sha_reqctx *rctx)
790 {
791 u32 hdr = 0;
792
793 hdr = rctx->mode;
794
795 if (rctx->first) {
796 hdr |= SAHARA_HDR_MDHA_SET_MODE_HASH;
797 hdr |= SAHARA_HDR_MDHA_INIT;
798 } else {
799 hdr |= SAHARA_HDR_MDHA_SET_MODE_MD_KEY;
800 }
801
802 if (rctx->last)
803 hdr |= SAHARA_HDR_MDHA_PDATA;
804
805 if (hweight_long(hdr) % 2 == 0)
806 hdr |= SAHARA_HDR_PARITY_BIT;
807
808 return hdr;
809 }
810
811 static int sahara_sha_hw_links_create(struct sahara_dev *dev,
812 struct sahara_sha_reqctx *rctx,
813 int start)
814 {
815 struct scatterlist *sg;
816 unsigned int i;
817 int ret;
818
819 dev->in_sg = rctx->in_sg;
820
821 dev->nb_in_sg = sahara_sg_length(dev->in_sg, rctx->total);
822 if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) {
823 dev_err(dev->device, "not enough hw links (%d)\n",
824 dev->nb_in_sg + dev->nb_out_sg);
825 return -EINVAL;
826 }
827
828 if (rctx->in_sg_chained) {
829 i = start;
830 sg = dev->in_sg;
831 while (sg) {
832 ret = dma_map_sg(dev->device, sg, 1,
833 DMA_TO_DEVICE);
834 if (!ret)
835 return -EFAULT;
836
837 dev->hw_link[i]->len = sg->length;
838 dev->hw_link[i]->p = sg->dma_address;
839 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
840 sg = sg_next(sg);
841 i += 1;
842 }
843 dev->hw_link[i-1]->next = 0;
844 } else {
845 sg = dev->in_sg;
846 ret = dma_map_sg(dev->device, dev->in_sg, dev->nb_in_sg,
847 DMA_TO_DEVICE);
848 if (!ret)
849 return -EFAULT;
850
851 for (i = start; i < dev->nb_in_sg + start; i++) {
852 dev->hw_link[i]->len = sg->length;
853 dev->hw_link[i]->p = sg->dma_address;
854 if (i == (dev->nb_in_sg + start - 1)) {
855 dev->hw_link[i]->next = 0;
856 } else {
857 dev->hw_link[i]->next = dev->hw_phys_link[i + 1];
858 sg = sg_next(sg);
859 }
860 }
861 }
862
863 return i;
864 }
865
866 static int sahara_sha_hw_data_descriptor_create(struct sahara_dev *dev,
867 struct sahara_sha_reqctx *rctx,
868 struct ahash_request *req,
869 int index)
870 {
871 unsigned result_len;
872 int i = index;
873
874 if (rctx->first)
875 /* Create initial descriptor: #8*/
876 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
877 else
878 /* Create hash descriptor: #10. Must follow #6. */
879 dev->hw_desc[index]->hdr = SAHARA_HDR_MDHA_HASH;
880
881 dev->hw_desc[index]->len1 = rctx->total;
882 if (dev->hw_desc[index]->len1 == 0) {
883 /* if len1 is 0, p1 must be 0, too */
884 dev->hw_desc[index]->p1 = 0;
885 rctx->sg_in_idx = 0;
886 } else {
887 /* Create input links */
888 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
889 i = sahara_sha_hw_links_create(dev, rctx, index);
890
891 rctx->sg_in_idx = index;
892 if (i < 0)
893 return i;
894 }
895
896 dev->hw_desc[index]->p2 = dev->hw_phys_link[i];
897
898 /* Save the context for the next operation */
899 result_len = rctx->context_size;
900 dev->hw_link[i]->p = dev->context_phys_base;
901
902 dev->hw_link[i]->len = result_len;
903 dev->hw_desc[index]->len2 = result_len;
904
905 dev->hw_link[i]->next = 0;
906
907 return 0;
908 }
909
910 /*
911 * Load descriptor aka #6
912 *
913 * To load a previously saved context back to the MDHA unit
914 *
915 * p1: Saved Context
916 * p2: NULL
917 *
918 */
919 static int sahara_sha_hw_context_descriptor_create(struct sahara_dev *dev,
920 struct sahara_sha_reqctx *rctx,
921 struct ahash_request *req,
922 int index)
923 {
924 dev->hw_desc[index]->hdr = sahara_sha_init_hdr(dev, rctx);
925
926 dev->hw_desc[index]->len1 = rctx->context_size;
927 dev->hw_desc[index]->p1 = dev->hw_phys_link[index];
928 dev->hw_desc[index]->len2 = 0;
929 dev->hw_desc[index]->p2 = 0;
930
931 dev->hw_link[index]->len = rctx->context_size;
932 dev->hw_link[index]->p = dev->context_phys_base;
933 dev->hw_link[index]->next = 0;
934
935 return 0;
936 }
937
938 static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes)
939 {
940 if (!sg || !sg->length)
941 return nbytes;
942
943 while (nbytes && sg) {
944 if (nbytes <= sg->length) {
945 sg->length = nbytes;
946 sg_mark_end(sg);
947 break;
948 }
949 nbytes -= sg->length;
950 sg = sg_next(sg);
951 }
952
953 return nbytes;
954 }
955
956 static int sahara_sha_prepare_request(struct ahash_request *req)
957 {
958 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
959 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
960 unsigned int hash_later;
961 unsigned int block_size;
962 unsigned int len;
963
964 block_size = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
965
966 /* append bytes from previous operation */
967 len = rctx->buf_cnt + req->nbytes;
968
969 /* only the last transfer can be padded in hardware */
970 if (!rctx->last && (len < block_size)) {
971 /* to few data, save for next operation */
972 scatterwalk_map_and_copy(rctx->buf + rctx->buf_cnt, req->src,
973 0, req->nbytes, 0);
974 rctx->buf_cnt += req->nbytes;
975
976 return 0;
977 }
978
979 /* add data from previous operation first */
980 if (rctx->buf_cnt)
981 memcpy(rctx->rembuf, rctx->buf, rctx->buf_cnt);
982
983 /* data must always be a multiple of block_size */
984 hash_later = rctx->last ? 0 : len & (block_size - 1);
985 if (hash_later) {
986 unsigned int offset = req->nbytes - hash_later;
987 /* Save remaining bytes for later use */
988 scatterwalk_map_and_copy(rctx->buf, req->src, offset,
989 hash_later, 0);
990 }
991
992 /* nbytes should now be multiple of blocksize */
993 req->nbytes = req->nbytes - hash_later;
994
995 sahara_walk_and_recalc(req->src, req->nbytes);
996
997 /* have data from previous operation and current */
998 if (rctx->buf_cnt && req->nbytes) {
999 sg_init_table(rctx->in_sg_chain, 2);
1000 sg_set_buf(rctx->in_sg_chain, rctx->rembuf, rctx->buf_cnt);
1001
1002 scatterwalk_sg_chain(rctx->in_sg_chain, 2, req->src);
1003
1004 rctx->total = req->nbytes + rctx->buf_cnt;
1005 rctx->in_sg = rctx->in_sg_chain;
1006
1007 rctx->in_sg_chained = true;
1008 req->src = rctx->in_sg_chain;
1009 /* only data from previous operation */
1010 } else if (rctx->buf_cnt) {
1011 if (req->src)
1012 rctx->in_sg = req->src;
1013 else
1014 rctx->in_sg = rctx->in_sg_chain;
1015 /* buf was copied into rembuf above */
1016 sg_init_one(rctx->in_sg, rctx->rembuf, rctx->buf_cnt);
1017 rctx->total = rctx->buf_cnt;
1018 rctx->in_sg_chained = false;
1019 /* no data from previous operation */
1020 } else {
1021 rctx->in_sg = req->src;
1022 rctx->total = req->nbytes;
1023 req->src = rctx->in_sg;
1024 rctx->in_sg_chained = false;
1025 }
1026
1027 /* on next call, we only have the remaining data in the buffer */
1028 rctx->buf_cnt = hash_later;
1029
1030 return -EINPROGRESS;
1031 }
1032
1033 static void sahara_sha_unmap_sg(struct sahara_dev *dev,
1034 struct sahara_sha_reqctx *rctx)
1035 {
1036 struct scatterlist *sg;
1037
1038 if (rctx->in_sg_chained) {
1039 sg = dev->in_sg;
1040 while (sg) {
1041 dma_unmap_sg(dev->device, sg, 1, DMA_TO_DEVICE);
1042 sg = sg_next(sg);
1043 }
1044 } else {
1045 dma_unmap_sg(dev->device, dev->in_sg, dev->nb_in_sg,
1046 DMA_TO_DEVICE);
1047 }
1048 }
1049
1050 static int sahara_sha_process(struct ahash_request *req)
1051 {
1052 struct sahara_dev *dev = dev_ptr;
1053 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1054 int ret;
1055 unsigned long timeout;
1056
1057 ret = sahara_sha_prepare_request(req);
1058 if (!ret)
1059 return ret;
1060
1061 if (rctx->first) {
1062 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 0);
1063 dev->hw_desc[0]->next = 0;
1064 rctx->first = 0;
1065 } else {
1066 memcpy(dev->context_base, rctx->context, rctx->context_size);
1067
1068 sahara_sha_hw_context_descriptor_create(dev, rctx, req, 0);
1069 dev->hw_desc[0]->next = dev->hw_phys_desc[1];
1070 sahara_sha_hw_data_descriptor_create(dev, rctx, req, 1);
1071 dev->hw_desc[1]->next = 0;
1072 }
1073
1074 sahara_dump_descriptors(dev);
1075 sahara_dump_links(dev);
1076
1077 reinit_completion(&dev->dma_completion);
1078
1079 sahara_write(dev, dev->hw_phys_desc[0], SAHARA_REG_DAR);
1080
1081 timeout = wait_for_completion_timeout(&dev->dma_completion,
1082 msecs_to_jiffies(SAHARA_TIMEOUT_MS));
1083 if (!timeout) {
1084 dev_err(dev->device, "SHA timeout\n");
1085 return -ETIMEDOUT;
1086 }
1087
1088 if (rctx->sg_in_idx)
1089 sahara_sha_unmap_sg(dev, rctx);
1090
1091 memcpy(rctx->context, dev->context_base, rctx->context_size);
1092
1093 if (req->result)
1094 memcpy(req->result, rctx->context, rctx->digest_size);
1095
1096 return 0;
1097 }
1098
1099 static int sahara_queue_manage(void *data)
1100 {
1101 struct sahara_dev *dev = (struct sahara_dev *)data;
1102 struct crypto_async_request *async_req;
1103 struct crypto_async_request *backlog;
1104 int ret = 0;
1105
1106 do {
1107 __set_current_state(TASK_INTERRUPTIBLE);
1108
1109 mutex_lock(&dev->queue_mutex);
1110 backlog = crypto_get_backlog(&dev->queue);
1111 async_req = crypto_dequeue_request(&dev->queue);
1112 mutex_unlock(&dev->queue_mutex);
1113
1114 if (backlog)
1115 backlog->complete(backlog, -EINPROGRESS);
1116
1117 if (async_req) {
1118 if (crypto_tfm_alg_type(async_req->tfm) ==
1119 CRYPTO_ALG_TYPE_AHASH) {
1120 struct ahash_request *req =
1121 ahash_request_cast(async_req);
1122
1123 ret = sahara_sha_process(req);
1124 } else {
1125 struct ablkcipher_request *req =
1126 ablkcipher_request_cast(async_req);
1127
1128 ret = sahara_aes_process(req);
1129 }
1130
1131 async_req->complete(async_req, ret);
1132
1133 continue;
1134 }
1135
1136 schedule();
1137 } while (!kthread_should_stop());
1138
1139 return 0;
1140 }
1141
1142 static int sahara_sha_enqueue(struct ahash_request *req, int last)
1143 {
1144 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1145 struct sahara_dev *dev = dev_ptr;
1146 int ret;
1147
1148 if (!req->nbytes && !last)
1149 return 0;
1150
1151 mutex_lock(&rctx->mutex);
1152 rctx->last = last;
1153
1154 if (!rctx->active) {
1155 rctx->active = 1;
1156 rctx->first = 1;
1157 }
1158
1159 mutex_lock(&dev->queue_mutex);
1160 ret = crypto_enqueue_request(&dev->queue, &req->base);
1161 mutex_unlock(&dev->queue_mutex);
1162
1163 wake_up_process(dev->kthread);
1164 mutex_unlock(&rctx->mutex);
1165
1166 return ret;
1167 }
1168
1169 static int sahara_sha_init(struct ahash_request *req)
1170 {
1171 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1172 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1173
1174 memset(rctx, 0, sizeof(*rctx));
1175
1176 switch (crypto_ahash_digestsize(tfm)) {
1177 case SHA1_DIGEST_SIZE:
1178 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA1;
1179 rctx->digest_size = SHA1_DIGEST_SIZE;
1180 break;
1181 case SHA256_DIGEST_SIZE:
1182 rctx->mode |= SAHARA_HDR_MDHA_ALG_SHA256;
1183 rctx->digest_size = SHA256_DIGEST_SIZE;
1184 break;
1185 default:
1186 return -EINVAL;
1187 }
1188
1189 rctx->context_size = rctx->digest_size + 4;
1190 rctx->active = 0;
1191
1192 mutex_init(&rctx->mutex);
1193
1194 return 0;
1195 }
1196
1197 static int sahara_sha_update(struct ahash_request *req)
1198 {
1199 return sahara_sha_enqueue(req, 0);
1200 }
1201
1202 static int sahara_sha_final(struct ahash_request *req)
1203 {
1204 req->nbytes = 0;
1205 return sahara_sha_enqueue(req, 1);
1206 }
1207
1208 static int sahara_sha_finup(struct ahash_request *req)
1209 {
1210 return sahara_sha_enqueue(req, 1);
1211 }
1212
1213 static int sahara_sha_digest(struct ahash_request *req)
1214 {
1215 sahara_sha_init(req);
1216
1217 return sahara_sha_finup(req);
1218 }
1219
1220 static int sahara_sha_export(struct ahash_request *req, void *out)
1221 {
1222 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1223 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1224 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1225
1226 memcpy(out, ctx, sizeof(struct sahara_ctx));
1227 memcpy(out + sizeof(struct sahara_sha_reqctx), rctx,
1228 sizeof(struct sahara_sha_reqctx));
1229
1230 return 0;
1231 }
1232
1233 static int sahara_sha_import(struct ahash_request *req, const void *in)
1234 {
1235 struct crypto_ahash *ahash = crypto_ahash_reqtfm(req);
1236 struct sahara_ctx *ctx = crypto_ahash_ctx(ahash);
1237 struct sahara_sha_reqctx *rctx = ahash_request_ctx(req);
1238
1239 memcpy(ctx, in, sizeof(struct sahara_ctx));
1240 memcpy(rctx, in + sizeof(struct sahara_sha_reqctx),
1241 sizeof(struct sahara_sha_reqctx));
1242
1243 return 0;
1244 }
1245
1246 static int sahara_sha_cra_init(struct crypto_tfm *tfm)
1247 {
1248 const char *name = crypto_tfm_alg_name(tfm);
1249 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1250
1251 ctx->shash_fallback = crypto_alloc_shash(name, 0,
1252 CRYPTO_ALG_NEED_FALLBACK);
1253 if (IS_ERR(ctx->shash_fallback)) {
1254 pr_err("Error allocating fallback algo %s\n", name);
1255 return PTR_ERR(ctx->shash_fallback);
1256 }
1257 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1258 sizeof(struct sahara_sha_reqctx) +
1259 SHA_BUFFER_LEN + SHA256_BLOCK_SIZE);
1260
1261 return 0;
1262 }
1263
1264 static void sahara_sha_cra_exit(struct crypto_tfm *tfm)
1265 {
1266 struct sahara_ctx *ctx = crypto_tfm_ctx(tfm);
1267
1268 crypto_free_shash(ctx->shash_fallback);
1269 ctx->shash_fallback = NULL;
1270 }
1271
1272 static struct crypto_alg aes_algs[] = {
1273 {
1274 .cra_name = "ecb(aes)",
1275 .cra_driver_name = "sahara-ecb-aes",
1276 .cra_priority = 300,
1277 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1278 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1279 .cra_blocksize = AES_BLOCK_SIZE,
1280 .cra_ctxsize = sizeof(struct sahara_ctx),
1281 .cra_alignmask = 0x0,
1282 .cra_type = &crypto_ablkcipher_type,
1283 .cra_module = THIS_MODULE,
1284 .cra_init = sahara_aes_cra_init,
1285 .cra_exit = sahara_aes_cra_exit,
1286 .cra_u.ablkcipher = {
1287 .min_keysize = AES_MIN_KEY_SIZE ,
1288 .max_keysize = AES_MAX_KEY_SIZE,
1289 .setkey = sahara_aes_setkey,
1290 .encrypt = sahara_aes_ecb_encrypt,
1291 .decrypt = sahara_aes_ecb_decrypt,
1292 }
1293 }, {
1294 .cra_name = "cbc(aes)",
1295 .cra_driver_name = "sahara-cbc-aes",
1296 .cra_priority = 300,
1297 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1298 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK,
1299 .cra_blocksize = AES_BLOCK_SIZE,
1300 .cra_ctxsize = sizeof(struct sahara_ctx),
1301 .cra_alignmask = 0x0,
1302 .cra_type = &crypto_ablkcipher_type,
1303 .cra_module = THIS_MODULE,
1304 .cra_init = sahara_aes_cra_init,
1305 .cra_exit = sahara_aes_cra_exit,
1306 .cra_u.ablkcipher = {
1307 .min_keysize = AES_MIN_KEY_SIZE ,
1308 .max_keysize = AES_MAX_KEY_SIZE,
1309 .ivsize = AES_BLOCK_SIZE,
1310 .setkey = sahara_aes_setkey,
1311 .encrypt = sahara_aes_cbc_encrypt,
1312 .decrypt = sahara_aes_cbc_decrypt,
1313 }
1314 }
1315 };
1316
1317 static struct ahash_alg sha_v3_algs[] = {
1318 {
1319 .init = sahara_sha_init,
1320 .update = sahara_sha_update,
1321 .final = sahara_sha_final,
1322 .finup = sahara_sha_finup,
1323 .digest = sahara_sha_digest,
1324 .export = sahara_sha_export,
1325 .import = sahara_sha_import,
1326 .halg.digestsize = SHA1_DIGEST_SIZE,
1327 .halg.base = {
1328 .cra_name = "sha1",
1329 .cra_driver_name = "sahara-sha1",
1330 .cra_priority = 300,
1331 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1332 CRYPTO_ALG_ASYNC |
1333 CRYPTO_ALG_NEED_FALLBACK,
1334 .cra_blocksize = SHA1_BLOCK_SIZE,
1335 .cra_ctxsize = sizeof(struct sahara_ctx),
1336 .cra_alignmask = 0,
1337 .cra_module = THIS_MODULE,
1338 .cra_init = sahara_sha_cra_init,
1339 .cra_exit = sahara_sha_cra_exit,
1340 }
1341 },
1342 };
1343
1344 static struct ahash_alg sha_v4_algs[] = {
1345 {
1346 .init = sahara_sha_init,
1347 .update = sahara_sha_update,
1348 .final = sahara_sha_final,
1349 .finup = sahara_sha_finup,
1350 .digest = sahara_sha_digest,
1351 .export = sahara_sha_export,
1352 .import = sahara_sha_import,
1353 .halg.digestsize = SHA256_DIGEST_SIZE,
1354 .halg.base = {
1355 .cra_name = "sha256",
1356 .cra_driver_name = "sahara-sha256",
1357 .cra_priority = 300,
1358 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1359 CRYPTO_ALG_ASYNC |
1360 CRYPTO_ALG_NEED_FALLBACK,
1361 .cra_blocksize = SHA256_BLOCK_SIZE,
1362 .cra_ctxsize = sizeof(struct sahara_ctx),
1363 .cra_alignmask = 0,
1364 .cra_module = THIS_MODULE,
1365 .cra_init = sahara_sha_cra_init,
1366 .cra_exit = sahara_sha_cra_exit,
1367 }
1368 },
1369 };
1370
1371 static irqreturn_t sahara_irq_handler(int irq, void *data)
1372 {
1373 struct sahara_dev *dev = (struct sahara_dev *)data;
1374 unsigned int stat = sahara_read(dev, SAHARA_REG_STATUS);
1375 unsigned int err = sahara_read(dev, SAHARA_REG_ERRSTATUS);
1376
1377 sahara_write(dev, SAHARA_CMD_CLEAR_INT | SAHARA_CMD_CLEAR_ERR,
1378 SAHARA_REG_CMD);
1379
1380 sahara_decode_status(dev, stat);
1381
1382 if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_BUSY) {
1383 return IRQ_NONE;
1384 } else if (SAHARA_STATUS_GET_STATE(stat) == SAHARA_STATE_COMPLETE) {
1385 dev->error = 0;
1386 } else {
1387 sahara_decode_error(dev, err);
1388 dev->error = -EINVAL;
1389 }
1390
1391 complete(&dev->dma_completion);
1392
1393 return IRQ_HANDLED;
1394 }
1395
1396
1397 static int sahara_register_algs(struct sahara_dev *dev)
1398 {
1399 int err;
1400 unsigned int i, j, k, l;
1401
1402 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
1403 INIT_LIST_HEAD(&aes_algs[i].cra_list);
1404 err = crypto_register_alg(&aes_algs[i]);
1405 if (err)
1406 goto err_aes_algs;
1407 }
1408
1409 for (k = 0; k < ARRAY_SIZE(sha_v3_algs); k++) {
1410 err = crypto_register_ahash(&sha_v3_algs[k]);
1411 if (err)
1412 goto err_sha_v3_algs;
1413 }
1414
1415 if (dev->version > SAHARA_VERSION_3)
1416 for (l = 0; l < ARRAY_SIZE(sha_v4_algs); l++) {
1417 err = crypto_register_ahash(&sha_v4_algs[l]);
1418 if (err)
1419 goto err_sha_v4_algs;
1420 }
1421
1422 return 0;
1423
1424 err_sha_v4_algs:
1425 for (j = 0; j < l; j++)
1426 crypto_unregister_ahash(&sha_v4_algs[j]);
1427
1428 err_sha_v3_algs:
1429 for (j = 0; j < k; j++)
1430 crypto_unregister_ahash(&sha_v4_algs[j]);
1431
1432 err_aes_algs:
1433 for (j = 0; j < i; j++)
1434 crypto_unregister_alg(&aes_algs[j]);
1435
1436 return err;
1437 }
1438
1439 static void sahara_unregister_algs(struct sahara_dev *dev)
1440 {
1441 unsigned int i;
1442
1443 for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
1444 crypto_unregister_alg(&aes_algs[i]);
1445
1446 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1447 crypto_unregister_ahash(&sha_v3_algs[i]);
1448
1449 if (dev->version > SAHARA_VERSION_3)
1450 for (i = 0; i < ARRAY_SIZE(sha_v4_algs); i++)
1451 crypto_unregister_ahash(&sha_v4_algs[i]);
1452 }
1453
1454 static struct platform_device_id sahara_platform_ids[] = {
1455 { .name = "sahara-imx27" },
1456 { /* sentinel */ }
1457 };
1458 MODULE_DEVICE_TABLE(platform, sahara_platform_ids);
1459
1460 static struct of_device_id sahara_dt_ids[] = {
1461 { .compatible = "fsl,imx53-sahara" },
1462 { .compatible = "fsl,imx27-sahara" },
1463 { /* sentinel */ }
1464 };
1465 MODULE_DEVICE_TABLE(of, sahara_dt_ids);
1466
1467 static int sahara_probe(struct platform_device *pdev)
1468 {
1469 struct sahara_dev *dev;
1470 struct resource *res;
1471 u32 version;
1472 int irq;
1473 int err;
1474 int i;
1475
1476 dev = devm_kzalloc(&pdev->dev, sizeof(struct sahara_dev), GFP_KERNEL);
1477 if (dev == NULL) {
1478 dev_err(&pdev->dev, "unable to alloc data struct.\n");
1479 return -ENOMEM;
1480 }
1481
1482 dev->device = &pdev->dev;
1483 platform_set_drvdata(pdev, dev);
1484
1485 /* Get the base address */
1486 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1487 dev->regs_base = devm_ioremap_resource(&pdev->dev, res);
1488 if (IS_ERR(dev->regs_base))
1489 return PTR_ERR(dev->regs_base);
1490
1491 /* Get the IRQ */
1492 irq = platform_get_irq(pdev, 0);
1493 if (irq < 0) {
1494 dev_err(&pdev->dev, "failed to get irq resource\n");
1495 return irq;
1496 }
1497
1498 err = devm_request_irq(&pdev->dev, irq, sahara_irq_handler,
1499 0, dev_name(&pdev->dev), dev);
1500 if (err) {
1501 dev_err(&pdev->dev, "failed to request irq\n");
1502 return err;
1503 }
1504
1505 /* clocks */
1506 dev->clk_ipg = devm_clk_get(&pdev->dev, "ipg");
1507 if (IS_ERR(dev->clk_ipg)) {
1508 dev_err(&pdev->dev, "Could not get ipg clock\n");
1509 return PTR_ERR(dev->clk_ipg);
1510 }
1511
1512 dev->clk_ahb = devm_clk_get(&pdev->dev, "ahb");
1513 if (IS_ERR(dev->clk_ahb)) {
1514 dev_err(&pdev->dev, "Could not get ahb clock\n");
1515 return PTR_ERR(dev->clk_ahb);
1516 }
1517
1518 /* Allocate HW descriptors */
1519 dev->hw_desc[0] = dma_alloc_coherent(&pdev->dev,
1520 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1521 &dev->hw_phys_desc[0], GFP_KERNEL);
1522 if (!dev->hw_desc[0]) {
1523 dev_err(&pdev->dev, "Could not allocate hw descriptors\n");
1524 return -ENOMEM;
1525 }
1526 dev->hw_desc[1] = dev->hw_desc[0] + 1;
1527 dev->hw_phys_desc[1] = dev->hw_phys_desc[0] +
1528 sizeof(struct sahara_hw_desc);
1529
1530 /* Allocate space for iv and key */
1531 dev->key_base = dma_alloc_coherent(&pdev->dev, 2 * AES_KEYSIZE_128,
1532 &dev->key_phys_base, GFP_KERNEL);
1533 if (!dev->key_base) {
1534 dev_err(&pdev->dev, "Could not allocate memory for key\n");
1535 err = -ENOMEM;
1536 goto err_key;
1537 }
1538 dev->iv_base = dev->key_base + AES_KEYSIZE_128;
1539 dev->iv_phys_base = dev->key_phys_base + AES_KEYSIZE_128;
1540
1541 /* Allocate space for context: largest digest + message length field */
1542 dev->context_base = dma_alloc_coherent(&pdev->dev,
1543 SHA256_DIGEST_SIZE + 4,
1544 &dev->context_phys_base, GFP_KERNEL);
1545 if (!dev->context_base) {
1546 dev_err(&pdev->dev, "Could not allocate memory for MDHA context\n");
1547 err = -ENOMEM;
1548 goto err_key;
1549 }
1550
1551 /* Allocate space for HW links */
1552 dev->hw_link[0] = dma_alloc_coherent(&pdev->dev,
1553 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1554 &dev->hw_phys_link[0], GFP_KERNEL);
1555 if (!dev->hw_link[0]) {
1556 dev_err(&pdev->dev, "Could not allocate hw links\n");
1557 err = -ENOMEM;
1558 goto err_link;
1559 }
1560 for (i = 1; i < SAHARA_MAX_HW_LINK; i++) {
1561 dev->hw_phys_link[i] = dev->hw_phys_link[i - 1] +
1562 sizeof(struct sahara_hw_link);
1563 dev->hw_link[i] = dev->hw_link[i - 1] + 1;
1564 }
1565
1566 crypto_init_queue(&dev->queue, SAHARA_QUEUE_LENGTH);
1567
1568 spin_lock_init(&dev->lock);
1569 mutex_init(&dev->queue_mutex);
1570
1571 dev_ptr = dev;
1572
1573 dev->kthread = kthread_run(sahara_queue_manage, dev, "sahara_crypto");
1574 if (IS_ERR(dev->kthread)) {
1575 err = PTR_ERR(dev->kthread);
1576 goto err_link;
1577 }
1578
1579 init_completion(&dev->dma_completion);
1580
1581 clk_prepare_enable(dev->clk_ipg);
1582 clk_prepare_enable(dev->clk_ahb);
1583
1584 version = sahara_read(dev, SAHARA_REG_VERSION);
1585 if (of_device_is_compatible(pdev->dev.of_node, "fsl,imx27-sahara")) {
1586 if (version != SAHARA_VERSION_3)
1587 err = -ENODEV;
1588 } else if (of_device_is_compatible(pdev->dev.of_node,
1589 "fsl,imx53-sahara")) {
1590 if (((version >> 8) & 0xff) != SAHARA_VERSION_4)
1591 err = -ENODEV;
1592 version = (version >> 8) & 0xff;
1593 }
1594 if (err == -ENODEV) {
1595 dev_err(&pdev->dev, "SAHARA version %d not supported\n",
1596 version);
1597 goto err_algs;
1598 }
1599
1600 dev->version = version;
1601
1602 sahara_write(dev, SAHARA_CMD_RESET | SAHARA_CMD_MODE_BATCH,
1603 SAHARA_REG_CMD);
1604 sahara_write(dev, SAHARA_CONTROL_SET_THROTTLE(0) |
1605 SAHARA_CONTROL_SET_MAXBURST(8) |
1606 SAHARA_CONTROL_RNG_AUTORSD |
1607 SAHARA_CONTROL_ENABLE_INT,
1608 SAHARA_REG_CONTROL);
1609
1610 err = sahara_register_algs(dev);
1611 if (err)
1612 goto err_algs;
1613
1614 dev_info(&pdev->dev, "SAHARA version %d initialized\n", version);
1615
1616 return 0;
1617
1618 err_algs:
1619 dma_free_coherent(&pdev->dev,
1620 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1621 dev->hw_link[0], dev->hw_phys_link[0]);
1622 clk_disable_unprepare(dev->clk_ipg);
1623 clk_disable_unprepare(dev->clk_ahb);
1624 kthread_stop(dev->kthread);
1625 dev_ptr = NULL;
1626 err_link:
1627 dma_free_coherent(&pdev->dev,
1628 2 * AES_KEYSIZE_128,
1629 dev->key_base, dev->key_phys_base);
1630 dma_free_coherent(&pdev->dev,
1631 SHA256_DIGEST_SIZE,
1632 dev->context_base, dev->context_phys_base);
1633 err_key:
1634 dma_free_coherent(&pdev->dev,
1635 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1636 dev->hw_desc[0], dev->hw_phys_desc[0]);
1637
1638 return err;
1639 }
1640
1641 static int sahara_remove(struct platform_device *pdev)
1642 {
1643 struct sahara_dev *dev = platform_get_drvdata(pdev);
1644
1645 dma_free_coherent(&pdev->dev,
1646 SAHARA_MAX_HW_LINK * sizeof(struct sahara_hw_link),
1647 dev->hw_link[0], dev->hw_phys_link[0]);
1648 dma_free_coherent(&pdev->dev,
1649 2 * AES_KEYSIZE_128,
1650 dev->key_base, dev->key_phys_base);
1651 dma_free_coherent(&pdev->dev,
1652 SAHARA_MAX_HW_DESC * sizeof(struct sahara_hw_desc),
1653 dev->hw_desc[0], dev->hw_phys_desc[0]);
1654
1655 kthread_stop(dev->kthread);
1656
1657 sahara_unregister_algs(dev);
1658
1659 clk_disable_unprepare(dev->clk_ipg);
1660 clk_disable_unprepare(dev->clk_ahb);
1661
1662 dev_ptr = NULL;
1663
1664 return 0;
1665 }
1666
1667 static struct platform_driver sahara_driver = {
1668 .probe = sahara_probe,
1669 .remove = sahara_remove,
1670 .driver = {
1671 .name = SAHARA_NAME,
1672 .of_match_table = sahara_dt_ids,
1673 },
1674 .id_table = sahara_platform_ids,
1675 };
1676
1677 module_platform_driver(sahara_driver);
1678
1679 MODULE_LICENSE("GPL");
1680 MODULE_AUTHOR("Javier Martin <javier.martin@vista-silicon.com>");
1681 MODULE_AUTHOR("Steffen Trumtrar <s.trumtrar@pengutronix.de>");
1682 MODULE_DESCRIPTION("SAHARA2 HW crypto accelerator");
Linux with added FPC-III support