?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / crypto / s5p-sss.c
blob5d64c08b7f47ef412916b804256716f6f8016779
1 // SPDX-License-Identifier: GPL-2.0
2 //
3 // Cryptographic API.
4 //
5 // Support for Samsung S5PV210 and Exynos HW acceleration.
6 //
7 // Copyright (C) 2011 NetUP Inc. All rights reserved.
8 // Copyright (c) 2017 Samsung Electronics Co., Ltd. All rights reserved.
9 //
10 // Hash part based on omap-sham.c driver.
11
12 #include <linux/clk.h>
13 #include <linux/crypto.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/err.h>
16 #include <linux/errno.h>
17 #include <linux/init.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/of.h>
23 #include <linux/platform_device.h>
24 #include <linux/scatterlist.h>
25
26 #include <crypto/ctr.h>
27 #include <crypto/aes.h>
28 #include <crypto/algapi.h>
29 #include <crypto/scatterwalk.h>
30
31 #include <crypto/hash.h>
32 #include <crypto/md5.h>
33 #include <crypto/sha.h>
34 #include <crypto/internal/hash.h>
35
36 #define _SBF(s, v) ((v) << (s))
37
38 /* Feed control registers */
39 #define SSS_REG_FCINTSTAT 0x0000
40 #define SSS_FCINTSTAT_HPARTINT BIT(7)
41 #define SSS_FCINTSTAT_HDONEINT BIT(5)
42 #define SSS_FCINTSTAT_BRDMAINT BIT(3)
43 #define SSS_FCINTSTAT_BTDMAINT BIT(2)
44 #define SSS_FCINTSTAT_HRDMAINT BIT(1)
45 #define SSS_FCINTSTAT_PKDMAINT BIT(0)
46
47 #define SSS_REG_FCINTENSET 0x0004
48 #define SSS_FCINTENSET_HPARTINTENSET BIT(7)
49 #define SSS_FCINTENSET_HDONEINTENSET BIT(5)
50 #define SSS_FCINTENSET_BRDMAINTENSET BIT(3)
51 #define SSS_FCINTENSET_BTDMAINTENSET BIT(2)
52 #define SSS_FCINTENSET_HRDMAINTENSET BIT(1)
53 #define SSS_FCINTENSET_PKDMAINTENSET BIT(0)
54
55 #define SSS_REG_FCINTENCLR 0x0008
56 #define SSS_FCINTENCLR_HPARTINTENCLR BIT(7)
57 #define SSS_FCINTENCLR_HDONEINTENCLR BIT(5)
58 #define SSS_FCINTENCLR_BRDMAINTENCLR BIT(3)
59 #define SSS_FCINTENCLR_BTDMAINTENCLR BIT(2)
60 #define SSS_FCINTENCLR_HRDMAINTENCLR BIT(1)
61 #define SSS_FCINTENCLR_PKDMAINTENCLR BIT(0)
62
63 #define SSS_REG_FCINTPEND 0x000C
64 #define SSS_FCINTPEND_HPARTINTP BIT(7)
65 #define SSS_FCINTPEND_HDONEINTP BIT(5)
66 #define SSS_FCINTPEND_BRDMAINTP BIT(3)
67 #define SSS_FCINTPEND_BTDMAINTP BIT(2)
68 #define SSS_FCINTPEND_HRDMAINTP BIT(1)
69 #define SSS_FCINTPEND_PKDMAINTP BIT(0)
70
71 #define SSS_REG_FCFIFOSTAT 0x0010
72 #define SSS_FCFIFOSTAT_BRFIFOFUL BIT(7)
73 #define SSS_FCFIFOSTAT_BRFIFOEMP BIT(6)
74 #define SSS_FCFIFOSTAT_BTFIFOFUL BIT(5)
75 #define SSS_FCFIFOSTAT_BTFIFOEMP BIT(4)
76 #define SSS_FCFIFOSTAT_HRFIFOFUL BIT(3)
77 #define SSS_FCFIFOSTAT_HRFIFOEMP BIT(2)
78 #define SSS_FCFIFOSTAT_PKFIFOFUL BIT(1)
79 #define SSS_FCFIFOSTAT_PKFIFOEMP BIT(0)
80
81 #define SSS_REG_FCFIFOCTRL 0x0014
82 #define SSS_FCFIFOCTRL_DESSEL BIT(2)
83 #define SSS_HASHIN_INDEPENDENT _SBF(0, 0x00)
84 #define SSS_HASHIN_CIPHER_INPUT _SBF(0, 0x01)
85 #define SSS_HASHIN_CIPHER_OUTPUT _SBF(0, 0x02)
86 #define SSS_HASHIN_MASK _SBF(0, 0x03)
87
88 #define SSS_REG_FCBRDMAS 0x0020
89 #define SSS_REG_FCBRDMAL 0x0024
90 #define SSS_REG_FCBRDMAC 0x0028
91 #define SSS_FCBRDMAC_BYTESWAP BIT(1)
92 #define SSS_FCBRDMAC_FLUSH BIT(0)
93
94 #define SSS_REG_FCBTDMAS 0x0030
95 #define SSS_REG_FCBTDMAL 0x0034
96 #define SSS_REG_FCBTDMAC 0x0038
97 #define SSS_FCBTDMAC_BYTESWAP BIT(1)
98 #define SSS_FCBTDMAC_FLUSH BIT(0)
99
100 #define SSS_REG_FCHRDMAS 0x0040
101 #define SSS_REG_FCHRDMAL 0x0044
102 #define SSS_REG_FCHRDMAC 0x0048
103 #define SSS_FCHRDMAC_BYTESWAP BIT(1)
104 #define SSS_FCHRDMAC_FLUSH BIT(0)
105
106 #define SSS_REG_FCPKDMAS 0x0050
107 #define SSS_REG_FCPKDMAL 0x0054
108 #define SSS_REG_FCPKDMAC 0x0058
109 #define SSS_FCPKDMAC_BYTESWAP BIT(3)
110 #define SSS_FCPKDMAC_DESCEND BIT(2)
111 #define SSS_FCPKDMAC_TRANSMIT BIT(1)
112 #define SSS_FCPKDMAC_FLUSH BIT(0)
113
114 #define SSS_REG_FCPKDMAO 0x005C
115
116 /* AES registers */
117 #define SSS_REG_AES_CONTROL 0x00
118 #define SSS_AES_BYTESWAP_DI BIT(11)
119 #define SSS_AES_BYTESWAP_DO BIT(10)
120 #define SSS_AES_BYTESWAP_IV BIT(9)
121 #define SSS_AES_BYTESWAP_CNT BIT(8)
122 #define SSS_AES_BYTESWAP_KEY BIT(7)
123 #define SSS_AES_KEY_CHANGE_MODE BIT(6)
124 #define SSS_AES_KEY_SIZE_128 _SBF(4, 0x00)
125 #define SSS_AES_KEY_SIZE_192 _SBF(4, 0x01)
126 #define SSS_AES_KEY_SIZE_256 _SBF(4, 0x02)
127 #define SSS_AES_FIFO_MODE BIT(3)
128 #define SSS_AES_CHAIN_MODE_ECB _SBF(1, 0x00)
129 #define SSS_AES_CHAIN_MODE_CBC _SBF(1, 0x01)
130 #define SSS_AES_CHAIN_MODE_CTR _SBF(1, 0x02)
131 #define SSS_AES_MODE_DECRYPT BIT(0)
132
133 #define SSS_REG_AES_STATUS 0x04
134 #define SSS_AES_BUSY BIT(2)
135 #define SSS_AES_INPUT_READY BIT(1)
136 #define SSS_AES_OUTPUT_READY BIT(0)
137
138 #define SSS_REG_AES_IN_DATA(s) (0x10 + (s << 2))
139 #define SSS_REG_AES_OUT_DATA(s) (0x20 + (s << 2))
140 #define SSS_REG_AES_IV_DATA(s) (0x30 + (s << 2))
141 #define SSS_REG_AES_CNT_DATA(s) (0x40 + (s << 2))
142 #define SSS_REG_AES_KEY_DATA(s) (0x80 + (s << 2))
143
144 #define SSS_REG(dev, reg) ((dev)->ioaddr + (SSS_REG_##reg))
145 #define SSS_READ(dev, reg) __raw_readl(SSS_REG(dev, reg))
146 #define SSS_WRITE(dev, reg, val) __raw_writel((val), SSS_REG(dev, reg))
147
148 #define SSS_AES_REG(dev, reg) ((dev)->aes_ioaddr + SSS_REG_##reg)
149 #define SSS_AES_WRITE(dev, reg, val) __raw_writel((val), \
150 SSS_AES_REG(dev, reg))
151
152 /* HW engine modes */
153 #define FLAGS_AES_DECRYPT BIT(0)
154 #define FLAGS_AES_MODE_MASK _SBF(1, 0x03)
155 #define FLAGS_AES_CBC _SBF(1, 0x01)
156 #define FLAGS_AES_CTR _SBF(1, 0x02)
157
158 #define AES_KEY_LEN 16
159 #define CRYPTO_QUEUE_LEN 1
160
161 /* HASH registers */
162 #define SSS_REG_HASH_CTRL 0x00
163
164 #define SSS_HASH_USER_IV_EN BIT(5)
165 #define SSS_HASH_INIT_BIT BIT(4)
166 #define SSS_HASH_ENGINE_SHA1 _SBF(1, 0x00)
167 #define SSS_HASH_ENGINE_MD5 _SBF(1, 0x01)
168 #define SSS_HASH_ENGINE_SHA256 _SBF(1, 0x02)
169
170 #define SSS_HASH_ENGINE_MASK _SBF(1, 0x03)
171
172 #define SSS_REG_HASH_CTRL_PAUSE 0x04
173
174 #define SSS_HASH_PAUSE BIT(0)
175
176 #define SSS_REG_HASH_CTRL_FIFO 0x08
177
178 #define SSS_HASH_FIFO_MODE_DMA BIT(0)
179 #define SSS_HASH_FIFO_MODE_CPU 0
180
181 #define SSS_REG_HASH_CTRL_SWAP 0x0C
182
183 #define SSS_HASH_BYTESWAP_DI BIT(3)
184 #define SSS_HASH_BYTESWAP_DO BIT(2)
185 #define SSS_HASH_BYTESWAP_IV BIT(1)
186 #define SSS_HASH_BYTESWAP_KEY BIT(0)
187
188 #define SSS_REG_HASH_STATUS 0x10
189
190 #define SSS_HASH_STATUS_MSG_DONE BIT(6)
191 #define SSS_HASH_STATUS_PARTIAL_DONE BIT(4)
192 #define SSS_HASH_STATUS_BUFFER_READY BIT(0)
193
194 #define SSS_REG_HASH_MSG_SIZE_LOW 0x20
195 #define SSS_REG_HASH_MSG_SIZE_HIGH 0x24
196
197 #define SSS_REG_HASH_PRE_MSG_SIZE_LOW 0x28
198 #define SSS_REG_HASH_PRE_MSG_SIZE_HIGH 0x2C
199
200 #define SSS_REG_HASH_IV(s) (0xB0 + ((s) << 2))
201 #define SSS_REG_HASH_OUT(s) (0x100 + ((s) << 2))
202
203 #define HASH_BLOCK_SIZE 64
204 #define HASH_REG_SIZEOF 4
205 #define HASH_MD5_MAX_REG (MD5_DIGEST_SIZE / HASH_REG_SIZEOF)
206 #define HASH_SHA1_MAX_REG (SHA1_DIGEST_SIZE / HASH_REG_SIZEOF)
207 #define HASH_SHA256_MAX_REG (SHA256_DIGEST_SIZE / HASH_REG_SIZEOF)
208
209 /*
210 * HASH bit numbers, used by device, setting in dev->hash_flags with
211 * functions set_bit(), clear_bit() or tested with test_bit() or BIT(),
212 * to keep HASH state BUSY or FREE, or to signal state from irq_handler
213 * to hash_tasklet. SGS keep track of allocated memory for scatterlist
214 */
215 #define HASH_FLAGS_BUSY 0
216 #define HASH_FLAGS_FINAL 1
217 #define HASH_FLAGS_DMA_ACTIVE 2
218 #define HASH_FLAGS_OUTPUT_READY 3
219 #define HASH_FLAGS_DMA_READY 4
220 #define HASH_FLAGS_SGS_COPIED 5
221 #define HASH_FLAGS_SGS_ALLOCED 6
222
223 /* HASH HW constants */
224 #define BUFLEN HASH_BLOCK_SIZE
225
226 #define SSS_HASH_DMA_LEN_ALIGN 8
227 #define SSS_HASH_DMA_ALIGN_MASK (SSS_HASH_DMA_LEN_ALIGN - 1)
228
229 #define SSS_HASH_QUEUE_LENGTH 10
230
231 /**
232 * struct samsung_aes_variant - platform specific SSS driver data
233 * @aes_offset: AES register offset from SSS module's base.
234 * @hash_offset: HASH register offset from SSS module's base.
235 *
236 * Specifies platform specific configuration of SSS module.
237 * Note: A structure for driver specific platform data is used for future
238 * expansion of its usage.
239 */
240 struct samsung_aes_variant {
241 unsigned int aes_offset;
242 unsigned int hash_offset;
243 };
244
245 struct s5p_aes_reqctx {
246 unsigned long mode;
247 };
248
249 struct s5p_aes_ctx {
250 struct s5p_aes_dev *dev;
251
252 uint8_t aes_key[AES_MAX_KEY_SIZE];
253 uint8_t nonce[CTR_RFC3686_NONCE_SIZE];
254 int keylen;
255 };
256
257 /**
258 * struct s5p_aes_dev - Crypto device state container
259 * @dev: Associated device
260 * @clk: Clock for accessing hardware
261 * @ioaddr: Mapped IO memory region
262 * @aes_ioaddr: Per-varian offset for AES block IO memory
263 * @irq_fc: Feed control interrupt line
264 * @req: Crypto request currently handled by the device
265 * @ctx: Configuration for currently handled crypto request
266 * @sg_src: Scatter list with source data for currently handled block
267 * in device. This is DMA-mapped into device.
268 * @sg_dst: Scatter list with destination data for currently handled block
269 * in device. This is DMA-mapped into device.
270 * @sg_src_cpy: In case of unaligned access, copied scatter list
271 * with source data.
272 * @sg_dst_cpy: In case of unaligned access, copied scatter list
273 * with destination data.
274 * @tasklet: New request scheduling jib
275 * @queue: Crypto queue
276 * @busy: Indicates whether the device is currently handling some request
277 * thus it uses some of the fields from this state, like:
278 * req, ctx, sg_src/dst (and copies). This essentially
279 * protects against concurrent access to these fields.
280 * @lock: Lock for protecting both access to device hardware registers
281 * and fields related to current request (including the busy field).
282 * @res: Resources for hash.
283 * @io_hash_base: Per-variant offset for HASH block IO memory.
284 * @hash_lock: Lock for protecting hash_req, hash_queue and hash_flags
285 * variable.
286 * @hash_flags: Flags for current HASH op.
287 * @hash_queue: Async hash queue.
288 * @hash_tasklet: New HASH request scheduling job.
289 * @xmit_buf: Buffer for current HASH request transfer into SSS block.
290 * @hash_req: Current request sending to SSS HASH block.
291 * @hash_sg_iter: Scatterlist transferred through DMA into SSS HASH block.
292 * @hash_sg_cnt: Counter for hash_sg_iter.
293 *
294 * @use_hash: true if HASH algs enabled
295 */
296 struct s5p_aes_dev {
297 struct device *dev;
298 struct clk *clk;
299 void __iomem *ioaddr;
300 void __iomem *aes_ioaddr;
301 int irq_fc;
302
303 struct ablkcipher_request *req;
304 struct s5p_aes_ctx *ctx;
305 struct scatterlist *sg_src;
306 struct scatterlist *sg_dst;
307
308 struct scatterlist *sg_src_cpy;
309 struct scatterlist *sg_dst_cpy;
310
311 struct tasklet_struct tasklet;
312 struct crypto_queue queue;
313 bool busy;
314 spinlock_t lock;
315
316 struct resource *res;
317 void __iomem *io_hash_base;
318
319 spinlock_t hash_lock; /* protect hash_ vars */
320 unsigned long hash_flags;
321 struct crypto_queue hash_queue;
322 struct tasklet_struct hash_tasklet;
323
324 u8 xmit_buf[BUFLEN];
325 struct ahash_request *hash_req;
326 struct scatterlist *hash_sg_iter;
327 unsigned int hash_sg_cnt;
328
329 bool use_hash;
330 };
331
332 /**
333 * struct s5p_hash_reqctx - HASH request context
334 * @dd: Associated device
335 * @op_update: Current request operation (OP_UPDATE or OP_FINAL)
336 * @digcnt: Number of bytes processed by HW (without buffer[] ones)
337 * @digest: Digest message or IV for partial result
338 * @nregs: Number of HW registers for digest or IV read/write
339 * @engine: Bits for selecting type of HASH in SSS block
340 * @sg: sg for DMA transfer
341 * @sg_len: Length of sg for DMA transfer
342 * @sgl[]: sg for joining buffer and req->src scatterlist
343 * @skip: Skip offset in req->src for current op
344 * @total: Total number of bytes for current request
345 * @finup: Keep state for finup or final.
346 * @error: Keep track of error.
347 * @bufcnt: Number of bytes holded in buffer[]
348 * @buffer[]: For byte(s) from end of req->src in UPDATE op
349 */
350 struct s5p_hash_reqctx {
351 struct s5p_aes_dev *dd;
352 bool op_update;
353
354 u64 digcnt;
355 u8 digest[SHA256_DIGEST_SIZE];
356
357 unsigned int nregs; /* digest_size / sizeof(reg) */
358 u32 engine;
359
360 struct scatterlist *sg;
361 unsigned int sg_len;
362 struct scatterlist sgl[2];
363 unsigned int skip;
364 unsigned int total;
365 bool finup;
366 bool error;
367
368 u32 bufcnt;
369 u8 buffer[0];
370 };
371
372 /**
373 * struct s5p_hash_ctx - HASH transformation context
374 * @dd: Associated device
375 * @flags: Bits for algorithm HASH.
376 * @fallback: Software transformation for zero message or size < BUFLEN.
377 */
378 struct s5p_hash_ctx {
379 struct s5p_aes_dev *dd;
380 unsigned long flags;
381 struct crypto_shash *fallback;
382 };
383
384 static const struct samsung_aes_variant s5p_aes_data = {
385 .aes_offset = 0x4000,
386 .hash_offset = 0x6000,
387 };
388
389 static const struct samsung_aes_variant exynos_aes_data = {
390 .aes_offset = 0x200,
391 .hash_offset = 0x400,
392 };
393
394 static const struct of_device_id s5p_sss_dt_match[] = {
395 {
396 .compatible = "samsung,s5pv210-secss",
397 .data = &s5p_aes_data,
398 },
399 {
400 .compatible = "samsung,exynos4210-secss",
401 .data = &exynos_aes_data,
402 },
403 { },
404 };
405 MODULE_DEVICE_TABLE(of, s5p_sss_dt_match);
406
407 static inline struct samsung_aes_variant *find_s5p_sss_version
408 (struct platform_device *pdev)
409 {
410 if (IS_ENABLED(CONFIG_OF) && (pdev->dev.of_node)) {
411 const struct of_device_id *match;
412
413 match = of_match_node(s5p_sss_dt_match,
414 pdev->dev.of_node);
415 return (struct samsung_aes_variant *)match->data;
416 }
417 return (struct samsung_aes_variant *)
418 platform_get_device_id(pdev)->driver_data;
419 }
420
421 static struct s5p_aes_dev *s5p_dev;
422
423 static void s5p_set_dma_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
424 {
425 SSS_WRITE(dev, FCBRDMAS, sg_dma_address(sg));
426 SSS_WRITE(dev, FCBRDMAL, sg_dma_len(sg));
427 }
428
429 static void s5p_set_dma_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
430 {
431 SSS_WRITE(dev, FCBTDMAS, sg_dma_address(sg));
432 SSS_WRITE(dev, FCBTDMAL, sg_dma_len(sg));
433 }
434
435 static void s5p_free_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist **sg)
436 {
437 int len;
438
439 if (!*sg)
440 return;
441
442 len = ALIGN(dev->req->nbytes, AES_BLOCK_SIZE);
443 free_pages((unsigned long)sg_virt(*sg), get_order(len));
444
445 kfree(*sg);
446 *sg = NULL;
447 }
448
449 static void s5p_sg_copy_buf(void *buf, struct scatterlist *sg,
450 unsigned int nbytes, int out)
451 {
452 struct scatter_walk walk;
453
454 if (!nbytes)
455 return;
456
457 scatterwalk_start(&walk, sg);
458 scatterwalk_copychunks(buf, &walk, nbytes, out);
459 scatterwalk_done(&walk, out, 0);
460 }
461
462 static void s5p_sg_done(struct s5p_aes_dev *dev)
463 {
464 if (dev->sg_dst_cpy) {
465 dev_dbg(dev->dev,
466 "Copying %d bytes of output data back to original place\n",
467 dev->req->nbytes);
468 s5p_sg_copy_buf(sg_virt(dev->sg_dst_cpy), dev->req->dst,
469 dev->req->nbytes, 1);
470 }
471 s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
472 s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
473 }
474
475 /* Calls the completion. Cannot be called with dev->lock hold. */
476 static void s5p_aes_complete(struct s5p_aes_dev *dev, int err)
477 {
478 dev->req->base.complete(&dev->req->base, err);
479 }
480
481 static void s5p_unset_outdata(struct s5p_aes_dev *dev)
482 {
483 dma_unmap_sg(dev->dev, dev->sg_dst, 1, DMA_FROM_DEVICE);
484 }
485
486 static void s5p_unset_indata(struct s5p_aes_dev *dev)
487 {
488 dma_unmap_sg(dev->dev, dev->sg_src, 1, DMA_TO_DEVICE);
489 }
490
491 static int s5p_make_sg_cpy(struct s5p_aes_dev *dev, struct scatterlist *src,
492 struct scatterlist **dst)
493 {
494 void *pages;
495 int len;
496
497 *dst = kmalloc(sizeof(**dst), GFP_ATOMIC);
498 if (!*dst)
499 return -ENOMEM;
500
501 len = ALIGN(dev->req->nbytes, AES_BLOCK_SIZE);
502 pages = (void *)__get_free_pages(GFP_ATOMIC, get_order(len));
503 if (!pages) {
504 kfree(*dst);
505 *dst = NULL;
506 return -ENOMEM;
507 }
508
509 s5p_sg_copy_buf(pages, src, dev->req->nbytes, 0);
510
511 sg_init_table(*dst, 1);
512 sg_set_buf(*dst, pages, len);
513
514 return 0;
515 }
516
517 static int s5p_set_outdata(struct s5p_aes_dev *dev, struct scatterlist *sg)
518 {
519 int err;
520
521 if (!sg->length) {
522 err = -EINVAL;
523 goto exit;
524 }
525
526 err = dma_map_sg(dev->dev, sg, 1, DMA_FROM_DEVICE);
527 if (!err) {
528 err = -ENOMEM;
529 goto exit;
530 }
531
532 dev->sg_dst = sg;
533 err = 0;
534
535 exit:
536 return err;
537 }
538
539 static int s5p_set_indata(struct s5p_aes_dev *dev, struct scatterlist *sg)
540 {
541 int err;
542
543 if (!sg->length) {
544 err = -EINVAL;
545 goto exit;
546 }
547
548 err = dma_map_sg(dev->dev, sg, 1, DMA_TO_DEVICE);
549 if (!err) {
550 err = -ENOMEM;
551 goto exit;
552 }
553
554 dev->sg_src = sg;
555 err = 0;
556
557 exit:
558 return err;
559 }
560
561 /*
562 * Returns -ERRNO on error (mapping of new data failed).
563 * On success returns:
564 * - 0 if there is no more data,
565 * - 1 if new transmitting (output) data is ready and its address+length
566 * have to be written to device (by calling s5p_set_dma_outdata()).
567 */
568 static int s5p_aes_tx(struct s5p_aes_dev *dev)
569 {
570 int ret = 0;
571
572 s5p_unset_outdata(dev);
573
574 if (!sg_is_last(dev->sg_dst)) {
575 ret = s5p_set_outdata(dev, sg_next(dev->sg_dst));
576 if (!ret)
577 ret = 1;
578 }
579
580 return ret;
581 }
582
583 /*
584 * Returns -ERRNO on error (mapping of new data failed).
585 * On success returns:
586 * - 0 if there is no more data,
587 * - 1 if new receiving (input) data is ready and its address+length
588 * have to be written to device (by calling s5p_set_dma_indata()).
589 */
590 static int s5p_aes_rx(struct s5p_aes_dev *dev/*, bool *set_dma*/)
591 {
592 int ret = 0;
593
594 s5p_unset_indata(dev);
595
596 if (!sg_is_last(dev->sg_src)) {
597 ret = s5p_set_indata(dev, sg_next(dev->sg_src));
598 if (!ret)
599 ret = 1;
600 }
601
602 return ret;
603 }
604
605 static inline u32 s5p_hash_read(struct s5p_aes_dev *dd, u32 offset)
606 {
607 return __raw_readl(dd->io_hash_base + offset);
608 }
609
610 static inline void s5p_hash_write(struct s5p_aes_dev *dd,
611 u32 offset, u32 value)
612 {
613 __raw_writel(value, dd->io_hash_base + offset);
614 }
615
616 /**
617 * s5p_set_dma_hashdata() - start DMA with sg
618 * @dev: device
619 * @sg: scatterlist ready to DMA transmit
620 */
621 static void s5p_set_dma_hashdata(struct s5p_aes_dev *dev,
622 struct scatterlist *sg)
623 {
624 dev->hash_sg_cnt--;
625 SSS_WRITE(dev, FCHRDMAS, sg_dma_address(sg));
626 SSS_WRITE(dev, FCHRDMAL, sg_dma_len(sg)); /* DMA starts */
627 }
628
629 /**
630 * s5p_hash_rx() - get next hash_sg_iter
631 * @dev: device
632 *
633 * Return:
634 * 2 if there is no more data and it is UPDATE op
635 * 1 if new receiving (input) data is ready and can be written to device
636 * 0 if there is no more data and it is FINAL op
637 */
638 static int s5p_hash_rx(struct s5p_aes_dev *dev)
639 {
640 if (dev->hash_sg_cnt > 0) {
641 dev->hash_sg_iter = sg_next(dev->hash_sg_iter);
642 return 1;
643 }
644
645 set_bit(HASH_FLAGS_DMA_READY, &dev->hash_flags);
646 if (test_bit(HASH_FLAGS_FINAL, &dev->hash_flags))
647 return 0;
648
649 return 2;
650 }
651
652 static irqreturn_t s5p_aes_interrupt(int irq, void *dev_id)
653 {
654 struct platform_device *pdev = dev_id;
655 struct s5p_aes_dev *dev = platform_get_drvdata(pdev);
656 int err_dma_tx = 0;
657 int err_dma_rx = 0;
658 int err_dma_hx = 0;
659 bool tx_end = false;
660 bool hx_end = false;
661 unsigned long flags;
662 uint32_t status;
663 u32 st_bits;
664 int err;
665
666 spin_lock_irqsave(&dev->lock, flags);
667
668 /*
669 * Handle rx or tx interrupt. If there is still data (scatterlist did not
670 * reach end), then map next scatterlist entry.
671 * In case of such mapping error, s5p_aes_complete() should be called.
672 *
673 * If there is no more data in tx scatter list, call s5p_aes_complete()
674 * and schedule new tasklet.
675 *
676 * Handle hx interrupt. If there is still data map next entry.
677 */
678 status = SSS_READ(dev, FCINTSTAT);
679 if (status & SSS_FCINTSTAT_BRDMAINT)
680 err_dma_rx = s5p_aes_rx(dev);
681
682 if (status & SSS_FCINTSTAT_BTDMAINT) {
683 if (sg_is_last(dev->sg_dst))
684 tx_end = true;
685 err_dma_tx = s5p_aes_tx(dev);
686 }
687
688 if (status & SSS_FCINTSTAT_HRDMAINT)
689 err_dma_hx = s5p_hash_rx(dev);
690
691 st_bits = status & (SSS_FCINTSTAT_BRDMAINT | SSS_FCINTSTAT_BTDMAINT |
692 SSS_FCINTSTAT_HRDMAINT);
693 /* clear DMA bits */
694 SSS_WRITE(dev, FCINTPEND, st_bits);
695
696 /* clear HASH irq bits */
697 if (status & (SSS_FCINTSTAT_HDONEINT | SSS_FCINTSTAT_HPARTINT)) {
698 /* cannot have both HPART and HDONE */
699 if (status & SSS_FCINTSTAT_HPARTINT)
700 st_bits = SSS_HASH_STATUS_PARTIAL_DONE;
701
702 if (status & SSS_FCINTSTAT_HDONEINT)
703 st_bits = SSS_HASH_STATUS_MSG_DONE;
704
705 set_bit(HASH_FLAGS_OUTPUT_READY, &dev->hash_flags);
706 s5p_hash_write(dev, SSS_REG_HASH_STATUS, st_bits);
707 hx_end = true;
708 /* when DONE or PART, do not handle HASH DMA */
709 err_dma_hx = 0;
710 }
711
712 if (err_dma_rx < 0) {
713 err = err_dma_rx;
714 goto error;
715 }
716 if (err_dma_tx < 0) {
717 err = err_dma_tx;
718 goto error;
719 }
720
721 if (tx_end) {
722 s5p_sg_done(dev);
723 if (err_dma_hx == 1)
724 s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
725
726 spin_unlock_irqrestore(&dev->lock, flags);
727
728 s5p_aes_complete(dev, 0);
729 /* Device is still busy */
730 tasklet_schedule(&dev->tasklet);
731 } else {
732 /*
733 * Writing length of DMA block (either receiving or
734 * transmitting) will start the operation immediately, so this
735 * should be done at the end (even after clearing pending
736 * interrupts to not miss the interrupt).
737 */
738 if (err_dma_tx == 1)
739 s5p_set_dma_outdata(dev, dev->sg_dst);
740 if (err_dma_rx == 1)
741 s5p_set_dma_indata(dev, dev->sg_src);
742 if (err_dma_hx == 1)
743 s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
744
745 spin_unlock_irqrestore(&dev->lock, flags);
746 }
747
748 goto hash_irq_end;
749
750 error:
751 s5p_sg_done(dev);
752 dev->busy = false;
753 if (err_dma_hx == 1)
754 s5p_set_dma_hashdata(dev, dev->hash_sg_iter);
755
756 spin_unlock_irqrestore(&dev->lock, flags);
757 s5p_aes_complete(dev, err);
758
759 hash_irq_end:
760 /*
761 * Note about else if:
762 * when hash_sg_iter reaches end and its UPDATE op,
763 * issue SSS_HASH_PAUSE and wait for HPART irq
764 */
765 if (hx_end)
766 tasklet_schedule(&dev->hash_tasklet);
767 else if (err_dma_hx == 2)
768 s5p_hash_write(dev, SSS_REG_HASH_CTRL_PAUSE,
769 SSS_HASH_PAUSE);
770
771 return IRQ_HANDLED;
772 }
773
774 /**
775 * s5p_hash_read_msg() - read message or IV from HW
776 * @req: AHASH request
777 */
778 static void s5p_hash_read_msg(struct ahash_request *req)
779 {
780 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
781 struct s5p_aes_dev *dd = ctx->dd;
782 u32 *hash = (u32 *)ctx->digest;
783 unsigned int i;
784
785 for (i = 0; i < ctx->nregs; i++)
786 hash[i] = s5p_hash_read(dd, SSS_REG_HASH_OUT(i));
787 }
788
789 /**
790 * s5p_hash_write_ctx_iv() - write IV for next partial/finup op.
791 * @dd: device
792 * @ctx: request context
793 */
794 static void s5p_hash_write_ctx_iv(struct s5p_aes_dev *dd,
795 struct s5p_hash_reqctx *ctx)
796 {
797 u32 *hash = (u32 *)ctx->digest;
798 unsigned int i;
799
800 for (i = 0; i < ctx->nregs; i++)
801 s5p_hash_write(dd, SSS_REG_HASH_IV(i), hash[i]);
802 }
803
804 /**
805 * s5p_hash_write_iv() - write IV for next partial/finup op.
806 * @req: AHASH request
807 */
808 static void s5p_hash_write_iv(struct ahash_request *req)
809 {
810 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
811
812 s5p_hash_write_ctx_iv(ctx->dd, ctx);
813 }
814
815 /**
816 * s5p_hash_copy_result() - copy digest into req->result
817 * @req: AHASH request
818 */
819 static void s5p_hash_copy_result(struct ahash_request *req)
820 {
821 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
822
823 if (!req->result)
824 return;
825
826 memcpy(req->result, ctx->digest, ctx->nregs * HASH_REG_SIZEOF);
827 }
828
829 /**
830 * s5p_hash_dma_flush() - flush HASH DMA
831 * @dev: secss device
832 */
833 static void s5p_hash_dma_flush(struct s5p_aes_dev *dev)
834 {
835 SSS_WRITE(dev, FCHRDMAC, SSS_FCHRDMAC_FLUSH);
836 }
837
838 /**
839 * s5p_hash_dma_enable() - enable DMA mode for HASH
840 * @dev: secss device
841 *
842 * enable DMA mode for HASH
843 */
844 static void s5p_hash_dma_enable(struct s5p_aes_dev *dev)
845 {
846 s5p_hash_write(dev, SSS_REG_HASH_CTRL_FIFO, SSS_HASH_FIFO_MODE_DMA);
847 }
848
849 /**
850 * s5p_hash_irq_disable() - disable irq HASH signals
851 * @dev: secss device
852 * @flags: bitfield with irq's to be disabled
853 */
854 static void s5p_hash_irq_disable(struct s5p_aes_dev *dev, u32 flags)
855 {
856 SSS_WRITE(dev, FCINTENCLR, flags);
857 }
858
859 /**
860 * s5p_hash_irq_enable() - enable irq signals
861 * @dev: secss device
862 * @flags: bitfield with irq's to be enabled
863 */
864 static void s5p_hash_irq_enable(struct s5p_aes_dev *dev, int flags)
865 {
866 SSS_WRITE(dev, FCINTENSET, flags);
867 }
868
869 /**
870 * s5p_hash_set_flow() - set flow inside SecSS AES/DES with/without HASH
871 * @dev: secss device
872 * @hashflow: HASH stream flow with/without crypto AES/DES
873 */
874 static void s5p_hash_set_flow(struct s5p_aes_dev *dev, u32 hashflow)
875 {
876 unsigned long flags;
877 u32 flow;
878
879 spin_lock_irqsave(&dev->lock, flags);
880
881 flow = SSS_READ(dev, FCFIFOCTRL);
882 flow &= ~SSS_HASHIN_MASK;
883 flow |= hashflow;
884 SSS_WRITE(dev, FCFIFOCTRL, flow);
885
886 spin_unlock_irqrestore(&dev->lock, flags);
887 }
888
889 /**
890 * s5p_ahash_dma_init() - enable DMA and set HASH flow inside SecSS
891 * @dev: secss device
892 * @hashflow: HASH stream flow with/without AES/DES
893 *
894 * flush HASH DMA and enable DMA, set HASH stream flow inside SecSS HW,
895 * enable HASH irq's HRDMA, HDONE, HPART
896 */
897 static void s5p_ahash_dma_init(struct s5p_aes_dev *dev, u32 hashflow)
898 {
899 s5p_hash_irq_disable(dev, SSS_FCINTENCLR_HRDMAINTENCLR |
900 SSS_FCINTENCLR_HDONEINTENCLR |
901 SSS_FCINTENCLR_HPARTINTENCLR);
902 s5p_hash_dma_flush(dev);
903
904 s5p_hash_dma_enable(dev);
905 s5p_hash_set_flow(dev, hashflow & SSS_HASHIN_MASK);
906 s5p_hash_irq_enable(dev, SSS_FCINTENSET_HRDMAINTENSET |
907 SSS_FCINTENSET_HDONEINTENSET |
908 SSS_FCINTENSET_HPARTINTENSET);
909 }
910
911 /**
912 * s5p_hash_write_ctrl() - prepare HASH block in SecSS for processing
913 * @dd: secss device
914 * @length: length for request
915 * @final: true if final op
916 *
917 * Prepare SSS HASH block for processing bytes in DMA mode. If it is called
918 * after previous updates, fill up IV words. For final, calculate and set
919 * lengths for HASH so SecSS can finalize hash. For partial, set SSS HASH
920 * length as 2^63 so it will be never reached and set to zero prelow and
921 * prehigh.
922 *
923 * This function does not start DMA transfer.
924 */
925 static void s5p_hash_write_ctrl(struct s5p_aes_dev *dd, size_t length,
926 bool final)
927 {
928 struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
929 u32 prelow, prehigh, low, high;
930 u32 configflags, swapflags;
931 u64 tmplen;
932
933 configflags = ctx->engine | SSS_HASH_INIT_BIT;
934
935 if (likely(ctx->digcnt)) {
936 s5p_hash_write_ctx_iv(dd, ctx);
937 configflags |= SSS_HASH_USER_IV_EN;
938 }
939
940 if (final) {
941 /* number of bytes for last part */
942 low = length;
943 high = 0;
944 /* total number of bits prev hashed */
945 tmplen = ctx->digcnt * 8;
946 prelow = (u32)tmplen;
947 prehigh = (u32)(tmplen >> 32);
948 } else {
949 prelow = 0;
950 prehigh = 0;
951 low = 0;
952 high = BIT(31);
953 }
954
955 swapflags = SSS_HASH_BYTESWAP_DI | SSS_HASH_BYTESWAP_DO |
956 SSS_HASH_BYTESWAP_IV | SSS_HASH_BYTESWAP_KEY;
957
958 s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_LOW, low);
959 s5p_hash_write(dd, SSS_REG_HASH_MSG_SIZE_HIGH, high);
960 s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_LOW, prelow);
961 s5p_hash_write(dd, SSS_REG_HASH_PRE_MSG_SIZE_HIGH, prehigh);
962
963 s5p_hash_write(dd, SSS_REG_HASH_CTRL_SWAP, swapflags);
964 s5p_hash_write(dd, SSS_REG_HASH_CTRL, configflags);
965 }
966
967 /**
968 * s5p_hash_xmit_dma() - start DMA hash processing
969 * @dd: secss device
970 * @length: length for request
971 * @final: true if final op
972 *
973 * Update digcnt here, as it is needed for finup/final op.
974 */
975 static int s5p_hash_xmit_dma(struct s5p_aes_dev *dd, size_t length,
976 bool final)
977 {
978 struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
979 unsigned int cnt;
980
981 cnt = dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
982 if (!cnt) {
983 dev_err(dd->dev, "dma_map_sg error\n");
984 ctx->error = true;
985 return -EINVAL;
986 }
987
988 set_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);
989 dd->hash_sg_iter = ctx->sg;
990 dd->hash_sg_cnt = cnt;
991 s5p_hash_write_ctrl(dd, length, final);
992 ctx->digcnt += length;
993 ctx->total -= length;
994
995 /* catch last interrupt */
996 if (final)
997 set_bit(HASH_FLAGS_FINAL, &dd->hash_flags);
998
999 s5p_set_dma_hashdata(dd, dd->hash_sg_iter); /* DMA starts */
1000
1001 return -EINPROGRESS;
1002 }
1003
1004 /**
1005 * s5p_hash_copy_sgs() - copy request's bytes into new buffer
1006 * @ctx: request context
1007 * @sg: source scatterlist request
1008 * @new_len: number of bytes to process from sg
1009 *
1010 * Allocate new buffer, copy data for HASH into it. If there was xmit_buf
1011 * filled, copy it first, then copy data from sg into it. Prepare one sgl[0]
1012 * with allocated buffer.
1013 *
1014 * Set bit in dd->hash_flag so we can free it after irq ends processing.
1015 */
1016 static int s5p_hash_copy_sgs(struct s5p_hash_reqctx *ctx,
1017 struct scatterlist *sg, unsigned int new_len)
1018 {
1019 unsigned int pages, len;
1020 void *buf;
1021
1022 len = new_len + ctx->bufcnt;
1023 pages = get_order(len);
1024
1025 buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
1026 if (!buf) {
1027 dev_err(ctx->dd->dev, "alloc pages for unaligned case.\n");
1028 ctx->error = true;
1029 return -ENOMEM;
1030 }
1031
1032 if (ctx->bufcnt)
1033 memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
1034
1035 scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->skip,
1036 new_len, 0);
1037 sg_init_table(ctx->sgl, 1);
1038 sg_set_buf(ctx->sgl, buf, len);
1039 ctx->sg = ctx->sgl;
1040 ctx->sg_len = 1;
1041 ctx->bufcnt = 0;
1042 ctx->skip = 0;
1043 set_bit(HASH_FLAGS_SGS_COPIED, &ctx->dd->hash_flags);
1044
1045 return 0;
1046 }
1047
1048 /**
1049 * s5p_hash_copy_sg_lists() - copy sg list and make fixes in copy
1050 * @ctx: request context
1051 * @sg: source scatterlist request
1052 * @new_len: number of bytes to process from sg
1053 *
1054 * Allocate new scatterlist table, copy data for HASH into it. If there was
1055 * xmit_buf filled, prepare it first, then copy page, length and offset from
1056 * source sg into it, adjusting begin and/or end for skip offset and
1057 * hash_later value.
1058 *
1059 * Resulting sg table will be assigned to ctx->sg. Set flag so we can free
1060 * it after irq ends processing.
1061 */
1062 static int s5p_hash_copy_sg_lists(struct s5p_hash_reqctx *ctx,
1063 struct scatterlist *sg, unsigned int new_len)
1064 {
1065 unsigned int skip = ctx->skip, n = sg_nents(sg);
1066 struct scatterlist *tmp;
1067 unsigned int len;
1068
1069 if (ctx->bufcnt)
1070 n++;
1071
1072 ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
1073 if (!ctx->sg) {
1074 ctx->error = true;
1075 return -ENOMEM;
1076 }
1077
1078 sg_init_table(ctx->sg, n);
1079
1080 tmp = ctx->sg;
1081
1082 ctx->sg_len = 0;
1083
1084 if (ctx->bufcnt) {
1085 sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
1086 tmp = sg_next(tmp);
1087 ctx->sg_len++;
1088 }
1089
1090 while (sg && skip >= sg->length) {
1091 skip -= sg->length;
1092 sg = sg_next(sg);
1093 }
1094
1095 while (sg && new_len) {
1096 len = sg->length - skip;
1097 if (new_len < len)
1098 len = new_len;
1099
1100 new_len -= len;
1101 sg_set_page(tmp, sg_page(sg), len, sg->offset + skip);
1102 skip = 0;
1103 if (new_len <= 0)
1104 sg_mark_end(tmp);
1105
1106 tmp = sg_next(tmp);
1107 ctx->sg_len++;
1108 sg = sg_next(sg);
1109 }
1110
1111 set_bit(HASH_FLAGS_SGS_ALLOCED, &ctx->dd->hash_flags);
1112
1113 return 0;
1114 }
1115
1116 /**
1117 * s5p_hash_prepare_sgs() - prepare sg for processing
1118 * @ctx: request context
1119 * @sg: source scatterlist request
1120 * @nbytes: number of bytes to process from sg
1121 * @final: final flag
1122 *
1123 * Check two conditions: (1) if buffers in sg have len aligned data, and (2)
1124 * sg table have good aligned elements (list_ok). If one of this checks fails,
1125 * then either (1) allocates new buffer for data with s5p_hash_copy_sgs, copy
1126 * data into this buffer and prepare request in sgl, or (2) allocates new sg
1127 * table and prepare sg elements.
1128 *
1129 * For digest or finup all conditions can be good, and we may not need any
1130 * fixes.
1131 */
1132 static int s5p_hash_prepare_sgs(struct s5p_hash_reqctx *ctx,
1133 struct scatterlist *sg,
1134 unsigned int new_len, bool final)
1135 {
1136 unsigned int skip = ctx->skip, nbytes = new_len, n = 0;
1137 bool aligned = true, list_ok = true;
1138 struct scatterlist *sg_tmp = sg;
1139
1140 if (!sg || !sg->length || !new_len)
1141 return 0;
1142
1143 if (skip || !final)
1144 list_ok = false;
1145
1146 while (nbytes > 0 && sg_tmp) {
1147 n++;
1148 if (skip >= sg_tmp->length) {
1149 skip -= sg_tmp->length;
1150 if (!sg_tmp->length) {
1151 aligned = false;
1152 break;
1153 }
1154 } else {
1155 if (!IS_ALIGNED(sg_tmp->length - skip, BUFLEN)) {
1156 aligned = false;
1157 break;
1158 }
1159
1160 if (nbytes < sg_tmp->length - skip) {
1161 list_ok = false;
1162 break;
1163 }
1164
1165 nbytes -= sg_tmp->length - skip;
1166 skip = 0;
1167 }
1168
1169 sg_tmp = sg_next(sg_tmp);
1170 }
1171
1172 if (!aligned)
1173 return s5p_hash_copy_sgs(ctx, sg, new_len);
1174 else if (!list_ok)
1175 return s5p_hash_copy_sg_lists(ctx, sg, new_len);
1176
1177 /*
1178 * Have aligned data from previous operation and/or current
1179 * Note: will enter here only if (digest or finup) and aligned
1180 */
1181 if (ctx->bufcnt) {
1182 ctx->sg_len = n;
1183 sg_init_table(ctx->sgl, 2);
1184 sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, ctx->bufcnt);
1185 sg_chain(ctx->sgl, 2, sg);
1186 ctx->sg = ctx->sgl;
1187 ctx->sg_len++;
1188 } else {
1189 ctx->sg = sg;
1190 ctx->sg_len = n;
1191 }
1192
1193 return 0;
1194 }
1195
1196 /**
1197 * s5p_hash_prepare_request() - prepare request for processing
1198 * @req: AHASH request
1199 * @update: true if UPDATE op
1200 *
1201 * Note 1: we can have update flag _and_ final flag at the same time.
1202 * Note 2: we enter here when digcnt > BUFLEN (=HASH_BLOCK_SIZE) or
1203 * either req->nbytes or ctx->bufcnt + req->nbytes is > BUFLEN or
1204 * we have final op
1205 */
1206 static int s5p_hash_prepare_request(struct ahash_request *req, bool update)
1207 {
1208 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1209 bool final = ctx->finup;
1210 int xmit_len, hash_later, nbytes;
1211 int ret;
1212
1213 if (!req)
1214 return 0;
1215
1216 if (update)
1217 nbytes = req->nbytes;
1218 else
1219 nbytes = 0;
1220
1221 ctx->total = nbytes + ctx->bufcnt;
1222 if (!ctx->total)
1223 return 0;
1224
1225 if (nbytes && (!IS_ALIGNED(ctx->bufcnt, BUFLEN))) {
1226 /* bytes left from previous request, so fill up to BUFLEN */
1227 int len = BUFLEN - ctx->bufcnt % BUFLEN;
1228
1229 if (len > nbytes)
1230 len = nbytes;
1231
1232 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1233 0, len, 0);
1234 ctx->bufcnt += len;
1235 nbytes -= len;
1236 ctx->skip = len;
1237 } else {
1238 ctx->skip = 0;
1239 }
1240
1241 if (ctx->bufcnt)
1242 memcpy(ctx->dd->xmit_buf, ctx->buffer, ctx->bufcnt);
1243
1244 xmit_len = ctx->total;
1245 if (final) {
1246 hash_later = 0;
1247 } else {
1248 if (IS_ALIGNED(xmit_len, BUFLEN))
1249 xmit_len -= BUFLEN;
1250 else
1251 xmit_len -= xmit_len & (BUFLEN - 1);
1252
1253 hash_later = ctx->total - xmit_len;
1254 /* copy hash_later bytes from end of req->src */
1255 /* previous bytes are in xmit_buf, so no overwrite */
1256 scatterwalk_map_and_copy(ctx->buffer, req->src,
1257 req->nbytes - hash_later,
1258 hash_later, 0);
1259 }
1260
1261 if (xmit_len > BUFLEN) {
1262 ret = s5p_hash_prepare_sgs(ctx, req->src, nbytes - hash_later,
1263 final);
1264 if (ret)
1265 return ret;
1266 } else {
1267 /* have buffered data only */
1268 if (unlikely(!ctx->bufcnt)) {
1269 /* first update didn't fill up buffer */
1270 scatterwalk_map_and_copy(ctx->dd->xmit_buf, req->src,
1271 0, xmit_len, 0);
1272 }
1273
1274 sg_init_table(ctx->sgl, 1);
1275 sg_set_buf(ctx->sgl, ctx->dd->xmit_buf, xmit_len);
1276
1277 ctx->sg = ctx->sgl;
1278 ctx->sg_len = 1;
1279 }
1280
1281 ctx->bufcnt = hash_later;
1282 if (!final)
1283 ctx->total = xmit_len;
1284
1285 return 0;
1286 }
1287
1288 /**
1289 * s5p_hash_update_dma_stop() - unmap DMA
1290 * @dd: secss device
1291 *
1292 * Unmap scatterlist ctx->sg.
1293 */
1294 static void s5p_hash_update_dma_stop(struct s5p_aes_dev *dd)
1295 {
1296 struct s5p_hash_reqctx *ctx = ahash_request_ctx(dd->hash_req);
1297
1298 dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
1299 clear_bit(HASH_FLAGS_DMA_ACTIVE, &dd->hash_flags);
1300 }
1301
1302 /**
1303 * s5p_hash_finish() - copy calculated digest to crypto layer
1304 * @req: AHASH request
1305 */
1306 static void s5p_hash_finish(struct ahash_request *req)
1307 {
1308 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1309 struct s5p_aes_dev *dd = ctx->dd;
1310
1311 if (ctx->digcnt)
1312 s5p_hash_copy_result(req);
1313
1314 dev_dbg(dd->dev, "hash_finish digcnt: %lld\n", ctx->digcnt);
1315 }
1316
1317 /**
1318 * s5p_hash_finish_req() - finish request
1319 * @req: AHASH request
1320 * @err: error
1321 */
1322 static void s5p_hash_finish_req(struct ahash_request *req, int err)
1323 {
1324 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1325 struct s5p_aes_dev *dd = ctx->dd;
1326 unsigned long flags;
1327
1328 if (test_bit(HASH_FLAGS_SGS_COPIED, &dd->hash_flags))
1329 free_pages((unsigned long)sg_virt(ctx->sg),
1330 get_order(ctx->sg->length));
1331
1332 if (test_bit(HASH_FLAGS_SGS_ALLOCED, &dd->hash_flags))
1333 kfree(ctx->sg);
1334
1335 ctx->sg = NULL;
1336 dd->hash_flags &= ~(BIT(HASH_FLAGS_SGS_ALLOCED) |
1337 BIT(HASH_FLAGS_SGS_COPIED));
1338
1339 if (!err && !ctx->error) {
1340 s5p_hash_read_msg(req);
1341 if (test_bit(HASH_FLAGS_FINAL, &dd->hash_flags))
1342 s5p_hash_finish(req);
1343 } else {
1344 ctx->error = true;
1345 }
1346
1347 spin_lock_irqsave(&dd->hash_lock, flags);
1348 dd->hash_flags &= ~(BIT(HASH_FLAGS_BUSY) | BIT(HASH_FLAGS_FINAL) |
1349 BIT(HASH_FLAGS_DMA_READY) |
1350 BIT(HASH_FLAGS_OUTPUT_READY));
1351 spin_unlock_irqrestore(&dd->hash_lock, flags);
1352
1353 if (req->base.complete)
1354 req->base.complete(&req->base, err);
1355 }
1356
1357 /**
1358 * s5p_hash_handle_queue() - handle hash queue
1359 * @dd: device s5p_aes_dev
1360 * @req: AHASH request
1361 *
1362 * If req!=NULL enqueue it on dd->queue, if FLAGS_BUSY is not set on the
1363 * device then processes the first request from the dd->queue
1364 *
1365 * Returns: see s5p_hash_final below.
1366 */
1367 static int s5p_hash_handle_queue(struct s5p_aes_dev *dd,
1368 struct ahash_request *req)
1369 {
1370 struct crypto_async_request *async_req, *backlog;
1371 struct s5p_hash_reqctx *ctx;
1372 unsigned long flags;
1373 int err = 0, ret = 0;
1374
1375 retry:
1376 spin_lock_irqsave(&dd->hash_lock, flags);
1377 if (req)
1378 ret = ahash_enqueue_request(&dd->hash_queue, req);
1379
1380 if (test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {
1381 spin_unlock_irqrestore(&dd->hash_lock, flags);
1382 return ret;
1383 }
1384
1385 backlog = crypto_get_backlog(&dd->hash_queue);
1386 async_req = crypto_dequeue_request(&dd->hash_queue);
1387 if (async_req)
1388 set_bit(HASH_FLAGS_BUSY, &dd->hash_flags);
1389
1390 spin_unlock_irqrestore(&dd->hash_lock, flags);
1391
1392 if (!async_req)
1393 return ret;
1394
1395 if (backlog)
1396 backlog->complete(backlog, -EINPROGRESS);
1397
1398 req = ahash_request_cast(async_req);
1399 dd->hash_req = req;
1400 ctx = ahash_request_ctx(req);
1401
1402 err = s5p_hash_prepare_request(req, ctx->op_update);
1403 if (err || !ctx->total)
1404 goto out;
1405
1406 dev_dbg(dd->dev, "handling new req, op_update: %u, nbytes: %d\n",
1407 ctx->op_update, req->nbytes);
1408
1409 s5p_ahash_dma_init(dd, SSS_HASHIN_INDEPENDENT);
1410 if (ctx->digcnt)
1411 s5p_hash_write_iv(req); /* restore hash IV */
1412
1413 if (ctx->op_update) { /* HASH_OP_UPDATE */
1414 err = s5p_hash_xmit_dma(dd, ctx->total, ctx->finup);
1415 if (err != -EINPROGRESS && ctx->finup && !ctx->error)
1416 /* no final() after finup() */
1417 err = s5p_hash_xmit_dma(dd, ctx->total, true);
1418 } else { /* HASH_OP_FINAL */
1419 err = s5p_hash_xmit_dma(dd, ctx->total, true);
1420 }
1421 out:
1422 if (err != -EINPROGRESS) {
1423 /* hash_tasklet_cb will not finish it, so do it here */
1424 s5p_hash_finish_req(req, err);
1425 req = NULL;
1426
1427 /*
1428 * Execute next request immediately if there is anything
1429 * in queue.
1430 */
1431 goto retry;
1432 }
1433
1434 return ret;
1435 }
1436
1437 /**
1438 * s5p_hash_tasklet_cb() - hash tasklet
1439 * @data: ptr to s5p_aes_dev
1440 */
1441 static void s5p_hash_tasklet_cb(unsigned long data)
1442 {
1443 struct s5p_aes_dev *dd = (struct s5p_aes_dev *)data;
1444
1445 if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags)) {
1446 s5p_hash_handle_queue(dd, NULL);
1447 return;
1448 }
1449
1450 if (test_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags)) {
1451 if (test_and_clear_bit(HASH_FLAGS_DMA_ACTIVE,
1452 &dd->hash_flags)) {
1453 s5p_hash_update_dma_stop(dd);
1454 }
1455
1456 if (test_and_clear_bit(HASH_FLAGS_OUTPUT_READY,
1457 &dd->hash_flags)) {
1458 /* hash or semi-hash ready */
1459 clear_bit(HASH_FLAGS_DMA_READY, &dd->hash_flags);
1460 goto finish;
1461 }
1462 }
1463
1464 return;
1465
1466 finish:
1467 /* finish curent request */
1468 s5p_hash_finish_req(dd->hash_req, 0);
1469
1470 /* If we are not busy, process next req */
1471 if (!test_bit(HASH_FLAGS_BUSY, &dd->hash_flags))
1472 s5p_hash_handle_queue(dd, NULL);
1473 }
1474
1475 /**
1476 * s5p_hash_enqueue() - enqueue request
1477 * @req: AHASH request
1478 * @op: operation UPDATE (true) or FINAL (false)
1479 *
1480 * Returns: see s5p_hash_final below.
1481 */
1482 static int s5p_hash_enqueue(struct ahash_request *req, bool op)
1483 {
1484 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1485 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1486
1487 ctx->op_update = op;
1488
1489 return s5p_hash_handle_queue(tctx->dd, req);
1490 }
1491
1492 /**
1493 * s5p_hash_update() - process the hash input data
1494 * @req: AHASH request
1495 *
1496 * If request will fit in buffer, copy it and return immediately
1497 * else enqueue it with OP_UPDATE.
1498 *
1499 * Returns: see s5p_hash_final below.
1500 */
1501 static int s5p_hash_update(struct ahash_request *req)
1502 {
1503 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1504
1505 if (!req->nbytes)
1506 return 0;
1507
1508 if (ctx->bufcnt + req->nbytes <= BUFLEN) {
1509 scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
1510 0, req->nbytes, 0);
1511 ctx->bufcnt += req->nbytes;
1512 return 0;
1513 }
1514
1515 return s5p_hash_enqueue(req, true); /* HASH_OP_UPDATE */
1516 }
1517
1518 /**
1519 * s5p_hash_shash_digest() - calculate shash digest
1520 * @tfm: crypto transformation
1521 * @flags: tfm flags
1522 * @data: input data
1523 * @len: length of data
1524 * @out: output buffer
1525 */
1526 static int s5p_hash_shash_digest(struct crypto_shash *tfm, u32 flags,
1527 const u8 *data, unsigned int len, u8 *out)
1528 {
1529 SHASH_DESC_ON_STACK(shash, tfm);
1530
1531 shash->tfm = tfm;
1532 shash->flags = flags & ~CRYPTO_TFM_REQ_MAY_SLEEP;
1533
1534 return crypto_shash_digest(shash, data, len, out);
1535 }
1536
1537 /**
1538 * s5p_hash_final_shash() - calculate shash digest
1539 * @req: AHASH request
1540 */
1541 static int s5p_hash_final_shash(struct ahash_request *req)
1542 {
1543 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
1544 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1545
1546 return s5p_hash_shash_digest(tctx->fallback, req->base.flags,
1547 ctx->buffer, ctx->bufcnt, req->result);
1548 }
1549
1550 /**
1551 * s5p_hash_final() - close up hash and calculate digest
1552 * @req: AHASH request
1553 *
1554 * Note: in final req->src do not have any data, and req->nbytes can be
1555 * non-zero.
1556 *
1557 * If there were no input data processed yet and the buffered hash data is
1558 * less than BUFLEN (64) then calculate the final hash immediately by using
1559 * SW algorithm fallback.
1560 *
1561 * Otherwise enqueues the current AHASH request with OP_FINAL operation op
1562 * and finalize hash message in HW. Note that if digcnt!=0 then there were
1563 * previous update op, so there are always some buffered bytes in ctx->buffer,
1564 * which means that ctx->bufcnt!=0
1565 *
1566 * Returns:
1567 * 0 if the request has been processed immediately,
1568 * -EINPROGRESS if the operation has been queued for later execution or is set
1569 * to processing by HW,
1570 * -EBUSY if queue is full and request should be resubmitted later,
1571 * other negative values denotes an error.
1572 */
1573 static int s5p_hash_final(struct ahash_request *req)
1574 {
1575 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1576
1577 ctx->finup = true;
1578 if (ctx->error)
1579 return -EINVAL; /* uncompleted hash is not needed */
1580
1581 if (!ctx->digcnt && ctx->bufcnt < BUFLEN)
1582 return s5p_hash_final_shash(req);
1583
1584 return s5p_hash_enqueue(req, false); /* HASH_OP_FINAL */
1585 }
1586
1587 /**
1588 * s5p_hash_finup() - process last req->src and calculate digest
1589 * @req: AHASH request containing the last update data
1590 *
1591 * Return values: see s5p_hash_final above.
1592 */
1593 static int s5p_hash_finup(struct ahash_request *req)
1594 {
1595 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1596 int err1, err2;
1597
1598 ctx->finup = true;
1599
1600 err1 = s5p_hash_update(req);
1601 if (err1 == -EINPROGRESS || err1 == -EBUSY)
1602 return err1;
1603
1604 /*
1605 * final() has to be always called to cleanup resources even if
1606 * update() failed, except EINPROGRESS or calculate digest for small
1607 * size
1608 */
1609 err2 = s5p_hash_final(req);
1610
1611 return err1 ?: err2;
1612 }
1613
1614 /**
1615 * s5p_hash_init() - initialize AHASH request contex
1616 * @req: AHASH request
1617 *
1618 * Init async hash request context.
1619 */
1620 static int s5p_hash_init(struct ahash_request *req)
1621 {
1622 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1623 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1624 struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);
1625
1626 ctx->dd = tctx->dd;
1627 ctx->error = false;
1628 ctx->finup = false;
1629 ctx->bufcnt = 0;
1630 ctx->digcnt = 0;
1631 ctx->total = 0;
1632 ctx->skip = 0;
1633
1634 dev_dbg(tctx->dd->dev, "init: digest size: %d\n",
1635 crypto_ahash_digestsize(tfm));
1636
1637 switch (crypto_ahash_digestsize(tfm)) {
1638 case MD5_DIGEST_SIZE:
1639 ctx->engine = SSS_HASH_ENGINE_MD5;
1640 ctx->nregs = HASH_MD5_MAX_REG;
1641 break;
1642 case SHA1_DIGEST_SIZE:
1643 ctx->engine = SSS_HASH_ENGINE_SHA1;
1644 ctx->nregs = HASH_SHA1_MAX_REG;
1645 break;
1646 case SHA256_DIGEST_SIZE:
1647 ctx->engine = SSS_HASH_ENGINE_SHA256;
1648 ctx->nregs = HASH_SHA256_MAX_REG;
1649 break;
1650 default:
1651 ctx->error = true;
1652 return -EINVAL;
1653 }
1654
1655 return 0;
1656 }
1657
1658 /**
1659 * s5p_hash_digest - calculate digest from req->src
1660 * @req: AHASH request
1661 *
1662 * Return values: see s5p_hash_final above.
1663 */
1664 static int s5p_hash_digest(struct ahash_request *req)
1665 {
1666 return s5p_hash_init(req) ?: s5p_hash_finup(req);
1667 }
1668
1669 /**
1670 * s5p_hash_cra_init_alg - init crypto alg transformation
1671 * @tfm: crypto transformation
1672 */
1673 static int s5p_hash_cra_init_alg(struct crypto_tfm *tfm)
1674 {
1675 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);
1676 const char *alg_name = crypto_tfm_alg_name(tfm);
1677
1678 tctx->dd = s5p_dev;
1679 /* Allocate a fallback and abort if it failed. */
1680 tctx->fallback = crypto_alloc_shash(alg_name, 0,
1681 CRYPTO_ALG_NEED_FALLBACK);
1682 if (IS_ERR(tctx->fallback)) {
1683 pr_err("fallback alloc fails for '%s'\n", alg_name);
1684 return PTR_ERR(tctx->fallback);
1685 }
1686
1687 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1688 sizeof(struct s5p_hash_reqctx) + BUFLEN);
1689
1690 return 0;
1691 }
1692
1693 /**
1694 * s5p_hash_cra_init - init crypto tfm
1695 * @tfm: crypto transformation
1696 */
1697 static int s5p_hash_cra_init(struct crypto_tfm *tfm)
1698 {
1699 return s5p_hash_cra_init_alg(tfm);
1700 }
1701
1702 /**
1703 * s5p_hash_cra_exit - exit crypto tfm
1704 * @tfm: crypto transformation
1705 *
1706 * free allocated fallback
1707 */
1708 static void s5p_hash_cra_exit(struct crypto_tfm *tfm)
1709 {
1710 struct s5p_hash_ctx *tctx = crypto_tfm_ctx(tfm);
1711
1712 crypto_free_shash(tctx->fallback);
1713 tctx->fallback = NULL;
1714 }
1715
1716 /**
1717 * s5p_hash_export - export hash state
1718 * @req: AHASH request
1719 * @out: buffer for exported state
1720 */
1721 static int s5p_hash_export(struct ahash_request *req, void *out)
1722 {
1723 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1724
1725 memcpy(out, ctx, sizeof(*ctx) + ctx->bufcnt);
1726
1727 return 0;
1728 }
1729
1730 /**
1731 * s5p_hash_import - import hash state
1732 * @req: AHASH request
1733 * @in: buffer with state to be imported from
1734 */
1735 static int s5p_hash_import(struct ahash_request *req, const void *in)
1736 {
1737 struct s5p_hash_reqctx *ctx = ahash_request_ctx(req);
1738 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1739 struct s5p_hash_ctx *tctx = crypto_ahash_ctx(tfm);
1740 const struct s5p_hash_reqctx *ctx_in = in;
1741
1742 memcpy(ctx, in, sizeof(*ctx) + BUFLEN);
1743 if (ctx_in->bufcnt > BUFLEN) {
1744 ctx->error = true;
1745 return -EINVAL;
1746 }
1747
1748 ctx->dd = tctx->dd;
1749 ctx->error = false;
1750
1751 return 0;
1752 }
1753
1754 static struct ahash_alg algs_sha1_md5_sha256[] = {
1755 {
1756 .init = s5p_hash_init,
1757 .update = s5p_hash_update,
1758 .final = s5p_hash_final,
1759 .finup = s5p_hash_finup,
1760 .digest = s5p_hash_digest,
1761 .export = s5p_hash_export,
1762 .import = s5p_hash_import,
1763 .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1764 .halg.digestsize = SHA1_DIGEST_SIZE,
1765 .halg.base = {
1766 .cra_name = "sha1",
1767 .cra_driver_name = "exynos-sha1",
1768 .cra_priority = 100,
1769 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1770 CRYPTO_ALG_KERN_DRIVER_ONLY |
1771 CRYPTO_ALG_ASYNC |
1772 CRYPTO_ALG_NEED_FALLBACK,
1773 .cra_blocksize = HASH_BLOCK_SIZE,
1774 .cra_ctxsize = sizeof(struct s5p_hash_ctx),
1775 .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK,
1776 .cra_module = THIS_MODULE,
1777 .cra_init = s5p_hash_cra_init,
1778 .cra_exit = s5p_hash_cra_exit,
1779 }
1780 },
1781 {
1782 .init = s5p_hash_init,
1783 .update = s5p_hash_update,
1784 .final = s5p_hash_final,
1785 .finup = s5p_hash_finup,
1786 .digest = s5p_hash_digest,
1787 .export = s5p_hash_export,
1788 .import = s5p_hash_import,
1789 .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1790 .halg.digestsize = MD5_DIGEST_SIZE,
1791 .halg.base = {
1792 .cra_name = "md5",
1793 .cra_driver_name = "exynos-md5",
1794 .cra_priority = 100,
1795 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1796 CRYPTO_ALG_KERN_DRIVER_ONLY |
1797 CRYPTO_ALG_ASYNC |
1798 CRYPTO_ALG_NEED_FALLBACK,
1799 .cra_blocksize = HASH_BLOCK_SIZE,
1800 .cra_ctxsize = sizeof(struct s5p_hash_ctx),
1801 .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK,
1802 .cra_module = THIS_MODULE,
1803 .cra_init = s5p_hash_cra_init,
1804 .cra_exit = s5p_hash_cra_exit,
1805 }
1806 },
1807 {
1808 .init = s5p_hash_init,
1809 .update = s5p_hash_update,
1810 .final = s5p_hash_final,
1811 .finup = s5p_hash_finup,
1812 .digest = s5p_hash_digest,
1813 .export = s5p_hash_export,
1814 .import = s5p_hash_import,
1815 .halg.statesize = sizeof(struct s5p_hash_reqctx) + BUFLEN,
1816 .halg.digestsize = SHA256_DIGEST_SIZE,
1817 .halg.base = {
1818 .cra_name = "sha256",
1819 .cra_driver_name = "exynos-sha256",
1820 .cra_priority = 100,
1821 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
1822 CRYPTO_ALG_KERN_DRIVER_ONLY |
1823 CRYPTO_ALG_ASYNC |
1824 CRYPTO_ALG_NEED_FALLBACK,
1825 .cra_blocksize = HASH_BLOCK_SIZE,
1826 .cra_ctxsize = sizeof(struct s5p_hash_ctx),
1827 .cra_alignmask = SSS_HASH_DMA_ALIGN_MASK,
1828 .cra_module = THIS_MODULE,
1829 .cra_init = s5p_hash_cra_init,
1830 .cra_exit = s5p_hash_cra_exit,
1831 }
1832 }
1833
1834 };
1835
1836 static void s5p_set_aes(struct s5p_aes_dev *dev,
1837 uint8_t *key, uint8_t *iv, unsigned int keylen)
1838 {
1839 void __iomem *keystart;
1840
1841 if (iv)
1842 memcpy_toio(dev->aes_ioaddr + SSS_REG_AES_IV_DATA(0), iv, 0x10);
1843
1844 if (keylen == AES_KEYSIZE_256)
1845 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(0);
1846 else if (keylen == AES_KEYSIZE_192)
1847 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(2);
1848 else
1849 keystart = dev->aes_ioaddr + SSS_REG_AES_KEY_DATA(4);
1850
1851 memcpy_toio(keystart, key, keylen);
1852 }
1853
1854 static bool s5p_is_sg_aligned(struct scatterlist *sg)
1855 {
1856 while (sg) {
1857 if (!IS_ALIGNED(sg->length, AES_BLOCK_SIZE))
1858 return false;
1859 sg = sg_next(sg);
1860 }
1861
1862 return true;
1863 }
1864
1865 static int s5p_set_indata_start(struct s5p_aes_dev *dev,
1866 struct ablkcipher_request *req)
1867 {
1868 struct scatterlist *sg;
1869 int err;
1870
1871 dev->sg_src_cpy = NULL;
1872 sg = req->src;
1873 if (!s5p_is_sg_aligned(sg)) {
1874 dev_dbg(dev->dev,
1875 "At least one unaligned source scatter list, making a copy\n");
1876 err = s5p_make_sg_cpy(dev, sg, &dev->sg_src_cpy);
1877 if (err)
1878 return err;
1879
1880 sg = dev->sg_src_cpy;
1881 }
1882
1883 err = s5p_set_indata(dev, sg);
1884 if (err) {
1885 s5p_free_sg_cpy(dev, &dev->sg_src_cpy);
1886 return err;
1887 }
1888
1889 return 0;
1890 }
1891
1892 static int s5p_set_outdata_start(struct s5p_aes_dev *dev,
1893 struct ablkcipher_request *req)
1894 {
1895 struct scatterlist *sg;
1896 int err;
1897
1898 dev->sg_dst_cpy = NULL;
1899 sg = req->dst;
1900 if (!s5p_is_sg_aligned(sg)) {
1901 dev_dbg(dev->dev,
1902 "At least one unaligned dest scatter list, making a copy\n");
1903 err = s5p_make_sg_cpy(dev, sg, &dev->sg_dst_cpy);
1904 if (err)
1905 return err;
1906
1907 sg = dev->sg_dst_cpy;
1908 }
1909
1910 err = s5p_set_outdata(dev, sg);
1911 if (err) {
1912 s5p_free_sg_cpy(dev, &dev->sg_dst_cpy);
1913 return err;
1914 }
1915
1916 return 0;
1917 }
1918
1919 static void s5p_aes_crypt_start(struct s5p_aes_dev *dev, unsigned long mode)
1920 {
1921 struct ablkcipher_request *req = dev->req;
1922 uint32_t aes_control;
1923 unsigned long flags;
1924 int err;
1925 u8 *iv;
1926
1927 aes_control = SSS_AES_KEY_CHANGE_MODE;
1928 if (mode & FLAGS_AES_DECRYPT)
1929 aes_control |= SSS_AES_MODE_DECRYPT;
1930
1931 if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CBC) {
1932 aes_control |= SSS_AES_CHAIN_MODE_CBC;
1933 iv = req->info;
1934 } else if ((mode & FLAGS_AES_MODE_MASK) == FLAGS_AES_CTR) {
1935 aes_control |= SSS_AES_CHAIN_MODE_CTR;
1936 iv = req->info;
1937 } else {
1938 iv = NULL; /* AES_ECB */
1939 }
1940
1941 if (dev->ctx->keylen == AES_KEYSIZE_192)
1942 aes_control |= SSS_AES_KEY_SIZE_192;
1943 else if (dev->ctx->keylen == AES_KEYSIZE_256)
1944 aes_control |= SSS_AES_KEY_SIZE_256;
1945
1946 aes_control |= SSS_AES_FIFO_MODE;
1947
1948 /* as a variant it is possible to use byte swapping on DMA side */
1949 aes_control |= SSS_AES_BYTESWAP_DI
1950 | SSS_AES_BYTESWAP_DO
1951 | SSS_AES_BYTESWAP_IV
1952 | SSS_AES_BYTESWAP_KEY
1953 | SSS_AES_BYTESWAP_CNT;
1954
1955 spin_lock_irqsave(&dev->lock, flags);
1956
1957 SSS_WRITE(dev, FCINTENCLR,
1958 SSS_FCINTENCLR_BTDMAINTENCLR | SSS_FCINTENCLR_BRDMAINTENCLR);
1959 SSS_WRITE(dev, FCFIFOCTRL, 0x00);
1960
1961 err = s5p_set_indata_start(dev, req);
1962 if (err)
1963 goto indata_error;
1964
1965 err = s5p_set_outdata_start(dev, req);
1966 if (err)
1967 goto outdata_error;
1968
1969 SSS_AES_WRITE(dev, AES_CONTROL, aes_control);
1970 s5p_set_aes(dev, dev->ctx->aes_key, iv, dev->ctx->keylen);
1971
1972 s5p_set_dma_indata(dev, dev->sg_src);
1973 s5p_set_dma_outdata(dev, dev->sg_dst);
1974
1975 SSS_WRITE(dev, FCINTENSET,
1976 SSS_FCINTENSET_BTDMAINTENSET | SSS_FCINTENSET_BRDMAINTENSET);
1977
1978 spin_unlock_irqrestore(&dev->lock, flags);
1979
1980 return;
1981
1982 outdata_error:
1983 s5p_unset_indata(dev);
1984
1985 indata_error:
1986 s5p_sg_done(dev);
1987 dev->busy = false;
1988 spin_unlock_irqrestore(&dev->lock, flags);
1989 s5p_aes_complete(dev, err);
1990 }
1991
1992 static void s5p_tasklet_cb(unsigned long data)
1993 {
1994 struct s5p_aes_dev *dev = (struct s5p_aes_dev *)data;
1995 struct crypto_async_request *async_req, *backlog;
1996 struct s5p_aes_reqctx *reqctx;
1997 unsigned long flags;
1998
1999 spin_lock_irqsave(&dev->lock, flags);
2000 backlog = crypto_get_backlog(&dev->queue);
2001 async_req = crypto_dequeue_request(&dev->queue);
2002
2003 if (!async_req) {
2004 dev->busy = false;
2005 spin_unlock_irqrestore(&dev->lock, flags);
2006 return;
2007 }
2008 spin_unlock_irqrestore(&dev->lock, flags);
2009
2010 if (backlog)
2011 backlog->complete(backlog, -EINPROGRESS);
2012
2013 dev->req = ablkcipher_request_cast(async_req);
2014 dev->ctx = crypto_tfm_ctx(dev->req->base.tfm);
2015 reqctx = ablkcipher_request_ctx(dev->req);
2016
2017 s5p_aes_crypt_start(dev, reqctx->mode);
2018 }
2019
2020 static int s5p_aes_handle_req(struct s5p_aes_dev *dev,
2021 struct ablkcipher_request *req)
2022 {
2023 unsigned long flags;
2024 int err;
2025
2026 spin_lock_irqsave(&dev->lock, flags);
2027 err = ablkcipher_enqueue_request(&dev->queue, req);
2028 if (dev->busy) {
2029 spin_unlock_irqrestore(&dev->lock, flags);
2030 goto exit;
2031 }
2032 dev->busy = true;
2033
2034 spin_unlock_irqrestore(&dev->lock, flags);
2035
2036 tasklet_schedule(&dev->tasklet);
2037
2038 exit:
2039 return err;
2040 }
2041
2042 static int s5p_aes_crypt(struct ablkcipher_request *req, unsigned long mode)
2043 {
2044 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
2045 struct s5p_aes_reqctx *reqctx = ablkcipher_request_ctx(req);
2046 struct s5p_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
2047 struct s5p_aes_dev *dev = ctx->dev;
2048
2049 if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) {
2050 dev_err(dev->dev, "request size is not exact amount of AES blocks\n");
2051 return -EINVAL;
2052 }
2053
2054 reqctx->mode = mode;
2055
2056 return s5p_aes_handle_req(dev, req);
2057 }
2058
2059 static int s5p_aes_setkey(struct crypto_ablkcipher *cipher,
2060 const uint8_t *key, unsigned int keylen)
2061 {
2062 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
2063 struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
2064
2065 if (keylen != AES_KEYSIZE_128 &&
2066 keylen != AES_KEYSIZE_192 &&
2067 keylen != AES_KEYSIZE_256)
2068 return -EINVAL;
2069
2070 memcpy(ctx->aes_key, key, keylen);
2071 ctx->keylen = keylen;
2072
2073 return 0;
2074 }
2075
2076 static int s5p_aes_ecb_encrypt(struct ablkcipher_request *req)
2077 {
2078 return s5p_aes_crypt(req, 0);
2079 }
2080
2081 static int s5p_aes_ecb_decrypt(struct ablkcipher_request *req)
2082 {
2083 return s5p_aes_crypt(req, FLAGS_AES_DECRYPT);
2084 }
2085
2086 static int s5p_aes_cbc_encrypt(struct ablkcipher_request *req)
2087 {
2088 return s5p_aes_crypt(req, FLAGS_AES_CBC);
2089 }
2090
2091 static int s5p_aes_cbc_decrypt(struct ablkcipher_request *req)
2092 {
2093 return s5p_aes_crypt(req, FLAGS_AES_DECRYPT | FLAGS_AES_CBC);
2094 }
2095
2096 static int s5p_aes_cra_init(struct crypto_tfm *tfm)
2097 {
2098 struct s5p_aes_ctx *ctx = crypto_tfm_ctx(tfm);
2099
2100 ctx->dev = s5p_dev;
2101 tfm->crt_ablkcipher.reqsize = sizeof(struct s5p_aes_reqctx);
2102
2103 return 0;
2104 }
2105
2106 static struct crypto_alg algs[] = {
2107 {
2108 .cra_name = "ecb(aes)",
2109 .cra_driver_name = "ecb-aes-s5p",
2110 .cra_priority = 100,
2111 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2112 CRYPTO_ALG_ASYNC |
2113 CRYPTO_ALG_KERN_DRIVER_ONLY,
2114 .cra_blocksize = AES_BLOCK_SIZE,
2115 .cra_ctxsize = sizeof(struct s5p_aes_ctx),
2116 .cra_alignmask = 0x0f,
2117 .cra_type = &crypto_ablkcipher_type,
2118 .cra_module = THIS_MODULE,
2119 .cra_init = s5p_aes_cra_init,
2120 .cra_u.ablkcipher = {
2121 .min_keysize = AES_MIN_KEY_SIZE,
2122 .max_keysize = AES_MAX_KEY_SIZE,
2123 .setkey = s5p_aes_setkey,
2124 .encrypt = s5p_aes_ecb_encrypt,
2125 .decrypt = s5p_aes_ecb_decrypt,
2126 }
2127 },
2128 {
2129 .cra_name = "cbc(aes)",
2130 .cra_driver_name = "cbc-aes-s5p",
2131 .cra_priority = 100,
2132 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2133 CRYPTO_ALG_ASYNC |
2134 CRYPTO_ALG_KERN_DRIVER_ONLY,
2135 .cra_blocksize = AES_BLOCK_SIZE,
2136 .cra_ctxsize = sizeof(struct s5p_aes_ctx),
2137 .cra_alignmask = 0x0f,
2138 .cra_type = &crypto_ablkcipher_type,
2139 .cra_module = THIS_MODULE,
2140 .cra_init = s5p_aes_cra_init,
2141 .cra_u.ablkcipher = {
2142 .min_keysize = AES_MIN_KEY_SIZE,
2143 .max_keysize = AES_MAX_KEY_SIZE,
2144 .ivsize = AES_BLOCK_SIZE,
2145 .setkey = s5p_aes_setkey,
2146 .encrypt = s5p_aes_cbc_encrypt,
2147 .decrypt = s5p_aes_cbc_decrypt,
2148 }
2149 },
2150 };
2151
2152 static int s5p_aes_probe(struct platform_device *pdev)
2153 {
2154 struct device *dev = &pdev->dev;
2155 int i, j, err = -ENODEV;
2156 struct samsung_aes_variant *variant;
2157 struct s5p_aes_dev *pdata;
2158 struct resource *res;
2159 unsigned int hash_i;
2160
2161 if (s5p_dev)
2162 return -EEXIST;
2163
2164 pdata = devm_kzalloc(dev, sizeof(*pdata), GFP_KERNEL);
2165 if (!pdata)
2166 return -ENOMEM;
2167
2168 variant = find_s5p_sss_version(pdev);
2169 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2170
2171 /*
2172 * Note: HASH and PRNG uses the same registers in secss, avoid
2173 * overwrite each other. This will drop HASH when CONFIG_EXYNOS_RNG
2174 * is enabled in config. We need larger size for HASH registers in
2175 * secss, current describe only AES/DES
2176 */
2177 if (IS_ENABLED(CONFIG_CRYPTO_DEV_EXYNOS_HASH)) {
2178 if (variant == &exynos_aes_data) {
2179 res->end += 0x300;
2180 pdata->use_hash = true;
2181 }
2182 }
2183
2184 pdata->res = res;
2185 pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res);
2186 if (IS_ERR(pdata->ioaddr)) {
2187 if (!pdata->use_hash)
2188 return PTR_ERR(pdata->ioaddr);
2189 /* try AES without HASH */
2190 res->end -= 0x300;
2191 pdata->use_hash = false;
2192 pdata->ioaddr = devm_ioremap_resource(&pdev->dev, res);
2193 if (IS_ERR(pdata->ioaddr))
2194 return PTR_ERR(pdata->ioaddr);
2195 }
2196
2197 pdata->clk = devm_clk_get(dev, "secss");
2198 if (IS_ERR(pdata->clk)) {
2199 dev_err(dev, "failed to find secss clock source\n");
2200 return -ENOENT;
2201 }
2202
2203 err = clk_prepare_enable(pdata->clk);
2204 if (err < 0) {
2205 dev_err(dev, "Enabling SSS clk failed, err %d\n", err);
2206 return err;
2207 }
2208
2209 spin_lock_init(&pdata->lock);
2210 spin_lock_init(&pdata->hash_lock);
2211
2212 pdata->aes_ioaddr = pdata->ioaddr + variant->aes_offset;
2213 pdata->io_hash_base = pdata->ioaddr + variant->hash_offset;
2214
2215 pdata->irq_fc = platform_get_irq(pdev, 0);
2216 if (pdata->irq_fc < 0) {
2217 err = pdata->irq_fc;
2218 dev_warn(dev, "feed control interrupt is not available.\n");
2219 goto err_irq;
2220 }
2221 err = devm_request_threaded_irq(dev, pdata->irq_fc, NULL,
2222 s5p_aes_interrupt, IRQF_ONESHOT,
2223 pdev->name, pdev);
2224 if (err < 0) {
2225 dev_warn(dev, "feed control interrupt is not available.\n");
2226 goto err_irq;
2227 }
2228
2229 pdata->busy = false;
2230 pdata->dev = dev;
2231 platform_set_drvdata(pdev, pdata);
2232 s5p_dev = pdata;
2233
2234 tasklet_init(&pdata->tasklet, s5p_tasklet_cb, (unsigned long)pdata);
2235 crypto_init_queue(&pdata->queue, CRYPTO_QUEUE_LEN);
2236
2237 for (i = 0; i < ARRAY_SIZE(algs); i++) {
2238 err = crypto_register_alg(&algs[i]);
2239 if (err)
2240 goto err_algs;
2241 }
2242
2243 if (pdata->use_hash) {
2244 tasklet_init(&pdata->hash_tasklet, s5p_hash_tasklet_cb,
2245 (unsigned long)pdata);
2246 crypto_init_queue(&pdata->hash_queue, SSS_HASH_QUEUE_LENGTH);
2247
2248 for (hash_i = 0; hash_i < ARRAY_SIZE(algs_sha1_md5_sha256);
2249 hash_i++) {
2250 struct ahash_alg *alg;
2251
2252 alg = &algs_sha1_md5_sha256[hash_i];
2253 err = crypto_register_ahash(alg);
2254 if (err) {
2255 dev_err(dev, "can't register '%s': %d\n",
2256 alg->halg.base.cra_driver_name, err);
2257 goto err_hash;
2258 }
2259 }
2260 }
2261
2262 dev_info(dev, "s5p-sss driver registered\n");
2263
2264 return 0;
2265
2266 err_hash:
2267 for (j = hash_i - 1; j >= 0; j--)
2268 crypto_unregister_ahash(&algs_sha1_md5_sha256[j]);
2269
2270 tasklet_kill(&pdata->hash_tasklet);
2271 res->end -= 0x300;
2272
2273 err_algs:
2274 if (i < ARRAY_SIZE(algs))
2275 dev_err(dev, "can't register '%s': %d\n", algs[i].cra_name,
2276 err);
2277
2278 for (j = 0; j < i; j++)
2279 crypto_unregister_alg(&algs[j]);
2280
2281 tasklet_kill(&pdata->tasklet);
2282
2283 err_irq:
2284 clk_disable_unprepare(pdata->clk);
2285
2286 s5p_dev = NULL;
2287
2288 return err;
2289 }
2290
2291 static int s5p_aes_remove(struct platform_device *pdev)
2292 {
2293 struct s5p_aes_dev *pdata = platform_get_drvdata(pdev);
2294 int i;
2295
2296 if (!pdata)
2297 return -ENODEV;
2298
2299 for (i = 0; i < ARRAY_SIZE(algs); i++)
2300 crypto_unregister_alg(&algs[i]);
2301
2302 tasklet_kill(&pdata->tasklet);
2303 if (pdata->use_hash) {
2304 for (i = ARRAY_SIZE(algs_sha1_md5_sha256) - 1; i >= 0; i--)
2305 crypto_unregister_ahash(&algs_sha1_md5_sha256[i]);
2306
2307 pdata->res->end -= 0x300;
2308 tasklet_kill(&pdata->hash_tasklet);
2309 pdata->use_hash = false;
2310 }
2311
2312 clk_disable_unprepare(pdata->clk);
2313 s5p_dev = NULL;
2314
2315 return 0;
2316 }
2317
2318 static struct platform_driver s5p_aes_crypto = {
2319 .probe = s5p_aes_probe,
2320 .remove = s5p_aes_remove,
2321 .driver = {
2322 .name = "s5p-secss",
2323 .of_match_table = s5p_sss_dt_match,
2324 },
2325 };
2326
2327 module_platform_driver(s5p_aes_crypto);
2328
2329 MODULE_DESCRIPTION("S5PV210 AES hw acceleration support.");
2330 MODULE_LICENSE("GPL v2");
2331 MODULE_AUTHOR("Vladimir Zapolskiy <vzapolskiy@gmail.com>");
2332 MODULE_AUTHOR("Kamil Konieczny <k.konieczny@partner.samsung.com>");
CRIS linux kernel tree