?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
[cris-mirror.git] / drivers / crypto / axis / artpec6_crypto.c
blob0fb8bbf41a8d9b33a11528bf142fc6c9721c0dfe
1 /*
2 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api.
3 *
4 * Copyright (C) 2014-2017 Axis Communications AB
5 */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/bitfield.h>
9 #include <linux/crypto.h>
10 #include <linux/debugfs.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/fault-inject.h>
14 #include <linux/init.h>
15 #include <linux/interrupt.h>
16 #include <linux/kernel.h>
17 #include <linux/list.h>
18 #include <linux/module.h>
19 #include <linux/of.h>
20 #include <linux/platform_device.h>
21 #include <linux/scatterlist.h>
22 #include <linux/slab.h>
23
24 #include <crypto/aes.h>
25 #include <crypto/gcm.h>
26 #include <crypto/internal/aead.h>
27 #include <crypto/internal/hash.h>
28 #include <crypto/internal/skcipher.h>
29 #include <crypto/scatterwalk.h>
30 #include <crypto/sha.h>
31 #include <crypto/xts.h>
32
33 /* Max length of a line in all cache levels for Artpec SoCs. */
34 #define ARTPEC_CACHE_LINE_MAX 32
35
36 #define PDMA_OUT_CFG 0x0000
37 #define PDMA_OUT_BUF_CFG 0x0004
38 #define PDMA_OUT_CMD 0x0008
39 #define PDMA_OUT_DESCRQ_PUSH 0x0010
40 #define PDMA_OUT_DESCRQ_STAT 0x0014
41
42 #define A6_PDMA_IN_CFG 0x0028
43 #define A6_PDMA_IN_BUF_CFG 0x002c
44 #define A6_PDMA_IN_CMD 0x0030
45 #define A6_PDMA_IN_STATQ_PUSH 0x0038
46 #define A6_PDMA_IN_DESCRQ_PUSH 0x0044
47 #define A6_PDMA_IN_DESCRQ_STAT 0x0048
48 #define A6_PDMA_INTR_MASK 0x0068
49 #define A6_PDMA_ACK_INTR 0x006c
50 #define A6_PDMA_MASKED_INTR 0x0074
51
52 #define A7_PDMA_IN_CFG 0x002c
53 #define A7_PDMA_IN_BUF_CFG 0x0030
54 #define A7_PDMA_IN_CMD 0x0034
55 #define A7_PDMA_IN_STATQ_PUSH 0x003c
56 #define A7_PDMA_IN_DESCRQ_PUSH 0x0048
57 #define A7_PDMA_IN_DESCRQ_STAT 0x004C
58 #define A7_PDMA_INTR_MASK 0x006c
59 #define A7_PDMA_ACK_INTR 0x0070
60 #define A7_PDMA_MASKED_INTR 0x0078
61
62 #define PDMA_OUT_CFG_EN BIT(0)
63
64 #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
65 #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
66
67 #define PDMA_OUT_CMD_START BIT(0)
68 #define A6_PDMA_OUT_CMD_STOP BIT(3)
69 #define A7_PDMA_OUT_CMD_STOP BIT(2)
70
71 #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0)
72 #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6)
73
74 #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0)
75 #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4)
76
77 #define PDMA_IN_CFG_EN BIT(0)
78
79 #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
80 #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
81 #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10)
82
83 #define PDMA_IN_CMD_START BIT(0)
84 #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2)
85 #define A6_PDMA_IN_CMD_STOP BIT(3)
86 #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1)
87 #define A7_PDMA_IN_CMD_STOP BIT(2)
88
89 #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0)
90 #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6)
91
92 #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0)
93 #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6)
94
95 #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0)
96 #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4)
97
98 #define A6_PDMA_INTR_MASK_IN_DATA BIT(2)
99 #define A6_PDMA_INTR_MASK_IN_EOP BIT(3)
100 #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4)
101
102 #define A7_PDMA_INTR_MASK_IN_DATA BIT(3)
103 #define A7_PDMA_INTR_MASK_IN_EOP BIT(4)
104 #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5)
105
106 #define A6_CRY_MD_OPER GENMASK(19, 16)
107
108 #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20)
109 #define A6_CRY_MD_HASH_HMAC_FIN BIT(23)
110
111 #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20)
112 #define A6_CRY_MD_CIPHER_DECR BIT(22)
113 #define A6_CRY_MD_CIPHER_TWEAK BIT(23)
114 #define A6_CRY_MD_CIPHER_DSEQ BIT(24)
115
116 #define A7_CRY_MD_OPER GENMASK(11, 8)
117
118 #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12)
119 #define A7_CRY_MD_HASH_HMAC_FIN BIT(15)
120
121 #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12)
122 #define A7_CRY_MD_CIPHER_DECR BIT(14)
123 #define A7_CRY_MD_CIPHER_TWEAK BIT(15)
124 #define A7_CRY_MD_CIPHER_DSEQ BIT(16)
125
126 /* DMA metadata constants */
127 #define regk_crypto_aes_cbc 0x00000002
128 #define regk_crypto_aes_ctr 0x00000003
129 #define regk_crypto_aes_ecb 0x00000001
130 #define regk_crypto_aes_gcm 0x00000004
131 #define regk_crypto_aes_xts 0x00000005
132 #define regk_crypto_cache 0x00000002
133 #define a6_regk_crypto_dlkey 0x0000000a
134 #define a7_regk_crypto_dlkey 0x0000000e
135 #define regk_crypto_ext 0x00000001
136 #define regk_crypto_hmac_sha1 0x00000007
137 #define regk_crypto_hmac_sha256 0x00000009
138 #define regk_crypto_hmac_sha384 0x0000000b
139 #define regk_crypto_hmac_sha512 0x0000000d
140 #define regk_crypto_init 0x00000000
141 #define regk_crypto_key_128 0x00000000
142 #define regk_crypto_key_192 0x00000001
143 #define regk_crypto_key_256 0x00000002
144 #define regk_crypto_null 0x00000000
145 #define regk_crypto_sha1 0x00000006
146 #define regk_crypto_sha256 0x00000008
147 #define regk_crypto_sha384 0x0000000a
148 #define regk_crypto_sha512 0x0000000c
149
150 /* DMA descriptor structures */
151 struct pdma_descr_ctrl {
152 unsigned char short_descr : 1;
153 unsigned char pad1 : 1;
154 unsigned char eop : 1;
155 unsigned char intr : 1;
156 unsigned char short_len : 3;
157 unsigned char pad2 : 1;
158 } __packed;
159
160 struct pdma_data_descr {
161 unsigned int len : 24;
162 unsigned int buf : 32;
163 } __packed;
164
165 struct pdma_short_descr {
166 unsigned char data[7];
167 } __packed;
168
169 struct pdma_descr {
170 struct pdma_descr_ctrl ctrl;
171 union {
172 struct pdma_data_descr data;
173 struct pdma_short_descr shrt;
174 };
175 };
176
177 struct pdma_stat_descr {
178 unsigned char pad1 : 1;
179 unsigned char pad2 : 1;
180 unsigned char eop : 1;
181 unsigned char pad3 : 5;
182 unsigned int len : 24;
183 };
184
185 /* Each descriptor array can hold max 64 entries */
186 #define PDMA_DESCR_COUNT 64
187
188 #define MODULE_NAME "Artpec-6 CA"
189
190 /* Hash modes (including HMAC variants) */
191 #define ARTPEC6_CRYPTO_HASH_SHA1 1
192 #define ARTPEC6_CRYPTO_HASH_SHA256 2
193 #define ARTPEC6_CRYPTO_HASH_SHA384 3
194 #define ARTPEC6_CRYPTO_HASH_SHA512 4
195
196 /* Crypto modes */
197 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1
198 #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2
199 #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3
200 #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5
201
202 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine.
203 * It operates on a descriptor array with up to 64 descriptor entries.
204 * The arrays must be 64 byte aligned in memory.
205 *
206 * The ciphering unit has no registers and is completely controlled by
207 * a 4-byte metadata that is inserted at the beginning of each dma packet.
208 *
209 * A dma packet is a sequence of descriptors terminated by setting the .eop
210 * field in the final descriptor of the packet.
211 *
212 * Multiple packets are used for providing context data, key data and
213 * the plain/ciphertext.
214 *
215 * PDMA Descriptors (Array)
216 * +------+------+------+~~+-------+------+----
217 * | 0 | 1 | 2 |~~| 11 EOP| 12 | ....
218 * +--+---+--+---+----+-+~~+-------+----+-+----
219 * | | | | |
220 * | | | | |
221 * __|__ +-------++-------++-------+ +----+
222 * | MD | |Payload||Payload||Payload| | MD |
223 * +-----+ +-------++-------++-------+ +----+
224 */
225
226 struct artpec6_crypto_bounce_buffer {
227 struct list_head list;
228 size_t length;
229 struct scatterlist *sg;
230 size_t offset;
231 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and
232 * holds up to ARTPEC_CACHE_LINE_MAX bytes data.
233 */
234 void *buf;
235 };
236
237 struct artpec6_crypto_dma_map {
238 dma_addr_t dma_addr;
239 size_t size;
240 enum dma_data_direction dir;
241 };
242
243 struct artpec6_crypto_dma_descriptors {
244 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64);
245 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64);
246 u32 stat[PDMA_DESCR_COUNT] __aligned(64);
247 struct list_head bounce_buffers;
248 /* Enough maps for all out/in buffers, and all three descr. arrays */
249 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2];
250 dma_addr_t out_dma_addr;
251 dma_addr_t in_dma_addr;
252 dma_addr_t stat_dma_addr;
253 size_t out_cnt;
254 size_t in_cnt;
255 size_t map_count;
256 };
257
258 enum artpec6_crypto_variant {
259 ARTPEC6_CRYPTO,
260 ARTPEC7_CRYPTO,
261 };
262
263 struct artpec6_crypto {
264 void __iomem *base;
265 spinlock_t queue_lock;
266 struct list_head queue; /* waiting for pdma fifo space */
267 struct list_head pending; /* submitted to pdma fifo */
268 struct tasklet_struct task;
269 struct kmem_cache *dma_cache;
270 int pending_count;
271 struct timer_list timer;
272 enum artpec6_crypto_variant variant;
273 void *pad_buffer; /* cache-aligned block padding buffer */
274 void *zero_buffer;
275 };
276
277 enum artpec6_crypto_hash_flags {
278 HASH_FLAG_INIT_CTX = 2,
279 HASH_FLAG_UPDATE = 4,
280 HASH_FLAG_FINALIZE = 8,
281 HASH_FLAG_HMAC = 16,
282 HASH_FLAG_UPDATE_KEY = 32,
283 };
284
285 struct artpec6_crypto_req_common {
286 struct list_head list;
287 struct artpec6_crypto_dma_descriptors *dma;
288 struct crypto_async_request *req;
289 void (*complete)(struct crypto_async_request *req);
290 gfp_t gfp_flags;
291 };
292
293 struct artpec6_hash_request_context {
294 char partial_buffer[SHA512_BLOCK_SIZE];
295 char partial_buffer_out[SHA512_BLOCK_SIZE];
296 char key_buffer[SHA512_BLOCK_SIZE];
297 char pad_buffer[SHA512_BLOCK_SIZE + 32];
298 unsigned char digeststate[SHA512_DIGEST_SIZE];
299 size_t partial_bytes;
300 u64 digcnt;
301 u32 key_md;
302 u32 hash_md;
303 enum artpec6_crypto_hash_flags hash_flags;
304 struct artpec6_crypto_req_common common;
305 };
306
307 struct artpec6_hash_export_state {
308 char partial_buffer[SHA512_BLOCK_SIZE];
309 unsigned char digeststate[SHA512_DIGEST_SIZE];
310 size_t partial_bytes;
311 u64 digcnt;
312 int oper;
313 unsigned int hash_flags;
314 };
315
316 struct artpec6_hashalg_context {
317 char hmac_key[SHA512_BLOCK_SIZE];
318 size_t hmac_key_length;
319 struct crypto_shash *child_hash;
320 };
321
322 struct artpec6_crypto_request_context {
323 u32 cipher_md;
324 bool decrypt;
325 struct artpec6_crypto_req_common common;
326 };
327
328 struct artpec6_cryptotfm_context {
329 unsigned char aes_key[2*AES_MAX_KEY_SIZE];
330 size_t key_length;
331 u32 key_md;
332 int crypto_type;
333 struct crypto_skcipher *fallback;
334 };
335
336 struct artpec6_crypto_aead_hw_ctx {
337 __be64 aad_length_bits;
338 __be64 text_length_bits;
339 __u8 J0[AES_BLOCK_SIZE];
340 };
341
342 struct artpec6_crypto_aead_req_ctx {
343 struct artpec6_crypto_aead_hw_ctx hw_ctx;
344 u32 cipher_md;
345 bool decrypt;
346 struct artpec6_crypto_req_common common;
347 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned;
348 };
349
350 /* The crypto framework makes it hard to avoid this global. */
351 static struct device *artpec6_crypto_dev;
352
353 #ifdef CONFIG_FAULT_INJECTION
354 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read);
355 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full);
356 #endif
357
358 enum {
359 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START,
360 ARTPEC6_CRYPTO_PREPARE_HASH_START,
361 };
362
363 static int artpec6_crypto_prepare_aead(struct aead_request *areq);
364 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq);
365 static int artpec6_crypto_prepare_hash(struct ahash_request *areq);
366
367 static void
368 artpec6_crypto_complete_crypto(struct crypto_async_request *req);
369 static void
370 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req);
371 static void
372 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req);
373 static void
374 artpec6_crypto_complete_aead(struct crypto_async_request *req);
375 static void
376 artpec6_crypto_complete_hash(struct crypto_async_request *req);
377
378 static int
379 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common);
380
381 static void
382 artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common);
383
384 struct artpec6_crypto_walk {
385 struct scatterlist *sg;
386 size_t offset;
387 };
388
389 static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk,
390 struct scatterlist *sg)
391 {
392 awalk->sg = sg;
393 awalk->offset = 0;
394 }
395
396 static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk,
397 size_t nbytes)
398 {
399 while (nbytes && awalk->sg) {
400 size_t piece;
401
402 WARN_ON(awalk->offset > awalk->sg->length);
403
404 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset);
405 nbytes -= piece;
406 awalk->offset += piece;
407 if (awalk->offset == awalk->sg->length) {
408 awalk->sg = sg_next(awalk->sg);
409 awalk->offset = 0;
410 }
411
412 }
413
414 return nbytes;
415 }
416
417 static size_t
418 artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk)
419 {
420 WARN_ON(awalk->sg->length == awalk->offset);
421
422 return awalk->sg->length - awalk->offset;
423 }
424
425 static dma_addr_t
426 artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk)
427 {
428 return sg_phys(awalk->sg) + awalk->offset;
429 }
430
431 static void
432 artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common)
433 {
434 struct artpec6_crypto_dma_descriptors *dma = common->dma;
435 struct artpec6_crypto_bounce_buffer *b;
436 struct artpec6_crypto_bounce_buffer *next;
437
438 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
439 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n",
440 b, b->length, b->offset, b->buf);
441 sg_pcopy_from_buffer(b->sg,
442 1,
443 b->buf,
444 b->length,
445 b->offset);
446
447 list_del(&b->list);
448 kfree(b);
449 }
450 }
451
452 static inline bool artpec6_crypto_busy(void)
453 {
454 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
455 int fifo_count = ac->pending_count;
456
457 return fifo_count > 6;
458 }
459
460 static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req)
461 {
462 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
463 int ret = -EBUSY;
464
465 spin_lock_bh(&ac->queue_lock);
466
467 if (!artpec6_crypto_busy()) {
468 list_add_tail(&req->list, &ac->pending);
469 artpec6_crypto_start_dma(req);
470 ret = -EINPROGRESS;
471 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
472 list_add_tail(&req->list, &ac->queue);
473 } else {
474 artpec6_crypto_common_destroy(req);
475 }
476
477 spin_unlock_bh(&ac->queue_lock);
478
479 return ret;
480 }
481
482 static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common)
483 {
484 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
485 enum artpec6_crypto_variant variant = ac->variant;
486 void __iomem *base = ac->base;
487 struct artpec6_crypto_dma_descriptors *dma = common->dma;
488 u32 ind, statd, outd;
489
490 /* Make descriptor content visible to the DMA before starting it. */
491 wmb();
492
493 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |
494 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6);
495
496 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |
497 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6);
498
499 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |
500 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6);
501
502 if (variant == ARTPEC6_CRYPTO) {
503 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH);
504 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH);
505 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD);
506 } else {
507 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH);
508 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH);
509 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD);
510 }
511
512 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH);
513 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD);
514
515 ac->pending_count++;
516 }
517
518 static void
519 artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common)
520 {
521 struct artpec6_crypto_dma_descriptors *dma = common->dma;
522
523 dma->out_cnt = 0;
524 dma->in_cnt = 0;
525 dma->map_count = 0;
526 INIT_LIST_HEAD(&dma->bounce_buffers);
527 }
528
529 static bool fault_inject_dma_descr(void)
530 {
531 #ifdef CONFIG_FAULT_INJECTION
532 return should_fail(&artpec6_crypto_fail_dma_array_full, 1);
533 #else
534 return false;
535 #endif
536 }
537
538 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a
539 * physical address
540 *
541 * @addr: The physical address of the data buffer
542 * @len: The length of the data buffer
543 * @eop: True if this is the last buffer in the packet
544 *
545 * @return 0 on success or -ENOSPC if there are no more descriptors available
546 */
547 static int
548 artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common,
549 dma_addr_t addr, size_t len, bool eop)
550 {
551 struct artpec6_crypto_dma_descriptors *dma = common->dma;
552 struct pdma_descr *d;
553
554 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
555 fault_inject_dma_descr()) {
556 pr_err("No free OUT DMA descriptors available!\n");
557 return -ENOSPC;
558 }
559
560 d = &dma->out[dma->out_cnt++];
561 memset(d, 0, sizeof(*d));
562
563 d->ctrl.short_descr = 0;
564 d->ctrl.eop = eop;
565 d->data.len = len;
566 d->data.buf = addr;
567 return 0;
568 }
569
570 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor
571 *
572 * @dst: The virtual address of the data
573 * @len: The length of the data, must be between 1 to 7 bytes
574 * @eop: True if this is the last buffer in the packet
575 *
576 * @return 0 on success
577 * -ENOSPC if no more descriptors are available
578 * -EINVAL if the data length exceeds 7 bytes
579 */
580 static int
581 artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common,
582 void *dst, unsigned int len, bool eop)
583 {
584 struct artpec6_crypto_dma_descriptors *dma = common->dma;
585 struct pdma_descr *d;
586
587 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
588 fault_inject_dma_descr()) {
589 pr_err("No free OUT DMA descriptors available!\n");
590 return -ENOSPC;
591 } else if (len > 7 || len < 1) {
592 return -EINVAL;
593 }
594 d = &dma->out[dma->out_cnt++];
595 memset(d, 0, sizeof(*d));
596
597 d->ctrl.short_descr = 1;
598 d->ctrl.short_len = len;
599 d->ctrl.eop = eop;
600 memcpy(d->shrt.data, dst, len);
601 return 0;
602 }
603
604 static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common,
605 struct page *page, size_t offset,
606 size_t size,
607 enum dma_data_direction dir,
608 dma_addr_t *dma_addr_out)
609 {
610 struct artpec6_crypto_dma_descriptors *dma = common->dma;
611 struct device *dev = artpec6_crypto_dev;
612 struct artpec6_crypto_dma_map *map;
613 dma_addr_t dma_addr;
614
615 *dma_addr_out = 0;
616
617 if (dma->map_count >= ARRAY_SIZE(dma->maps))
618 return -ENOMEM;
619
620 dma_addr = dma_map_page(dev, page, offset, size, dir);
621 if (dma_mapping_error(dev, dma_addr))
622 return -ENOMEM;
623
624 map = &dma->maps[dma->map_count++];
625 map->size = size;
626 map->dma_addr = dma_addr;
627 map->dir = dir;
628
629 *dma_addr_out = dma_addr;
630
631 return 0;
632 }
633
634 static int
635 artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common,
636 void *ptr, size_t size,
637 enum dma_data_direction dir,
638 dma_addr_t *dma_addr_out)
639 {
640 struct page *page = virt_to_page(ptr);
641 size_t offset = (uintptr_t)ptr & ~PAGE_MASK;
642
643 return artpec6_crypto_dma_map_page(common, page, offset, size, dir,
644 dma_addr_out);
645 }
646
647 static int
648 artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common)
649 {
650 struct artpec6_crypto_dma_descriptors *dma = common->dma;
651 int ret;
652
653 ret = artpec6_crypto_dma_map_single(common, dma->in,
654 sizeof(dma->in[0]) * dma->in_cnt,
655 DMA_TO_DEVICE, &dma->in_dma_addr);
656 if (ret)
657 return ret;
658
659 ret = artpec6_crypto_dma_map_single(common, dma->out,
660 sizeof(dma->out[0]) * dma->out_cnt,
661 DMA_TO_DEVICE, &dma->out_dma_addr);
662 if (ret)
663 return ret;
664
665 /* We only read one stat descriptor */
666 dma->stat[dma->in_cnt - 1] = 0;
667
668 /*
669 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor
670 * to be written.
671 */
672 return artpec6_crypto_dma_map_single(common,
673 dma->stat + dma->in_cnt - 1,
674 sizeof(dma->stat[0]),
675 DMA_BIDIRECTIONAL,
676 &dma->stat_dma_addr);
677 }
678
679 static void
680 artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common)
681 {
682 struct artpec6_crypto_dma_descriptors *dma = common->dma;
683 struct device *dev = artpec6_crypto_dev;
684 int i;
685
686 for (i = 0; i < dma->map_count; i++) {
687 struct artpec6_crypto_dma_map *map = &dma->maps[i];
688
689 dma_unmap_page(dev, map->dma_addr, map->size, map->dir);
690 }
691
692 dma->map_count = 0;
693 }
694
695 /** artpec6_crypto_setup_out_descr - Setup an out descriptor
696 *
697 * @dst: The virtual address of the data
698 * @len: The length of the data
699 * @eop: True if this is the last buffer in the packet
700 * @use_short: If this is true and the data length is 7 bytes or less then
701 * a short descriptor will be used
702 *
703 * @return 0 on success
704 * Any errors from artpec6_crypto_setup_out_descr_short() or
705 * setup_out_descr_phys()
706 */
707 static int
708 artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common,
709 void *dst, unsigned int len, bool eop,
710 bool use_short)
711 {
712 if (use_short && len < 7) {
713 return artpec6_crypto_setup_out_descr_short(common, dst, len,
714 eop);
715 } else {
716 int ret;
717 dma_addr_t dma_addr;
718
719 ret = artpec6_crypto_dma_map_single(common, dst, len,
720 DMA_TO_DEVICE,
721 &dma_addr);
722 if (ret)
723 return ret;
724
725 return artpec6_crypto_setup_out_descr_phys(common, dma_addr,
726 len, eop);
727 }
728 }
729
730 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
731 * physical address
732 *
733 * @addr: The physical address of the data buffer
734 * @len: The length of the data buffer
735 * @intr: True if an interrupt should be fired after HW processing of this
736 * descriptor
737 *
738 */
739 static int
740 artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common,
741 dma_addr_t addr, unsigned int len, bool intr)
742 {
743 struct artpec6_crypto_dma_descriptors *dma = common->dma;
744 struct pdma_descr *d;
745
746 if (dma->in_cnt >= PDMA_DESCR_COUNT ||
747 fault_inject_dma_descr()) {
748 pr_err("No free IN DMA descriptors available!\n");
749 return -ENOSPC;
750 }
751 d = &dma->in[dma->in_cnt++];
752 memset(d, 0, sizeof(*d));
753
754 d->ctrl.intr = intr;
755 d->data.len = len;
756 d->data.buf = addr;
757 return 0;
758 }
759
760 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor
761 *
762 * @buffer: The virtual address to of the data buffer
763 * @len: The length of the data buffer
764 * @last: If this is the last data buffer in the request (i.e. an interrupt
765 * is needed
766 *
767 * Short descriptors are not used for the in channel
768 */
769 static int
770 artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common,
771 void *buffer, unsigned int len, bool last)
772 {
773 dma_addr_t dma_addr;
774 int ret;
775
776 ret = artpec6_crypto_dma_map_single(common, buffer, len,
777 DMA_FROM_DEVICE, &dma_addr);
778 if (ret)
779 return ret;
780
781 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last);
782 }
783
784 static struct artpec6_crypto_bounce_buffer *
785 artpec6_crypto_alloc_bounce(gfp_t flags)
786 {
787 void *base;
788 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) +
789 2 * ARTPEC_CACHE_LINE_MAX;
790 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags);
791
792 if (!bbuf)
793 return NULL;
794
795 base = bbuf + 1;
796 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX);
797 return bbuf;
798 }
799
800 static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common,
801 struct artpec6_crypto_walk *walk, size_t size)
802 {
803 struct artpec6_crypto_bounce_buffer *bbuf;
804 int ret;
805
806 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags);
807 if (!bbuf)
808 return -ENOMEM;
809
810 bbuf->length = size;
811 bbuf->sg = walk->sg;
812 bbuf->offset = walk->offset;
813
814 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false);
815 if (ret) {
816 kfree(bbuf);
817 return ret;
818 }
819
820 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset);
821 list_add_tail(&bbuf->list, &common->dma->bounce_buffers);
822 return 0;
823 }
824
825 static int
826 artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common,
827 struct artpec6_crypto_walk *walk,
828 size_t count)
829 {
830 size_t chunk;
831 int ret;
832 dma_addr_t addr;
833
834 while (walk->sg && count) {
835 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
836 addr = artpec6_crypto_walk_chunk_phys(walk);
837
838 /* When destination buffers are not aligned to the cache line
839 * size we need bounce buffers. The DMA-API requires that the
840 * entire line is owned by the DMA buffer and this holds also
841 * for the case when coherent DMA is used.
842 */
843 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) {
844 chunk = min_t(dma_addr_t, chunk,
845 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) -
846 addr);
847
848 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
849 ret = setup_bounce_buffer_in(common, walk, chunk);
850 } else if (chunk < ARTPEC_CACHE_LINE_MAX) {
851 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
852 ret = setup_bounce_buffer_in(common, walk, chunk);
853 } else {
854 dma_addr_t dma_addr;
855
856 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);
857
858 pr_debug("CHUNK %pad:%zu\n", &addr, chunk);
859
860 ret = artpec6_crypto_dma_map_page(common,
861 sg_page(walk->sg),
862 walk->sg->offset +
863 walk->offset,
864 chunk,
865 DMA_FROM_DEVICE,
866 &dma_addr);
867 if (ret)
868 return ret;
869
870 ret = artpec6_crypto_setup_in_descr_phys(common,
871 dma_addr,
872 chunk, false);
873 }
874
875 if (ret)
876 return ret;
877
878 count = count - chunk;
879 artpec6_crypto_walk_advance(walk, chunk);
880 }
881
882 if (count)
883 pr_err("EOL unexpected %zu bytes left\n", count);
884
885 return count ? -EINVAL : 0;
886 }
887
888 static int
889 artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common,
890 struct artpec6_crypto_walk *walk,
891 size_t count)
892 {
893 size_t chunk;
894 int ret;
895 dma_addr_t addr;
896
897 while (walk->sg && count) {
898 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
899 addr = artpec6_crypto_walk_chunk_phys(walk);
900
901 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk);
902
903 if (addr & 3) {
904 char buf[3];
905
906 chunk = min_t(size_t, chunk, (4-(addr&3)));
907
908 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,
909 walk->offset);
910
911 ret = artpec6_crypto_setup_out_descr_short(common, buf,
912 chunk,
913 false);
914 } else {
915 dma_addr_t dma_addr;
916
917 ret = artpec6_crypto_dma_map_page(common,
918 sg_page(walk->sg),
919 walk->sg->offset +
920 walk->offset,
921 chunk,
922 DMA_TO_DEVICE,
923 &dma_addr);
924 if (ret)
925 return ret;
926
927 ret = artpec6_crypto_setup_out_descr_phys(common,
928 dma_addr,
929 chunk, false);
930 }
931
932 if (ret)
933 return ret;
934
935 count = count - chunk;
936 artpec6_crypto_walk_advance(walk, chunk);
937 }
938
939 if (count)
940 pr_err("EOL unexpected %zu bytes left\n", count);
941
942 return count ? -EINVAL : 0;
943 }
944
945
946 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor
947 *
948 * If the out descriptor list is non-empty, then the eop flag on the
949 * last used out descriptor will be set.
950 *
951 * @return 0 on success
952 * -EINVAL if the out descriptor is empty or has overflown
953 */
954 static int
955 artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common)
956 {
957 struct artpec6_crypto_dma_descriptors *dma = common->dma;
958 struct pdma_descr *d;
959
960 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) {
961 pr_err("%s: OUT descriptor list is %s\n",
962 MODULE_NAME, dma->out_cnt ? "empty" : "full");
963 return -EINVAL;
964
965 }
966
967 d = &dma->out[dma->out_cnt-1];
968 d->ctrl.eop = 1;
969
970 return 0;
971 }
972
973 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last
974 * in descriptor
975 *
976 * See artpec6_crypto_terminate_out_descrs() for return values
977 */
978 static int
979 artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common)
980 {
981 struct artpec6_crypto_dma_descriptors *dma = common->dma;
982 struct pdma_descr *d;
983
984 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) {
985 pr_err("%s: IN descriptor list is %s\n",
986 MODULE_NAME, dma->in_cnt ? "empty" : "full");
987 return -EINVAL;
988 }
989
990 d = &dma->in[dma->in_cnt-1];
991 d->ctrl.intr = 1;
992 return 0;
993 }
994
995 /** create_hash_pad - Create a Secure Hash conformant pad
996 *
997 * @dst: The destination buffer to write the pad. Must be at least 64 bytes
998 * @dgstlen: The total length of the hash digest in bytes
999 * @bitcount: The total length of the digest in bits
1000 *
1001 * @return The total number of padding bytes written to @dst
1002 */
1003 static size_t
1004 create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount)
1005 {
1006 unsigned int mod, target, diff, pad_bytes, size_bytes;
1007 __be64 bits = __cpu_to_be64(bitcount);
1008
1009 switch (oper) {
1010 case regk_crypto_sha1:
1011 case regk_crypto_sha256:
1012 case regk_crypto_hmac_sha1:
1013 case regk_crypto_hmac_sha256:
1014 target = 448 / 8;
1015 mod = 512 / 8;
1016 size_bytes = 8;
1017 break;
1018 default:
1019 target = 896 / 8;
1020 mod = 1024 / 8;
1021 size_bytes = 16;
1022 break;
1023 }
1024
1025 target -= 1;
1026 diff = dgstlen & (mod - 1);
1027 pad_bytes = diff > target ? target + mod - diff : target - diff;
1028
1029 memset(dst + 1, 0, pad_bytes);
1030 dst[0] = 0x80;
1031
1032 if (size_bytes == 16) {
1033 memset(dst + 1 + pad_bytes, 0, 8);
1034 memcpy(dst + 1 + pad_bytes + 8, &bits, 8);
1035 } else {
1036 memcpy(dst + 1 + pad_bytes, &bits, 8);
1037 }
1038
1039 return pad_bytes + size_bytes + 1;
1040 }
1041
1042 static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common,
1043 struct crypto_async_request *parent,
1044 void (*complete)(struct crypto_async_request *req),
1045 struct scatterlist *dstsg, unsigned int nbytes)
1046 {
1047 gfp_t flags;
1048 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1049
1050 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1051 GFP_KERNEL : GFP_ATOMIC;
1052
1053 common->gfp_flags = flags;
1054 common->dma = kmem_cache_alloc(ac->dma_cache, flags);
1055 if (!common->dma)
1056 return -ENOMEM;
1057
1058 common->req = parent;
1059 common->complete = complete;
1060 return 0;
1061 }
1062
1063 static void
1064 artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma)
1065 {
1066 struct artpec6_crypto_bounce_buffer *b;
1067 struct artpec6_crypto_bounce_buffer *next;
1068
1069 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
1070 kfree(b);
1071 }
1072 }
1073
1074 static int
1075 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common)
1076 {
1077 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1078
1079 artpec6_crypto_dma_unmap_all(common);
1080 artpec6_crypto_bounce_destroy(common->dma);
1081 kmem_cache_free(ac->dma_cache, common->dma);
1082 common->dma = NULL;
1083 return 0;
1084 }
1085
1086 /*
1087 * Ciphering functions.
1088 */
1089 static int artpec6_crypto_encrypt(struct skcipher_request *req)
1090 {
1091 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1092 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1093 struct artpec6_crypto_request_context *req_ctx = NULL;
1094 void (*complete)(struct crypto_async_request *req);
1095 int ret;
1096
1097 req_ctx = skcipher_request_ctx(req);
1098
1099 switch (ctx->crypto_type) {
1100 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1101 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1102 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1103 req_ctx->decrypt = 0;
1104 break;
1105 default:
1106 break;
1107 }
1108
1109 switch (ctx->crypto_type) {
1110 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1111 complete = artpec6_crypto_complete_cbc_encrypt;
1112 break;
1113 default:
1114 complete = artpec6_crypto_complete_crypto;
1115 break;
1116 }
1117
1118 ret = artpec6_crypto_common_init(&req_ctx->common,
1119 &req->base,
1120 complete,
1121 req->dst, req->cryptlen);
1122 if (ret)
1123 return ret;
1124
1125 ret = artpec6_crypto_prepare_crypto(req);
1126 if (ret) {
1127 artpec6_crypto_common_destroy(&req_ctx->common);
1128 return ret;
1129 }
1130
1131 return artpec6_crypto_submit(&req_ctx->common);
1132 }
1133
1134 static int artpec6_crypto_decrypt(struct skcipher_request *req)
1135 {
1136 int ret;
1137 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1138 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1139 struct artpec6_crypto_request_context *req_ctx = NULL;
1140 void (*complete)(struct crypto_async_request *req);
1141
1142 req_ctx = skcipher_request_ctx(req);
1143
1144 switch (ctx->crypto_type) {
1145 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1146 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1147 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1148 req_ctx->decrypt = 1;
1149 break;
1150 default:
1151 break;
1152 }
1153
1154
1155 switch (ctx->crypto_type) {
1156 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1157 complete = artpec6_crypto_complete_cbc_decrypt;
1158 break;
1159 default:
1160 complete = artpec6_crypto_complete_crypto;
1161 break;
1162 }
1163
1164 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1165 complete,
1166 req->dst, req->cryptlen);
1167 if (ret)
1168 return ret;
1169
1170 ret = artpec6_crypto_prepare_crypto(req);
1171 if (ret) {
1172 artpec6_crypto_common_destroy(&req_ctx->common);
1173 return ret;
1174 }
1175
1176 return artpec6_crypto_submit(&req_ctx->common);
1177 }
1178
1179 static int
1180 artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt)
1181 {
1182 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1183 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1184 size_t iv_len = crypto_skcipher_ivsize(cipher);
1185 unsigned int counter = be32_to_cpup((__be32 *)
1186 (req->iv + iv_len - 4));
1187 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
1188 AES_BLOCK_SIZE;
1189
1190 /*
1191 * The hardware uses only the last 32-bits as the counter while the
1192 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
1193 * the whole IV is a counter. So fallback if the counter is going to
1194 * overlow.
1195 */
1196 if (counter + nblks < counter) {
1197 int ret;
1198
1199 pr_debug("counter %x will overflow (nblks %u), falling back\n",
1200 counter, counter + nblks);
1201
1202 ret = crypto_skcipher_setkey(ctx->fallback, ctx->aes_key,
1203 ctx->key_length);
1204 if (ret)
1205 return ret;
1206
1207 {
1208 SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
1209
1210 skcipher_request_set_tfm(subreq, ctx->fallback);
1211 skcipher_request_set_callback(subreq, req->base.flags,
1212 NULL, NULL);
1213 skcipher_request_set_crypt(subreq, req->src, req->dst,
1214 req->cryptlen, req->iv);
1215 ret = encrypt ? crypto_skcipher_encrypt(subreq)
1216 : crypto_skcipher_decrypt(subreq);
1217 skcipher_request_zero(subreq);
1218 }
1219 return ret;
1220 }
1221
1222 return encrypt ? artpec6_crypto_encrypt(req)
1223 : artpec6_crypto_decrypt(req);
1224 }
1225
1226 static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req)
1227 {
1228 return artpec6_crypto_ctr_crypt(req, true);
1229 }
1230
1231 static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req)
1232 {
1233 return artpec6_crypto_ctr_crypt(req, false);
1234 }
1235
1236 /*
1237 * AEAD functions
1238 */
1239 static int artpec6_crypto_aead_init(struct crypto_aead *tfm)
1240 {
1241 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm);
1242
1243 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
1244
1245 crypto_aead_set_reqsize(tfm,
1246 sizeof(struct artpec6_crypto_aead_req_ctx));
1247
1248 return 0;
1249 }
1250
1251 static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key,
1252 unsigned int len)
1253 {
1254 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base);
1255
1256 if (len != 16 && len != 24 && len != 32) {
1257 crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
1258 return -1;
1259 }
1260
1261 ctx->key_length = len;
1262
1263 memcpy(ctx->aes_key, key, len);
1264 return 0;
1265 }
1266
1267 static int artpec6_crypto_aead_encrypt(struct aead_request *req)
1268 {
1269 int ret;
1270 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1271
1272 req_ctx->decrypt = false;
1273 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1274 artpec6_crypto_complete_aead,
1275 NULL, 0);
1276 if (ret)
1277 return ret;
1278
1279 ret = artpec6_crypto_prepare_aead(req);
1280 if (ret) {
1281 artpec6_crypto_common_destroy(&req_ctx->common);
1282 return ret;
1283 }
1284
1285 return artpec6_crypto_submit(&req_ctx->common);
1286 }
1287
1288 static int artpec6_crypto_aead_decrypt(struct aead_request *req)
1289 {
1290 int ret;
1291 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1292
1293 req_ctx->decrypt = true;
1294 if (req->cryptlen < AES_BLOCK_SIZE)
1295 return -EINVAL;
1296
1297 ret = artpec6_crypto_common_init(&req_ctx->common,
1298 &req->base,
1299 artpec6_crypto_complete_aead,
1300 NULL, 0);
1301 if (ret)
1302 return ret;
1303
1304 ret = artpec6_crypto_prepare_aead(req);
1305 if (ret) {
1306 artpec6_crypto_common_destroy(&req_ctx->common);
1307 return ret;
1308 }
1309
1310 return artpec6_crypto_submit(&req_ctx->common);
1311 }
1312
1313 static int artpec6_crypto_prepare_hash(struct ahash_request *areq)
1314 {
1315 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1316 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq);
1317 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1318 size_t contextsize = digestsize == SHA384_DIGEST_SIZE ?
1319 SHA512_DIGEST_SIZE : digestsize;
1320 size_t blocksize = crypto_tfm_alg_blocksize(
1321 crypto_ahash_tfm(crypto_ahash_reqtfm(areq)));
1322 struct artpec6_crypto_req_common *common = &req_ctx->common;
1323 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1324 enum artpec6_crypto_variant variant = ac->variant;
1325 u32 sel_ctx;
1326 bool ext_ctx = false;
1327 bool run_hw = false;
1328 int error = 0;
1329
1330 artpec6_crypto_init_dma_operation(common);
1331
1332 /* Upload HMAC key, must be first the first packet */
1333 if (req_ctx->hash_flags & HASH_FLAG_HMAC) {
1334 if (variant == ARTPEC6_CRYPTO) {
1335 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1336 a6_regk_crypto_dlkey);
1337 } else {
1338 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1339 a7_regk_crypto_dlkey);
1340 }
1341
1342 /* Copy and pad up the key */
1343 memcpy(req_ctx->key_buffer, ctx->hmac_key,
1344 ctx->hmac_key_length);
1345 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0,
1346 blocksize - ctx->hmac_key_length);
1347
1348 error = artpec6_crypto_setup_out_descr(common,
1349 (void *)&req_ctx->key_md,
1350 sizeof(req_ctx->key_md), false, false);
1351 if (error)
1352 return error;
1353
1354 error = artpec6_crypto_setup_out_descr(common,
1355 req_ctx->key_buffer, blocksize,
1356 true, false);
1357 if (error)
1358 return error;
1359 }
1360
1361 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) {
1362 /* Restore context */
1363 sel_ctx = regk_crypto_ext;
1364 ext_ctx = true;
1365 } else {
1366 sel_ctx = regk_crypto_init;
1367 }
1368
1369 if (variant == ARTPEC6_CRYPTO) {
1370 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX;
1371 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx);
1372
1373 /* If this is the final round, set the final flag */
1374 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1375 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN;
1376 } else {
1377 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX;
1378 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx);
1379
1380 /* If this is the final round, set the final flag */
1381 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1382 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN;
1383 }
1384
1385 /* Setup up metadata descriptors */
1386 error = artpec6_crypto_setup_out_descr(common,
1387 (void *)&req_ctx->hash_md,
1388 sizeof(req_ctx->hash_md), false, false);
1389 if (error)
1390 return error;
1391
1392 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1393 if (error)
1394 return error;
1395
1396 if (ext_ctx) {
1397 error = artpec6_crypto_setup_out_descr(common,
1398 req_ctx->digeststate,
1399 contextsize, false, false);
1400
1401 if (error)
1402 return error;
1403 }
1404
1405 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) {
1406 size_t done_bytes = 0;
1407 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes;
1408 size_t ready_bytes = round_down(total_bytes, blocksize);
1409 struct artpec6_crypto_walk walk;
1410
1411 run_hw = ready_bytes > 0;
1412 if (req_ctx->partial_bytes && ready_bytes) {
1413 /* We have a partial buffer and will at least some bytes
1414 * to the HW. Empty this partial buffer before tackling
1415 * the SG lists
1416 */
1417 memcpy(req_ctx->partial_buffer_out,
1418 req_ctx->partial_buffer,
1419 req_ctx->partial_bytes);
1420
1421 error = artpec6_crypto_setup_out_descr(common,
1422 req_ctx->partial_buffer_out,
1423 req_ctx->partial_bytes,
1424 false, true);
1425 if (error)
1426 return error;
1427
1428 /* Reset partial buffer */
1429 done_bytes += req_ctx->partial_bytes;
1430 req_ctx->partial_bytes = 0;
1431 }
1432
1433 artpec6_crypto_walk_init(&walk, areq->src);
1434
1435 error = artpec6_crypto_setup_sg_descrs_out(common, &walk,
1436 ready_bytes -
1437 done_bytes);
1438 if (error)
1439 return error;
1440
1441 if (walk.sg) {
1442 size_t sg_skip = ready_bytes - done_bytes;
1443 size_t sg_rem = areq->nbytes - sg_skip;
1444
1445 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
1446 req_ctx->partial_buffer +
1447 req_ctx->partial_bytes,
1448 sg_rem, sg_skip);
1449
1450 req_ctx->partial_bytes += sg_rem;
1451 }
1452
1453 req_ctx->digcnt += ready_bytes;
1454 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE);
1455 }
1456
1457 /* Finalize */
1458 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) {
1459 bool needtrim = contextsize != digestsize;
1460 size_t hash_pad_len;
1461 u64 digest_bits;
1462 u32 oper;
1463
1464 if (variant == ARTPEC6_CRYPTO)
1465 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md);
1466 else
1467 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md);
1468
1469 /* Write out the partial buffer if present */
1470 if (req_ctx->partial_bytes) {
1471 memcpy(req_ctx->partial_buffer_out,
1472 req_ctx->partial_buffer,
1473 req_ctx->partial_bytes);
1474 error = artpec6_crypto_setup_out_descr(common,
1475 req_ctx->partial_buffer_out,
1476 req_ctx->partial_bytes,
1477 false, true);
1478 if (error)
1479 return error;
1480
1481 req_ctx->digcnt += req_ctx->partial_bytes;
1482 req_ctx->partial_bytes = 0;
1483 }
1484
1485 if (req_ctx->hash_flags & HASH_FLAG_HMAC)
1486 digest_bits = 8 * (req_ctx->digcnt + blocksize);
1487 else
1488 digest_bits = 8 * req_ctx->digcnt;
1489
1490 /* Add the hash pad */
1491 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer,
1492 req_ctx->digcnt, digest_bits);
1493 error = artpec6_crypto_setup_out_descr(common,
1494 req_ctx->pad_buffer,
1495 hash_pad_len, false,
1496 true);
1497 req_ctx->digcnt = 0;
1498
1499 if (error)
1500 return error;
1501
1502 /* Descriptor for the final result */
1503 error = artpec6_crypto_setup_in_descr(common, areq->result,
1504 digestsize,
1505 !needtrim);
1506 if (error)
1507 return error;
1508
1509 if (needtrim) {
1510 /* Discard the extra context bytes for SHA-384 */
1511 error = artpec6_crypto_setup_in_descr(common,
1512 req_ctx->partial_buffer,
1513 digestsize - contextsize, true);
1514 if (error)
1515 return error;
1516 }
1517
1518 } else { /* This is not the final operation for this request */
1519 if (!run_hw)
1520 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START;
1521
1522 /* Save the result to the context */
1523 error = artpec6_crypto_setup_in_descr(common,
1524 req_ctx->digeststate,
1525 contextsize, false);
1526 if (error)
1527 return error;
1528 /* fall through */
1529 }
1530
1531 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE |
1532 HASH_FLAG_FINALIZE);
1533
1534 error = artpec6_crypto_terminate_in_descrs(common);
1535 if (error)
1536 return error;
1537
1538 error = artpec6_crypto_terminate_out_descrs(common);
1539 if (error)
1540 return error;
1541
1542 error = artpec6_crypto_dma_map_descs(common);
1543 if (error)
1544 return error;
1545
1546 return ARTPEC6_CRYPTO_PREPARE_HASH_START;
1547 }
1548
1549
1550 static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm)
1551 {
1552 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1553
1554 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1555 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB;
1556
1557 return 0;
1558 }
1559
1560 static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm)
1561 {
1562 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1563
1564 ctx->fallback = crypto_alloc_skcipher(crypto_tfm_alg_name(&tfm->base),
1565 0,
1566 CRYPTO_ALG_ASYNC |
1567 CRYPTO_ALG_NEED_FALLBACK);
1568 if (IS_ERR(ctx->fallback))
1569 return PTR_ERR(ctx->fallback);
1570
1571 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1572 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR;
1573
1574 return 0;
1575 }
1576
1577 static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm)
1578 {
1579 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1580
1581 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1582 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC;
1583
1584 return 0;
1585 }
1586
1587 static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm)
1588 {
1589 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1590
1591 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1592 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS;
1593
1594 return 0;
1595 }
1596
1597 static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm)
1598 {
1599 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1600
1601 memset(ctx, 0, sizeof(*ctx));
1602 }
1603
1604 static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm)
1605 {
1606 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1607
1608 crypto_free_skcipher(ctx->fallback);
1609 artpec6_crypto_aes_exit(tfm);
1610 }
1611
1612 static int
1613 artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key,
1614 unsigned int keylen)
1615 {
1616 struct artpec6_cryptotfm_context *ctx =
1617 crypto_skcipher_ctx(cipher);
1618
1619 switch (keylen) {
1620 case 16:
1621 case 24:
1622 case 32:
1623 break;
1624 default:
1625 crypto_skcipher_set_flags(cipher,
1626 CRYPTO_TFM_RES_BAD_KEY_LEN);
1627 return -EINVAL;
1628 }
1629
1630 memcpy(ctx->aes_key, key, keylen);
1631 ctx->key_length = keylen;
1632 return 0;
1633 }
1634
1635 static int
1636 artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key,
1637 unsigned int keylen)
1638 {
1639 struct artpec6_cryptotfm_context *ctx =
1640 crypto_skcipher_ctx(cipher);
1641 int ret;
1642
1643 ret = xts_check_key(&cipher->base, key, keylen);
1644 if (ret)
1645 return ret;
1646
1647 switch (keylen) {
1648 case 32:
1649 case 48:
1650 case 64:
1651 break;
1652 default:
1653 crypto_skcipher_set_flags(cipher,
1654 CRYPTO_TFM_RES_BAD_KEY_LEN);
1655 return -EINVAL;
1656 }
1657
1658 memcpy(ctx->aes_key, key, keylen);
1659 ctx->key_length = keylen;
1660 return 0;
1661 }
1662
1663 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request
1664 *
1665 * @req: The asynch request to process
1666 *
1667 * @return 0 if the dma job was successfully prepared
1668 * <0 on error
1669 *
1670 * This function sets up the PDMA descriptors for a block cipher request.
1671 *
1672 * The required padding is added for AES-CTR using a statically defined
1673 * buffer.
1674 *
1675 * The PDMA descriptor list will be as follows:
1676 *
1677 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop>
1678 * IN: <CIPHER_MD><data_0>...[data_n]<intr>
1679 *
1680 */
1681 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq)
1682 {
1683 int ret;
1684 struct artpec6_crypto_walk walk;
1685 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1686 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1687 struct artpec6_crypto_request_context *req_ctx = NULL;
1688 size_t iv_len = crypto_skcipher_ivsize(cipher);
1689 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1690 enum artpec6_crypto_variant variant = ac->variant;
1691 struct artpec6_crypto_req_common *common;
1692 bool cipher_decr = false;
1693 size_t cipher_klen;
1694 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */
1695 u32 oper;
1696
1697 req_ctx = skcipher_request_ctx(areq);
1698 common = &req_ctx->common;
1699
1700 artpec6_crypto_init_dma_operation(common);
1701
1702 if (variant == ARTPEC6_CRYPTO)
1703 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey);
1704 else
1705 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey);
1706
1707 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1708 sizeof(ctx->key_md), false, false);
1709 if (ret)
1710 return ret;
1711
1712 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1713 ctx->key_length, true, false);
1714 if (ret)
1715 return ret;
1716
1717 req_ctx->cipher_md = 0;
1718
1719 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS)
1720 cipher_klen = ctx->key_length/2;
1721 else
1722 cipher_klen = ctx->key_length;
1723
1724 /* Metadata */
1725 switch (cipher_klen) {
1726 case 16:
1727 cipher_len = regk_crypto_key_128;
1728 break;
1729 case 24:
1730 cipher_len = regk_crypto_key_192;
1731 break;
1732 case 32:
1733 cipher_len = regk_crypto_key_256;
1734 break;
1735 default:
1736 pr_err("%s: Invalid key length %d!\n",
1737 MODULE_NAME, ctx->key_length);
1738 return -EINVAL;
1739 }
1740
1741 switch (ctx->crypto_type) {
1742 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1743 oper = regk_crypto_aes_ecb;
1744 cipher_decr = req_ctx->decrypt;
1745 break;
1746
1747 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1748 oper = regk_crypto_aes_cbc;
1749 cipher_decr = req_ctx->decrypt;
1750 break;
1751
1752 case ARTPEC6_CRYPTO_CIPHER_AES_CTR:
1753 oper = regk_crypto_aes_ctr;
1754 cipher_decr = false;
1755 break;
1756
1757 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1758 oper = regk_crypto_aes_xts;
1759 cipher_decr = req_ctx->decrypt;
1760
1761 if (variant == ARTPEC6_CRYPTO)
1762 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ;
1763 else
1764 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ;
1765 break;
1766
1767 default:
1768 pr_err("%s: Invalid cipher mode %d!\n",
1769 MODULE_NAME, ctx->crypto_type);
1770 return -EINVAL;
1771 }
1772
1773 if (variant == ARTPEC6_CRYPTO) {
1774 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper);
1775 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1776 cipher_len);
1777 if (cipher_decr)
1778 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1779 } else {
1780 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper);
1781 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1782 cipher_len);
1783 if (cipher_decr)
1784 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1785 }
1786
1787 ret = artpec6_crypto_setup_out_descr(common,
1788 &req_ctx->cipher_md,
1789 sizeof(req_ctx->cipher_md),
1790 false, false);
1791 if (ret)
1792 return ret;
1793
1794 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1795 if (ret)
1796 return ret;
1797
1798 if (iv_len) {
1799 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len,
1800 false, false);
1801 if (ret)
1802 return ret;
1803 }
1804 /* Data out */
1805 artpec6_crypto_walk_init(&walk, areq->src);
1806 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen);
1807 if (ret)
1808 return ret;
1809
1810 /* Data in */
1811 artpec6_crypto_walk_init(&walk, areq->dst);
1812 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen);
1813 if (ret)
1814 return ret;
1815
1816 /* CTR-mode padding required by the HW. */
1817 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR ||
1818 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) {
1819 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) -
1820 areq->cryptlen;
1821
1822 if (pad) {
1823 ret = artpec6_crypto_setup_out_descr(common,
1824 ac->pad_buffer,
1825 pad, false, false);
1826 if (ret)
1827 return ret;
1828
1829 ret = artpec6_crypto_setup_in_descr(common,
1830 ac->pad_buffer, pad,
1831 false);
1832 if (ret)
1833 return ret;
1834 }
1835 }
1836
1837 ret = artpec6_crypto_terminate_out_descrs(common);
1838 if (ret)
1839 return ret;
1840
1841 ret = artpec6_crypto_terminate_in_descrs(common);
1842 if (ret)
1843 return ret;
1844
1845 return artpec6_crypto_dma_map_descs(common);
1846 }
1847
1848 static int artpec6_crypto_prepare_aead(struct aead_request *areq)
1849 {
1850 size_t count;
1851 int ret;
1852 size_t input_length;
1853 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1854 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
1855 struct crypto_aead *cipher = crypto_aead_reqtfm(areq);
1856 struct artpec6_crypto_req_common *common = &req_ctx->common;
1857 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1858 enum artpec6_crypto_variant variant = ac->variant;
1859 u32 md_cipher_len;
1860
1861 artpec6_crypto_init_dma_operation(common);
1862
1863 /* Key */
1864 if (variant == ARTPEC6_CRYPTO) {
1865 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1866 a6_regk_crypto_dlkey);
1867 } else {
1868 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1869 a7_regk_crypto_dlkey);
1870 }
1871 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1872 sizeof(ctx->key_md), false, false);
1873 if (ret)
1874 return ret;
1875
1876 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1877 ctx->key_length, true, false);
1878 if (ret)
1879 return ret;
1880
1881 req_ctx->cipher_md = 0;
1882
1883 switch (ctx->key_length) {
1884 case 16:
1885 md_cipher_len = regk_crypto_key_128;
1886 break;
1887 case 24:
1888 md_cipher_len = regk_crypto_key_192;
1889 break;
1890 case 32:
1891 md_cipher_len = regk_crypto_key_256;
1892 break;
1893 default:
1894 return -EINVAL;
1895 }
1896
1897 if (variant == ARTPEC6_CRYPTO) {
1898 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER,
1899 regk_crypto_aes_gcm);
1900 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1901 md_cipher_len);
1902 if (req_ctx->decrypt)
1903 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1904 } else {
1905 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER,
1906 regk_crypto_aes_gcm);
1907 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1908 md_cipher_len);
1909 if (req_ctx->decrypt)
1910 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1911 }
1912
1913 ret = artpec6_crypto_setup_out_descr(common,
1914 (void *) &req_ctx->cipher_md,
1915 sizeof(req_ctx->cipher_md), false,
1916 false);
1917 if (ret)
1918 return ret;
1919
1920 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1921 if (ret)
1922 return ret;
1923
1924 /* For the decryption, cryptlen includes the tag. */
1925 input_length = areq->cryptlen;
1926 if (req_ctx->decrypt)
1927 input_length -= AES_BLOCK_SIZE;
1928
1929 /* Prepare the context buffer */
1930 req_ctx->hw_ctx.aad_length_bits =
1931 __cpu_to_be64(8*areq->assoclen);
1932
1933 req_ctx->hw_ctx.text_length_bits =
1934 __cpu_to_be64(8*input_length);
1935
1936 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher));
1937 // The HW omits the initial increment of the counter field.
1938 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4);
1939
1940 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx,
1941 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false);
1942 if (ret)
1943 return ret;
1944
1945 {
1946 struct artpec6_crypto_walk walk;
1947
1948 artpec6_crypto_walk_init(&walk, areq->src);
1949
1950 /* Associated data */
1951 count = areq->assoclen;
1952 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1953 if (ret)
1954 return ret;
1955
1956 if (!IS_ALIGNED(areq->assoclen, 16)) {
1957 size_t assoc_pad = 16 - (areq->assoclen % 16);
1958 /* The HW mandates zero padding here */
1959 ret = artpec6_crypto_setup_out_descr(common,
1960 ac->zero_buffer,
1961 assoc_pad, false,
1962 false);
1963 if (ret)
1964 return ret;
1965 }
1966
1967 /* Data to crypto */
1968 count = input_length;
1969 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1970 if (ret)
1971 return ret;
1972
1973 if (!IS_ALIGNED(input_length, 16)) {
1974 size_t crypto_pad = 16 - (input_length % 16);
1975 /* The HW mandates zero padding here */
1976 ret = artpec6_crypto_setup_out_descr(common,
1977 ac->zero_buffer,
1978 crypto_pad,
1979 false,
1980 false);
1981 if (ret)
1982 return ret;
1983 }
1984 }
1985
1986 /* Data from crypto */
1987 {
1988 struct artpec6_crypto_walk walk;
1989 size_t output_len = areq->cryptlen;
1990
1991 if (req_ctx->decrypt)
1992 output_len -= AES_BLOCK_SIZE;
1993
1994 artpec6_crypto_walk_init(&walk, areq->dst);
1995
1996 /* skip associated data in the output */
1997 count = artpec6_crypto_walk_advance(&walk, areq->assoclen);
1998 if (count)
1999 return -EINVAL;
2000
2001 count = output_len;
2002 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count);
2003 if (ret)
2004 return ret;
2005
2006 /* Put padding between the cryptotext and the auth tag */
2007 if (!IS_ALIGNED(output_len, 16)) {
2008 size_t crypto_pad = 16 - (output_len % 16);
2009
2010 ret = artpec6_crypto_setup_in_descr(common,
2011 ac->pad_buffer,
2012 crypto_pad, false);
2013 if (ret)
2014 return ret;
2015 }
2016
2017 /* The authentication tag shall follow immediately after
2018 * the output ciphertext. For decryption it is put in a context
2019 * buffer for later compare against the input tag.
2020 */
2021 count = AES_BLOCK_SIZE;
2022
2023 if (req_ctx->decrypt) {
2024 ret = artpec6_crypto_setup_in_descr(common,
2025 req_ctx->decryption_tag, count, false);
2026 if (ret)
2027 return ret;
2028
2029 } else {
2030 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk,
2031 count);
2032 if (ret)
2033 return ret;
2034 }
2035
2036 }
2037
2038 ret = artpec6_crypto_terminate_in_descrs(common);
2039 if (ret)
2040 return ret;
2041
2042 ret = artpec6_crypto_terminate_out_descrs(common);
2043 if (ret)
2044 return ret;
2045
2046 return artpec6_crypto_dma_map_descs(common);
2047 }
2048
2049 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac)
2050 {
2051 struct artpec6_crypto_req_common *req;
2052
2053 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {
2054 req = list_first_entry(&ac->queue,
2055 struct artpec6_crypto_req_common,
2056 list);
2057 list_move_tail(&req->list, &ac->pending);
2058 artpec6_crypto_start_dma(req);
2059
2060 req->req->complete(req->req, -EINPROGRESS);
2061 }
2062
2063 /*
2064 * In some cases, the hardware can raise an in_eop_flush interrupt
2065 * before actually updating the status, so we have an timer which will
2066 * recheck the status on timeout. Since the cases are expected to be
2067 * very rare, we use a relatively large timeout value. There should be
2068 * no noticeable negative effect if we timeout spuriously.
2069 */
2070 if (ac->pending_count)
2071 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));
2072 else
2073 del_timer(&ac->timer);
2074 }
2075
2076 static void artpec6_crypto_timeout(struct timer_list *t)
2077 {
2078 struct artpec6_crypto *ac = from_timer(ac, t, timer);
2079
2080 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n");
2081
2082 tasklet_schedule(&ac->task);
2083 }
2084
2085 static void artpec6_crypto_task(unsigned long data)
2086 {
2087 struct artpec6_crypto *ac = (struct artpec6_crypto *)data;
2088 struct artpec6_crypto_req_common *req;
2089 struct artpec6_crypto_req_common *n;
2090
2091 if (list_empty(&ac->pending)) {
2092 pr_debug("Spurious IRQ\n");
2093 return;
2094 }
2095
2096 spin_lock_bh(&ac->queue_lock);
2097
2098 list_for_each_entry_safe(req, n, &ac->pending, list) {
2099 struct artpec6_crypto_dma_descriptors *dma = req->dma;
2100 u32 stat;
2101
2102 dma_sync_single_for_cpu(artpec6_crypto_dev, dma->stat_dma_addr,
2103 sizeof(dma->stat[0]),
2104 DMA_BIDIRECTIONAL);
2105
2106 stat = req->dma->stat[req->dma->in_cnt-1];
2107
2108 /* A non-zero final status descriptor indicates
2109 * this job has finished.
2110 */
2111 pr_debug("Request %p status is %X\n", req, stat);
2112 if (!stat)
2113 break;
2114
2115 /* Allow testing of timeout handling with fault injection */
2116 #ifdef CONFIG_FAULT_INJECTION
2117 if (should_fail(&artpec6_crypto_fail_status_read, 1))
2118 continue;
2119 #endif
2120
2121 pr_debug("Completing request %p\n", req);
2122
2123 list_del(&req->list);
2124
2125 artpec6_crypto_dma_unmap_all(req);
2126 artpec6_crypto_copy_bounce_buffers(req);
2127
2128 ac->pending_count--;
2129 artpec6_crypto_common_destroy(req);
2130 req->complete(req->req);
2131 }
2132
2133 artpec6_crypto_process_queue(ac);
2134
2135 spin_unlock_bh(&ac->queue_lock);
2136 }
2137
2138 static void artpec6_crypto_complete_crypto(struct crypto_async_request *req)
2139 {
2140 req->complete(req, 0);
2141 }
2142
2143 static void
2144 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req)
2145 {
2146 struct skcipher_request *cipher_req = container_of(req,
2147 struct skcipher_request, base);
2148
2149 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src,
2150 cipher_req->cryptlen - AES_BLOCK_SIZE,
2151 AES_BLOCK_SIZE, 0);
2152 req->complete(req, 0);
2153 }
2154
2155 static void
2156 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req)
2157 {
2158 struct skcipher_request *cipher_req = container_of(req,
2159 struct skcipher_request, base);
2160
2161 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst,
2162 cipher_req->cryptlen - AES_BLOCK_SIZE,
2163 AES_BLOCK_SIZE, 0);
2164 req->complete(req, 0);
2165 }
2166
2167 static void artpec6_crypto_complete_aead(struct crypto_async_request *req)
2168 {
2169 int result = 0;
2170
2171 /* Verify GCM hashtag. */
2172 struct aead_request *areq = container_of(req,
2173 struct aead_request, base);
2174 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
2175
2176 if (req_ctx->decrypt) {
2177 u8 input_tag[AES_BLOCK_SIZE];
2178
2179 sg_pcopy_to_buffer(areq->src,
2180 sg_nents(areq->src),
2181 input_tag,
2182 AES_BLOCK_SIZE,
2183 areq->assoclen + areq->cryptlen -
2184 AES_BLOCK_SIZE);
2185
2186 if (memcmp(req_ctx->decryption_tag,
2187 input_tag,
2188 AES_BLOCK_SIZE)) {
2189 pr_debug("***EBADMSG:\n");
2190 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,
2191 input_tag, AES_BLOCK_SIZE, true);
2192 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,
2193 req_ctx->decryption_tag,
2194 AES_BLOCK_SIZE, true);
2195
2196 result = -EBADMSG;
2197 }
2198 }
2199
2200 req->complete(req, result);
2201 }
2202
2203 static void artpec6_crypto_complete_hash(struct crypto_async_request *req)
2204 {
2205 req->complete(req, 0);
2206 }
2207
2208
2209 /*------------------- Hash functions -----------------------------------------*/
2210 static int
2211 artpec6_crypto_hash_set_key(struct crypto_ahash *tfm,
2212 const u8 *key, unsigned int keylen)
2213 {
2214 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base);
2215 size_t blocksize;
2216 int ret;
2217
2218 if (!keylen) {
2219 pr_err("Invalid length (%d) of HMAC key\n",
2220 keylen);
2221 return -EINVAL;
2222 }
2223
2224 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2225
2226 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2227
2228 if (keylen > blocksize) {
2229 SHASH_DESC_ON_STACK(hdesc, tfm_ctx->child_hash);
2230
2231 hdesc->tfm = tfm_ctx->child_hash;
2232 hdesc->flags = crypto_ahash_get_flags(tfm) &
2233 CRYPTO_TFM_REQ_MAY_SLEEP;
2234
2235 tfm_ctx->hmac_key_length = blocksize;
2236 ret = crypto_shash_digest(hdesc, key, keylen,
2237 tfm_ctx->hmac_key);
2238 if (ret)
2239 return ret;
2240
2241 } else {
2242 memcpy(tfm_ctx->hmac_key, key, keylen);
2243 tfm_ctx->hmac_key_length = keylen;
2244 }
2245
2246 return 0;
2247 }
2248
2249 static int
2250 artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac)
2251 {
2252 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2253 enum artpec6_crypto_variant variant = ac->variant;
2254 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2255 u32 oper;
2256
2257 memset(req_ctx, 0, sizeof(*req_ctx));
2258
2259 req_ctx->hash_flags = HASH_FLAG_INIT_CTX;
2260 if (hmac)
2261 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY);
2262
2263 switch (type) {
2264 case ARTPEC6_CRYPTO_HASH_SHA1:
2265 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1;
2266 break;
2267 case ARTPEC6_CRYPTO_HASH_SHA256:
2268 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256;
2269 break;
2270 case ARTPEC6_CRYPTO_HASH_SHA384:
2271 oper = hmac ? regk_crypto_hmac_sha384 : regk_crypto_sha384;
2272 break;
2273 case ARTPEC6_CRYPTO_HASH_SHA512:
2274 oper = hmac ? regk_crypto_hmac_sha512 : regk_crypto_sha512;
2275 break;
2276
2277 default:
2278 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type);
2279 return -EINVAL;
2280 }
2281
2282 if (variant == ARTPEC6_CRYPTO)
2283 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper);
2284 else
2285 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper);
2286
2287 return 0;
2288 }
2289
2290 static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req)
2291 {
2292 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2293 int ret;
2294
2295 if (!req_ctx->common.dma) {
2296 ret = artpec6_crypto_common_init(&req_ctx->common,
2297 &req->base,
2298 artpec6_crypto_complete_hash,
2299 NULL, 0);
2300
2301 if (ret)
2302 return ret;
2303 }
2304
2305 ret = artpec6_crypto_prepare_hash(req);
2306 switch (ret) {
2307 case ARTPEC6_CRYPTO_PREPARE_HASH_START:
2308 ret = artpec6_crypto_submit(&req_ctx->common);
2309 break;
2310
2311 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START:
2312 ret = 0;
2313 /* Fallthrough */
2314
2315 default:
2316 artpec6_crypto_common_destroy(&req_ctx->common);
2317 break;
2318 }
2319
2320 return ret;
2321 }
2322
2323 static int artpec6_crypto_hash_final(struct ahash_request *req)
2324 {
2325 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2326
2327 req_ctx->hash_flags |= HASH_FLAG_FINALIZE;
2328
2329 return artpec6_crypto_prepare_submit_hash(req);
2330 }
2331
2332 static int artpec6_crypto_hash_update(struct ahash_request *req)
2333 {
2334 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2335
2336 req_ctx->hash_flags |= HASH_FLAG_UPDATE;
2337
2338 return artpec6_crypto_prepare_submit_hash(req);
2339 }
2340
2341 static int artpec6_crypto_sha1_init(struct ahash_request *req)
2342 {
2343 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2344 }
2345
2346 static int artpec6_crypto_sha1_digest(struct ahash_request *req)
2347 {
2348 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2349
2350 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2351
2352 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2353
2354 return artpec6_crypto_prepare_submit_hash(req);
2355 }
2356
2357 static int artpec6_crypto_sha256_init(struct ahash_request *req)
2358 {
2359 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2360 }
2361
2362 static int artpec6_crypto_sha256_digest(struct ahash_request *req)
2363 {
2364 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2365
2366 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2367 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2368
2369 return artpec6_crypto_prepare_submit_hash(req);
2370 }
2371
2372 static int __maybe_unused artpec6_crypto_sha384_init(struct ahash_request *req)
2373 {
2374 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0);
2375 }
2376
2377 static int __maybe_unused
2378 artpec6_crypto_sha384_digest(struct ahash_request *req)
2379 {
2380 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2381
2382 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 0);
2383 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2384
2385 return artpec6_crypto_prepare_submit_hash(req);
2386 }
2387
2388 static int artpec6_crypto_sha512_init(struct ahash_request *req)
2389 {
2390 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0);
2391 }
2392
2393 static int artpec6_crypto_sha512_digest(struct ahash_request *req)
2394 {
2395 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2396
2397 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 0);
2398 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2399
2400 return artpec6_crypto_prepare_submit_hash(req);
2401 }
2402
2403 static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req)
2404 {
2405 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2406 }
2407
2408 static int __maybe_unused
2409 artpec6_crypto_hmac_sha384_init(struct ahash_request *req)
2410 {
2411 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1);
2412 }
2413
2414 static int artpec6_crypto_hmac_sha512_init(struct ahash_request *req)
2415 {
2416 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1);
2417 }
2418
2419 static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req)
2420 {
2421 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2422
2423 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2424 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2425
2426 return artpec6_crypto_prepare_submit_hash(req);
2427 }
2428
2429 static int __maybe_unused
2430 artpec6_crypto_hmac_sha384_digest(struct ahash_request *req)
2431 {
2432 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2433
2434 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA384, 1);
2435 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2436
2437 return artpec6_crypto_prepare_submit_hash(req);
2438 }
2439
2440 static int artpec6_crypto_hmac_sha512_digest(struct ahash_request *req)
2441 {
2442 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2443
2444 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA512, 1);
2445 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2446
2447 return artpec6_crypto_prepare_submit_hash(req);
2448 }
2449
2450 static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm,
2451 const char *base_hash_name)
2452 {
2453 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2454
2455 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2456 sizeof(struct artpec6_hash_request_context));
2457 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
2458
2459 if (base_hash_name) {
2460 struct crypto_shash *child;
2461
2462 child = crypto_alloc_shash(base_hash_name, 0,
2463 CRYPTO_ALG_NEED_FALLBACK);
2464
2465 if (IS_ERR(child))
2466 return PTR_ERR(child);
2467
2468 tfm_ctx->child_hash = child;
2469 }
2470
2471 return 0;
2472 }
2473
2474 static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm)
2475 {
2476 return artpec6_crypto_ahash_init_common(tfm, NULL);
2477 }
2478
2479 static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm)
2480 {
2481 return artpec6_crypto_ahash_init_common(tfm, "sha256");
2482 }
2483
2484 static int __maybe_unused
2485 artpec6_crypto_ahash_init_hmac_sha384(struct crypto_tfm *tfm)
2486 {
2487 return artpec6_crypto_ahash_init_common(tfm, "sha384");
2488 }
2489
2490 static int artpec6_crypto_ahash_init_hmac_sha512(struct crypto_tfm *tfm)
2491 {
2492 return artpec6_crypto_ahash_init_common(tfm, "sha512");
2493 }
2494
2495 static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm)
2496 {
2497 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2498
2499 if (tfm_ctx->child_hash)
2500 crypto_free_shash(tfm_ctx->child_hash);
2501
2502 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2503 tfm_ctx->hmac_key_length = 0;
2504 }
2505
2506 static int artpec6_crypto_hash_export(struct ahash_request *req, void *out)
2507 {
2508 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2509 struct artpec6_hash_export_state *state = out;
2510 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2511 enum artpec6_crypto_variant variant = ac->variant;
2512
2513 BUILD_BUG_ON(sizeof(state->partial_buffer) !=
2514 sizeof(ctx->partial_buffer));
2515 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate));
2516
2517 state->digcnt = ctx->digcnt;
2518 state->partial_bytes = ctx->partial_bytes;
2519 state->hash_flags = ctx->hash_flags;
2520
2521 if (variant == ARTPEC6_CRYPTO)
2522 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md);
2523 else
2524 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md);
2525
2526 memcpy(state->partial_buffer, ctx->partial_buffer,
2527 sizeof(state->partial_buffer));
2528 memcpy(state->digeststate, ctx->digeststate,
2529 sizeof(state->digeststate));
2530
2531 return 0;
2532 }
2533
2534 static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in)
2535 {
2536 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2537 const struct artpec6_hash_export_state *state = in;
2538 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2539 enum artpec6_crypto_variant variant = ac->variant;
2540
2541 memset(ctx, 0, sizeof(*ctx));
2542
2543 ctx->digcnt = state->digcnt;
2544 ctx->partial_bytes = state->partial_bytes;
2545 ctx->hash_flags = state->hash_flags;
2546
2547 if (variant == ARTPEC6_CRYPTO)
2548 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper);
2549 else
2550 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper);
2551
2552 memcpy(ctx->partial_buffer, state->partial_buffer,
2553 sizeof(state->partial_buffer));
2554 memcpy(ctx->digeststate, state->digeststate,
2555 sizeof(state->digeststate));
2556
2557 return 0;
2558 }
2559
2560 static int init_crypto_hw(struct artpec6_crypto *ac)
2561 {
2562 enum artpec6_crypto_variant variant = ac->variant;
2563 void __iomem *base = ac->base;
2564 u32 out_descr_buf_size;
2565 u32 out_data_buf_size;
2566 u32 in_data_buf_size;
2567 u32 in_descr_buf_size;
2568 u32 in_stat_buf_size;
2569 u32 in, out;
2570
2571 /*
2572 * The PDMA unit contains 1984 bytes of internal memory for the OUT
2573 * channels and 1024 bytes for the IN channel. This is an elastic
2574 * memory used to internally store the descriptors and data. The values
2575 * ares specified in 64 byte incremements. Trustzone buffers are not
2576 * used at this stage.
2577 */
2578 out_data_buf_size = 16; /* 1024 bytes for data */
2579 out_descr_buf_size = 15; /* 960 bytes for descriptors */
2580 in_data_buf_size = 8; /* 512 bytes for data */
2581 in_descr_buf_size = 4; /* 256 bytes for descriptors */
2582 in_stat_buf_size = 4; /* 256 bytes for stat descrs */
2583
2584 BUILD_BUG_ON_MSG((out_data_buf_size
2585 + out_descr_buf_size) * 64 > 1984,
2586 "Invalid OUT configuration");
2587
2588 BUILD_BUG_ON_MSG((in_data_buf_size
2589 + in_descr_buf_size
2590 + in_stat_buf_size) * 64 > 1024,
2591 "Invalid IN configuration");
2592
2593 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) |
2594 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) |
2595 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size);
2596
2597 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) |
2598 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size);
2599
2600 writel_relaxed(out, base + PDMA_OUT_BUF_CFG);
2601 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG);
2602
2603 if (variant == ARTPEC6_CRYPTO) {
2604 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG);
2605 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG);
2606 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA |
2607 A6_PDMA_INTR_MASK_IN_EOP_FLUSH,
2608 base + A6_PDMA_INTR_MASK);
2609 } else {
2610 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG);
2611 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG);
2612 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA |
2613 A7_PDMA_INTR_MASK_IN_EOP_FLUSH,
2614 base + A7_PDMA_INTR_MASK);
2615 }
2616
2617 return 0;
2618 }
2619
2620 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)
2621 {
2622 enum artpec6_crypto_variant variant = ac->variant;
2623 void __iomem *base = ac->base;
2624
2625 if (variant == ARTPEC6_CRYPTO) {
2626 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD);
2627 writel_relaxed(0, base + A6_PDMA_IN_CFG);
2628 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2629 } else {
2630 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD);
2631 writel_relaxed(0, base + A7_PDMA_IN_CFG);
2632 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2633 }
2634
2635 writel_relaxed(0, base + PDMA_OUT_CFG);
2636
2637 }
2638
2639 static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id)
2640 {
2641 struct artpec6_crypto *ac = dev_id;
2642 enum artpec6_crypto_variant variant = ac->variant;
2643 void __iomem *base = ac->base;
2644 u32 mask_in_data, mask_in_eop_flush;
2645 u32 in_cmd_flush_stat, in_cmd_reg;
2646 u32 ack_intr_reg;
2647 u32 ack = 0;
2648 u32 intr;
2649
2650 if (variant == ARTPEC6_CRYPTO) {
2651 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR);
2652 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA;
2653 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH;
2654 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT;
2655 in_cmd_reg = A6_PDMA_IN_CMD;
2656 ack_intr_reg = A6_PDMA_ACK_INTR;
2657 } else {
2658 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR);
2659 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA;
2660 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH;
2661 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT;
2662 in_cmd_reg = A7_PDMA_IN_CMD;
2663 ack_intr_reg = A7_PDMA_ACK_INTR;
2664 }
2665
2666 /* We get two interrupt notifications from each job.
2667 * The in_data means all data was sent to memory and then
2668 * we request a status flush command to write the per-job
2669 * status to its status vector. This ensures that the
2670 * tasklet can detect exactly how many submitted jobs
2671 * that have finished.
2672 */
2673 if (intr & mask_in_data)
2674 ack |= mask_in_data;
2675
2676 if (intr & mask_in_eop_flush)
2677 ack |= mask_in_eop_flush;
2678 else
2679 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg);
2680
2681 writel_relaxed(ack, base + ack_intr_reg);
2682
2683 if (intr & mask_in_eop_flush)
2684 tasklet_schedule(&ac->task);
2685
2686 return IRQ_HANDLED;
2687 }
2688
2689 /*------------------- Algorithm definitions ----------------------------------*/
2690
2691 /* Hashes */
2692 static struct ahash_alg hash_algos[] = {
2693 /* SHA-1 */
2694 {
2695 .init = artpec6_crypto_sha1_init,
2696 .update = artpec6_crypto_hash_update,
2697 .final = artpec6_crypto_hash_final,
2698 .digest = artpec6_crypto_sha1_digest,
2699 .import = artpec6_crypto_hash_import,
2700 .export = artpec6_crypto_hash_export,
2701 .halg.digestsize = SHA1_DIGEST_SIZE,
2702 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2703 .halg.base = {
2704 .cra_name = "sha1",
2705 .cra_driver_name = "artpec-sha1",
2706 .cra_priority = 300,
2707 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2708 .cra_blocksize = SHA1_BLOCK_SIZE,
2709 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2710 .cra_alignmask = 3,
2711 .cra_module = THIS_MODULE,
2712 .cra_init = artpec6_crypto_ahash_init,
2713 .cra_exit = artpec6_crypto_ahash_exit,
2714 }
2715 },
2716 /* SHA-256 */
2717 {
2718 .init = artpec6_crypto_sha256_init,
2719 .update = artpec6_crypto_hash_update,
2720 .final = artpec6_crypto_hash_final,
2721 .digest = artpec6_crypto_sha256_digest,
2722 .import = artpec6_crypto_hash_import,
2723 .export = artpec6_crypto_hash_export,
2724 .halg.digestsize = SHA256_DIGEST_SIZE,
2725 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2726 .halg.base = {
2727 .cra_name = "sha256",
2728 .cra_driver_name = "artpec-sha256",
2729 .cra_priority = 300,
2730 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2731 .cra_blocksize = SHA256_BLOCK_SIZE,
2732 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2733 .cra_alignmask = 3,
2734 .cra_module = THIS_MODULE,
2735 .cra_init = artpec6_crypto_ahash_init,
2736 .cra_exit = artpec6_crypto_ahash_exit,
2737 }
2738 },
2739 /* HMAC SHA-256 */
2740 {
2741 .init = artpec6_crypto_hmac_sha256_init,
2742 .update = artpec6_crypto_hash_update,
2743 .final = artpec6_crypto_hash_final,
2744 .digest = artpec6_crypto_hmac_sha256_digest,
2745 .import = artpec6_crypto_hash_import,
2746 .export = artpec6_crypto_hash_export,
2747 .setkey = artpec6_crypto_hash_set_key,
2748 .halg.digestsize = SHA256_DIGEST_SIZE,
2749 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2750 .halg.base = {
2751 .cra_name = "hmac(sha256)",
2752 .cra_driver_name = "artpec-hmac-sha256",
2753 .cra_priority = 300,
2754 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2755 .cra_blocksize = SHA256_BLOCK_SIZE,
2756 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2757 .cra_alignmask = 3,
2758 .cra_module = THIS_MODULE,
2759 .cra_init = artpec6_crypto_ahash_init_hmac_sha256,
2760 .cra_exit = artpec6_crypto_ahash_exit,
2761 }
2762 },
2763 };
2764
2765 static struct ahash_alg artpec7_hash_algos[] = {
2766 /* SHA-384 */
2767 {
2768 .init = artpec6_crypto_sha384_init,
2769 .update = artpec6_crypto_hash_update,
2770 .final = artpec6_crypto_hash_final,
2771 .digest = artpec6_crypto_sha384_digest,
2772 .import = artpec6_crypto_hash_import,
2773 .export = artpec6_crypto_hash_export,
2774 .halg.digestsize = SHA384_DIGEST_SIZE,
2775 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2776 .halg.base = {
2777 .cra_name = "sha384",
2778 .cra_driver_name = "artpec-sha384",
2779 .cra_priority = 300,
2780 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2781 .cra_blocksize = SHA384_BLOCK_SIZE,
2782 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2783 .cra_alignmask = 3,
2784 .cra_module = THIS_MODULE,
2785 .cra_init = artpec6_crypto_ahash_init,
2786 .cra_exit = artpec6_crypto_ahash_exit,
2787 }
2788 },
2789 /* HMAC SHA-384 */
2790 {
2791 .init = artpec6_crypto_hmac_sha384_init,
2792 .update = artpec6_crypto_hash_update,
2793 .final = artpec6_crypto_hash_final,
2794 .digest = artpec6_crypto_hmac_sha384_digest,
2795 .import = artpec6_crypto_hash_import,
2796 .export = artpec6_crypto_hash_export,
2797 .setkey = artpec6_crypto_hash_set_key,
2798 .halg.digestsize = SHA384_DIGEST_SIZE,
2799 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2800 .halg.base = {
2801 .cra_name = "hmac(sha384)",
2802 .cra_driver_name = "artpec-hmac-sha384",
2803 .cra_priority = 300,
2804 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2805 .cra_blocksize = SHA384_BLOCK_SIZE,
2806 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2807 .cra_alignmask = 3,
2808 .cra_module = THIS_MODULE,
2809 .cra_init = artpec6_crypto_ahash_init_hmac_sha384,
2810 .cra_exit = artpec6_crypto_ahash_exit,
2811 }
2812 },
2813 /* SHA-512 */
2814 {
2815 .init = artpec6_crypto_sha512_init,
2816 .update = artpec6_crypto_hash_update,
2817 .final = artpec6_crypto_hash_final,
2818 .digest = artpec6_crypto_sha512_digest,
2819 .import = artpec6_crypto_hash_import,
2820 .export = artpec6_crypto_hash_export,
2821 .halg.digestsize = SHA512_DIGEST_SIZE,
2822 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2823 .halg.base = {
2824 .cra_name = "sha512",
2825 .cra_driver_name = "artpec-sha512",
2826 .cra_priority = 300,
2827 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2828 .cra_blocksize = SHA512_BLOCK_SIZE,
2829 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2830 .cra_alignmask = 3,
2831 .cra_module = THIS_MODULE,
2832 .cra_init = artpec6_crypto_ahash_init,
2833 .cra_exit = artpec6_crypto_ahash_exit,
2834 }
2835 },
2836 /* HMAC SHA-512 */
2837 {
2838 .init = artpec6_crypto_hmac_sha512_init,
2839 .update = artpec6_crypto_hash_update,
2840 .final = artpec6_crypto_hash_final,
2841 .digest = artpec6_crypto_hmac_sha512_digest,
2842 .import = artpec6_crypto_hash_import,
2843 .export = artpec6_crypto_hash_export,
2844 .setkey = artpec6_crypto_hash_set_key,
2845 .halg.digestsize = SHA512_DIGEST_SIZE,
2846 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2847 .halg.base = {
2848 .cra_name = "hmac(sha512)",
2849 .cra_driver_name = "artpec-hmac-sha512",
2850 .cra_priority = 300,
2851 .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC,
2852 .cra_blocksize = SHA512_BLOCK_SIZE,
2853 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2854 .cra_alignmask = 3,
2855 .cra_module = THIS_MODULE,
2856 .cra_init = artpec6_crypto_ahash_init_hmac_sha512,
2857 .cra_exit = artpec6_crypto_ahash_exit,
2858 }
2859 },
2860 };
2861
2862 /* Crypto */
2863 static struct skcipher_alg crypto_algos[] = {
2864 /* AES - ECB */
2865 {
2866 .base = {
2867 .cra_name = "ecb(aes)",
2868 .cra_driver_name = "artpec6-ecb-aes",
2869 .cra_priority = 300,
2870 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2871 CRYPTO_ALG_ASYNC,
2872 .cra_blocksize = AES_BLOCK_SIZE,
2873 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2874 .cra_alignmask = 3,
2875 .cra_module = THIS_MODULE,
2876 },
2877 .min_keysize = AES_MIN_KEY_SIZE,
2878 .max_keysize = AES_MAX_KEY_SIZE,
2879 .setkey = artpec6_crypto_cipher_set_key,
2880 .encrypt = artpec6_crypto_encrypt,
2881 .decrypt = artpec6_crypto_decrypt,
2882 .init = artpec6_crypto_aes_ecb_init,
2883 .exit = artpec6_crypto_aes_exit,
2884 },
2885 /* AES - CTR */
2886 {
2887 .base = {
2888 .cra_name = "ctr(aes)",
2889 .cra_driver_name = "artpec6-ctr-aes",
2890 .cra_priority = 300,
2891 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2892 CRYPTO_ALG_ASYNC |
2893 CRYPTO_ALG_NEED_FALLBACK,
2894 .cra_blocksize = 1,
2895 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2896 .cra_alignmask = 3,
2897 .cra_module = THIS_MODULE,
2898 },
2899 .min_keysize = AES_MIN_KEY_SIZE,
2900 .max_keysize = AES_MAX_KEY_SIZE,
2901 .ivsize = AES_BLOCK_SIZE,
2902 .setkey = artpec6_crypto_cipher_set_key,
2903 .encrypt = artpec6_crypto_ctr_encrypt,
2904 .decrypt = artpec6_crypto_ctr_decrypt,
2905 .init = artpec6_crypto_aes_ctr_init,
2906 .exit = artpec6_crypto_aes_ctr_exit,
2907 },
2908 /* AES - CBC */
2909 {
2910 .base = {
2911 .cra_name = "cbc(aes)",
2912 .cra_driver_name = "artpec6-cbc-aes",
2913 .cra_priority = 300,
2914 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2915 CRYPTO_ALG_ASYNC,
2916 .cra_blocksize = AES_BLOCK_SIZE,
2917 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2918 .cra_alignmask = 3,
2919 .cra_module = THIS_MODULE,
2920 },
2921 .min_keysize = AES_MIN_KEY_SIZE,
2922 .max_keysize = AES_MAX_KEY_SIZE,
2923 .ivsize = AES_BLOCK_SIZE,
2924 .setkey = artpec6_crypto_cipher_set_key,
2925 .encrypt = artpec6_crypto_encrypt,
2926 .decrypt = artpec6_crypto_decrypt,
2927 .init = artpec6_crypto_aes_cbc_init,
2928 .exit = artpec6_crypto_aes_exit
2929 },
2930 /* AES - XTS */
2931 {
2932 .base = {
2933 .cra_name = "xts(aes)",
2934 .cra_driver_name = "artpec6-xts-aes",
2935 .cra_priority = 300,
2936 .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2937 CRYPTO_ALG_ASYNC,
2938 .cra_blocksize = 1,
2939 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2940 .cra_alignmask = 3,
2941 .cra_module = THIS_MODULE,
2942 },
2943 .min_keysize = 2*AES_MIN_KEY_SIZE,
2944 .max_keysize = 2*AES_MAX_KEY_SIZE,
2945 .ivsize = 16,
2946 .setkey = artpec6_crypto_xts_set_key,
2947 .encrypt = artpec6_crypto_encrypt,
2948 .decrypt = artpec6_crypto_decrypt,
2949 .init = artpec6_crypto_aes_xts_init,
2950 .exit = artpec6_crypto_aes_exit,
2951 },
2952 };
2953
2954 static struct aead_alg aead_algos[] = {
2955 {
2956 .init = artpec6_crypto_aead_init,
2957 .setkey = artpec6_crypto_aead_set_key,
2958 .encrypt = artpec6_crypto_aead_encrypt,
2959 .decrypt = artpec6_crypto_aead_decrypt,
2960 .ivsize = GCM_AES_IV_SIZE,
2961 .maxauthsize = AES_BLOCK_SIZE,
2962
2963 .base = {
2964 .cra_name = "gcm(aes)",
2965 .cra_driver_name = "artpec-gcm-aes",
2966 .cra_priority = 300,
2967 .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
2968 CRYPTO_ALG_KERN_DRIVER_ONLY,
2969 .cra_blocksize = 1,
2970 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2971 .cra_alignmask = 3,
2972 .cra_module = THIS_MODULE,
2973 },
2974 }
2975 };
2976
2977 #ifdef CONFIG_DEBUG_FS
2978
2979 struct dbgfs_u32 {
2980 char *name;
2981 mode_t mode;
2982 u32 *flag;
2983 char *desc;
2984 };
2985
2986 static struct dentry *dbgfs_root;
2987
2988 static void artpec6_crypto_init_debugfs(void)
2989 {
2990 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL);
2991
2992 if (!dbgfs_root || IS_ERR(dbgfs_root)) {
2993 dbgfs_root = NULL;
2994 pr_err("%s: Could not initialise debugfs!\n", MODULE_NAME);
2995 return;
2996 }
2997
2998 #ifdef CONFIG_FAULT_INJECTION
2999 fault_create_debugfs_attr("fail_status_read", dbgfs_root,
3000 &artpec6_crypto_fail_status_read);
3001
3002 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root,
3003 &artpec6_crypto_fail_dma_array_full);
3004 #endif
3005 }
3006
3007 static void artpec6_crypto_free_debugfs(void)
3008 {
3009 if (!dbgfs_root)
3010 return;
3011
3012 debugfs_remove_recursive(dbgfs_root);
3013 dbgfs_root = NULL;
3014 }
3015 #endif
3016
3017 static const struct of_device_id artpec6_crypto_of_match[] = {
3018 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO },
3019 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO },
3020 {}
3021 };
3022 MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match);
3023
3024 static int artpec6_crypto_probe(struct platform_device *pdev)
3025 {
3026 const struct of_device_id *match;
3027 enum artpec6_crypto_variant variant;
3028 struct artpec6_crypto *ac;
3029 struct device *dev = &pdev->dev;
3030 void __iomem *base;
3031 struct resource *res;
3032 int irq;
3033 int err;
3034
3035 if (artpec6_crypto_dev)
3036 return -ENODEV;
3037
3038 match = of_match_node(artpec6_crypto_of_match, dev->of_node);
3039 if (!match)
3040 return -EINVAL;
3041
3042 variant = (enum artpec6_crypto_variant)match->data;
3043
3044 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3045 base = devm_ioremap_resource(&pdev->dev, res);
3046 if (IS_ERR(base))
3047 return PTR_ERR(base);
3048
3049 irq = platform_get_irq(pdev, 0);
3050 if (irq < 0)
3051 return -ENODEV;
3052
3053 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),
3054 GFP_KERNEL);
3055 if (!ac)
3056 return -ENOMEM;
3057
3058 platform_set_drvdata(pdev, ac);
3059 ac->variant = variant;
3060
3061 spin_lock_init(&ac->queue_lock);
3062 INIT_LIST_HEAD(&ac->queue);
3063 INIT_LIST_HEAD(&ac->pending);
3064 timer_setup(&ac->timer, artpec6_crypto_timeout, 0);
3065
3066 ac->base = base;
3067
3068 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",
3069 sizeof(struct artpec6_crypto_dma_descriptors),
3070 64,
3071 0,
3072 NULL);
3073 if (!ac->dma_cache)
3074 return -ENOMEM;
3075
3076 #ifdef CONFIG_DEBUG_FS
3077 artpec6_crypto_init_debugfs();
3078 #endif
3079
3080 tasklet_init(&ac->task, artpec6_crypto_task,
3081 (unsigned long)ac);
3082
3083 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
3084 GFP_KERNEL);
3085 if (!ac->pad_buffer)
3086 return -ENOMEM;
3087 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);
3088
3089 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
3090 GFP_KERNEL);
3091 if (!ac->zero_buffer)
3092 return -ENOMEM;
3093 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);
3094
3095 err = init_crypto_hw(ac);
3096 if (err)
3097 goto free_cache;
3098
3099 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0,
3100 "artpec6-crypto", ac);
3101 if (err)
3102 goto disable_hw;
3103
3104 artpec6_crypto_dev = &pdev->dev;
3105
3106 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
3107 if (err) {
3108 dev_err(dev, "Failed to register ahashes\n");
3109 goto disable_hw;
3110 }
3111
3112 if (variant != ARTPEC6_CRYPTO) {
3113 err = crypto_register_ahashes(artpec7_hash_algos,
3114 ARRAY_SIZE(artpec7_hash_algos));
3115 if (err) {
3116 dev_err(dev, "Failed to register ahashes\n");
3117 goto unregister_ahashes;
3118 }
3119 }
3120
3121 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
3122 if (err) {
3123 dev_err(dev, "Failed to register ciphers\n");
3124 goto unregister_a7_ahashes;
3125 }
3126
3127 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos));
3128 if (err) {
3129 dev_err(dev, "Failed to register aeads\n");
3130 goto unregister_algs;
3131 }
3132
3133 return 0;
3134
3135 unregister_algs:
3136 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
3137 unregister_a7_ahashes:
3138 if (variant != ARTPEC6_CRYPTO)
3139 crypto_unregister_ahashes(artpec7_hash_algos,
3140 ARRAY_SIZE(artpec7_hash_algos));
3141 unregister_ahashes:
3142 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
3143 disable_hw:
3144 artpec6_crypto_disable_hw(ac);
3145 free_cache:
3146 kmem_cache_destroy(ac->dma_cache);
3147 return err;
3148 }
3149
3150 static int artpec6_crypto_remove(struct platform_device *pdev)
3151 {
3152 struct artpec6_crypto *ac = platform_get_drvdata(pdev);
3153 int irq = platform_get_irq(pdev, 0);
3154
3155 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
3156 if (ac->variant != ARTPEC6_CRYPTO)
3157 crypto_unregister_ahashes(artpec7_hash_algos,
3158 ARRAY_SIZE(artpec7_hash_algos));
3159 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
3160 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos));
3161
3162 tasklet_disable(&ac->task);
3163 devm_free_irq(&pdev->dev, irq, ac);
3164 tasklet_kill(&ac->task);
3165 del_timer_sync(&ac->timer);
3166
3167 artpec6_crypto_disable_hw(ac);
3168
3169 kmem_cache_destroy(ac->dma_cache);
3170 #ifdef CONFIG_DEBUG_FS
3171 artpec6_crypto_free_debugfs();
3172 #endif
3173 return 0;
3174 }
3175
3176 static struct platform_driver artpec6_crypto_driver = {
3177 .probe = artpec6_crypto_probe,
3178 .remove = artpec6_crypto_remove,
3179 .driver = {
3180 .name = "artpec6-crypto",
3181 .owner = THIS_MODULE,
3182 .of_match_table = artpec6_crypto_of_match,
3183 },
3184 };
3185
3186 module_platform_driver(artpec6_crypto_driver);
3187
3188 MODULE_AUTHOR("Axis Communications AB");
3189 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver");
3190 MODULE_LICENSE("GPL");
CRIS linux kernel tree