?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'regmap-fix-v5.11-rc2' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux/fpc-iii.git] / drivers / crypto / axis / artpec6_crypto.c
blob9ad188cffd0d71273c70ead63abbedbd903d5d6f
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Driver for ARTPEC-6 crypto block using the kernel asynchronous crypto api.
4 *
5 * Copyright (C) 2014-2017 Axis Communications AB
6 */
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8
9 #include <linux/bitfield.h>
10 #include <linux/crypto.h>
11 #include <linux/debugfs.h>
12 #include <linux/delay.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/fault-inject.h>
15 #include <linux/init.h>
16 #include <linux/interrupt.h>
17 #include <linux/kernel.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/of.h>
21 #include <linux/platform_device.h>
22 #include <linux/scatterlist.h>
23 #include <linux/slab.h>
24
25 #include <crypto/aes.h>
26 #include <crypto/gcm.h>
27 #include <crypto/internal/aead.h>
28 #include <crypto/internal/hash.h>
29 #include <crypto/internal/skcipher.h>
30 #include <crypto/scatterwalk.h>
31 #include <crypto/sha1.h>
32 #include <crypto/sha2.h>
33 #include <crypto/xts.h>
34
35 /* Max length of a line in all cache levels for Artpec SoCs. */
36 #define ARTPEC_CACHE_LINE_MAX 32
37
38 #define PDMA_OUT_CFG 0x0000
39 #define PDMA_OUT_BUF_CFG 0x0004
40 #define PDMA_OUT_CMD 0x0008
41 #define PDMA_OUT_DESCRQ_PUSH 0x0010
42 #define PDMA_OUT_DESCRQ_STAT 0x0014
43
44 #define A6_PDMA_IN_CFG 0x0028
45 #define A6_PDMA_IN_BUF_CFG 0x002c
46 #define A6_PDMA_IN_CMD 0x0030
47 #define A6_PDMA_IN_STATQ_PUSH 0x0038
48 #define A6_PDMA_IN_DESCRQ_PUSH 0x0044
49 #define A6_PDMA_IN_DESCRQ_STAT 0x0048
50 #define A6_PDMA_INTR_MASK 0x0068
51 #define A6_PDMA_ACK_INTR 0x006c
52 #define A6_PDMA_MASKED_INTR 0x0074
53
54 #define A7_PDMA_IN_CFG 0x002c
55 #define A7_PDMA_IN_BUF_CFG 0x0030
56 #define A7_PDMA_IN_CMD 0x0034
57 #define A7_PDMA_IN_STATQ_PUSH 0x003c
58 #define A7_PDMA_IN_DESCRQ_PUSH 0x0048
59 #define A7_PDMA_IN_DESCRQ_STAT 0x004C
60 #define A7_PDMA_INTR_MASK 0x006c
61 #define A7_PDMA_ACK_INTR 0x0070
62 #define A7_PDMA_MASKED_INTR 0x0078
63
64 #define PDMA_OUT_CFG_EN BIT(0)
65
66 #define PDMA_OUT_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
67 #define PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
68
69 #define PDMA_OUT_CMD_START BIT(0)
70 #define A6_PDMA_OUT_CMD_STOP BIT(3)
71 #define A7_PDMA_OUT_CMD_STOP BIT(2)
72
73 #define PDMA_OUT_DESCRQ_PUSH_LEN GENMASK(5, 0)
74 #define PDMA_OUT_DESCRQ_PUSH_ADDR GENMASK(31, 6)
75
76 #define PDMA_OUT_DESCRQ_STAT_LEVEL GENMASK(3, 0)
77 #define PDMA_OUT_DESCRQ_STAT_SIZE GENMASK(7, 4)
78
79 #define PDMA_IN_CFG_EN BIT(0)
80
81 #define PDMA_IN_BUF_CFG_DATA_BUF_SIZE GENMASK(4, 0)
82 #define PDMA_IN_BUF_CFG_DESCR_BUF_SIZE GENMASK(9, 5)
83 #define PDMA_IN_BUF_CFG_STAT_BUF_SIZE GENMASK(14, 10)
84
85 #define PDMA_IN_CMD_START BIT(0)
86 #define A6_PDMA_IN_CMD_FLUSH_STAT BIT(2)
87 #define A6_PDMA_IN_CMD_STOP BIT(3)
88 #define A7_PDMA_IN_CMD_FLUSH_STAT BIT(1)
89 #define A7_PDMA_IN_CMD_STOP BIT(2)
90
91 #define PDMA_IN_STATQ_PUSH_LEN GENMASK(5, 0)
92 #define PDMA_IN_STATQ_PUSH_ADDR GENMASK(31, 6)
93
94 #define PDMA_IN_DESCRQ_PUSH_LEN GENMASK(5, 0)
95 #define PDMA_IN_DESCRQ_PUSH_ADDR GENMASK(31, 6)
96
97 #define PDMA_IN_DESCRQ_STAT_LEVEL GENMASK(3, 0)
98 #define PDMA_IN_DESCRQ_STAT_SIZE GENMASK(7, 4)
99
100 #define A6_PDMA_INTR_MASK_IN_DATA BIT(2)
101 #define A6_PDMA_INTR_MASK_IN_EOP BIT(3)
102 #define A6_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(4)
103
104 #define A7_PDMA_INTR_MASK_IN_DATA BIT(3)
105 #define A7_PDMA_INTR_MASK_IN_EOP BIT(4)
106 #define A7_PDMA_INTR_MASK_IN_EOP_FLUSH BIT(5)
107
108 #define A6_CRY_MD_OPER GENMASK(19, 16)
109
110 #define A6_CRY_MD_HASH_SEL_CTX GENMASK(21, 20)
111 #define A6_CRY_MD_HASH_HMAC_FIN BIT(23)
112
113 #define A6_CRY_MD_CIPHER_LEN GENMASK(21, 20)
114 #define A6_CRY_MD_CIPHER_DECR BIT(22)
115 #define A6_CRY_MD_CIPHER_TWEAK BIT(23)
116 #define A6_CRY_MD_CIPHER_DSEQ BIT(24)
117
118 #define A7_CRY_MD_OPER GENMASK(11, 8)
119
120 #define A7_CRY_MD_HASH_SEL_CTX GENMASK(13, 12)
121 #define A7_CRY_MD_HASH_HMAC_FIN BIT(15)
122
123 #define A7_CRY_MD_CIPHER_LEN GENMASK(13, 12)
124 #define A7_CRY_MD_CIPHER_DECR BIT(14)
125 #define A7_CRY_MD_CIPHER_TWEAK BIT(15)
126 #define A7_CRY_MD_CIPHER_DSEQ BIT(16)
127
128 /* DMA metadata constants */
129 #define regk_crypto_aes_cbc 0x00000002
130 #define regk_crypto_aes_ctr 0x00000003
131 #define regk_crypto_aes_ecb 0x00000001
132 #define regk_crypto_aes_gcm 0x00000004
133 #define regk_crypto_aes_xts 0x00000005
134 #define regk_crypto_cache 0x00000002
135 #define a6_regk_crypto_dlkey 0x0000000a
136 #define a7_regk_crypto_dlkey 0x0000000e
137 #define regk_crypto_ext 0x00000001
138 #define regk_crypto_hmac_sha1 0x00000007
139 #define regk_crypto_hmac_sha256 0x00000009
140 #define regk_crypto_init 0x00000000
141 #define regk_crypto_key_128 0x00000000
142 #define regk_crypto_key_192 0x00000001
143 #define regk_crypto_key_256 0x00000002
144 #define regk_crypto_null 0x00000000
145 #define regk_crypto_sha1 0x00000006
146 #define regk_crypto_sha256 0x00000008
147
148 /* DMA descriptor structures */
149 struct pdma_descr_ctrl {
150 unsigned char short_descr : 1;
151 unsigned char pad1 : 1;
152 unsigned char eop : 1;
153 unsigned char intr : 1;
154 unsigned char short_len : 3;
155 unsigned char pad2 : 1;
156 } __packed;
157
158 struct pdma_data_descr {
159 unsigned int len : 24;
160 unsigned int buf : 32;
161 } __packed;
162
163 struct pdma_short_descr {
164 unsigned char data[7];
165 } __packed;
166
167 struct pdma_descr {
168 struct pdma_descr_ctrl ctrl;
169 union {
170 struct pdma_data_descr data;
171 struct pdma_short_descr shrt;
172 };
173 };
174
175 struct pdma_stat_descr {
176 unsigned char pad1 : 1;
177 unsigned char pad2 : 1;
178 unsigned char eop : 1;
179 unsigned char pad3 : 5;
180 unsigned int len : 24;
181 };
182
183 /* Each descriptor array can hold max 64 entries */
184 #define PDMA_DESCR_COUNT 64
185
186 #define MODULE_NAME "Artpec-6 CA"
187
188 /* Hash modes (including HMAC variants) */
189 #define ARTPEC6_CRYPTO_HASH_SHA1 1
190 #define ARTPEC6_CRYPTO_HASH_SHA256 2
191
192 /* Crypto modes */
193 #define ARTPEC6_CRYPTO_CIPHER_AES_ECB 1
194 #define ARTPEC6_CRYPTO_CIPHER_AES_CBC 2
195 #define ARTPEC6_CRYPTO_CIPHER_AES_CTR 3
196 #define ARTPEC6_CRYPTO_CIPHER_AES_XTS 5
197
198 /* The PDMA is a DMA-engine tightly coupled with a ciphering engine.
199 * It operates on a descriptor array with up to 64 descriptor entries.
200 * The arrays must be 64 byte aligned in memory.
201 *
202 * The ciphering unit has no registers and is completely controlled by
203 * a 4-byte metadata that is inserted at the beginning of each dma packet.
204 *
205 * A dma packet is a sequence of descriptors terminated by setting the .eop
206 * field in the final descriptor of the packet.
207 *
208 * Multiple packets are used for providing context data, key data and
209 * the plain/ciphertext.
210 *
211 * PDMA Descriptors (Array)
212 * +------+------+------+~~+-------+------+----
213 * | 0 | 1 | 2 |~~| 11 EOP| 12 | ....
214 * +--+---+--+---+----+-+~~+-------+----+-+----
215 * | | | | |
216 * | | | | |
217 * __|__ +-------++-------++-------+ +----+
218 * | MD | |Payload||Payload||Payload| | MD |
219 * +-----+ +-------++-------++-------+ +----+
220 */
221
222 struct artpec6_crypto_bounce_buffer {
223 struct list_head list;
224 size_t length;
225 struct scatterlist *sg;
226 size_t offset;
227 /* buf is aligned to ARTPEC_CACHE_LINE_MAX and
228 * holds up to ARTPEC_CACHE_LINE_MAX bytes data.
229 */
230 void *buf;
231 };
232
233 struct artpec6_crypto_dma_map {
234 dma_addr_t dma_addr;
235 size_t size;
236 enum dma_data_direction dir;
237 };
238
239 struct artpec6_crypto_dma_descriptors {
240 struct pdma_descr out[PDMA_DESCR_COUNT] __aligned(64);
241 struct pdma_descr in[PDMA_DESCR_COUNT] __aligned(64);
242 u32 stat[PDMA_DESCR_COUNT] __aligned(64);
243 struct list_head bounce_buffers;
244 /* Enough maps for all out/in buffers, and all three descr. arrays */
245 struct artpec6_crypto_dma_map maps[PDMA_DESCR_COUNT * 2 + 2];
246 dma_addr_t out_dma_addr;
247 dma_addr_t in_dma_addr;
248 dma_addr_t stat_dma_addr;
249 size_t out_cnt;
250 size_t in_cnt;
251 size_t map_count;
252 };
253
254 enum artpec6_crypto_variant {
255 ARTPEC6_CRYPTO,
256 ARTPEC7_CRYPTO,
257 };
258
259 struct artpec6_crypto {
260 void __iomem *base;
261 spinlock_t queue_lock;
262 struct list_head queue; /* waiting for pdma fifo space */
263 struct list_head pending; /* submitted to pdma fifo */
264 struct tasklet_struct task;
265 struct kmem_cache *dma_cache;
266 int pending_count;
267 struct timer_list timer;
268 enum artpec6_crypto_variant variant;
269 void *pad_buffer; /* cache-aligned block padding buffer */
270 void *zero_buffer;
271 };
272
273 enum artpec6_crypto_hash_flags {
274 HASH_FLAG_INIT_CTX = 2,
275 HASH_FLAG_UPDATE = 4,
276 HASH_FLAG_FINALIZE = 8,
277 HASH_FLAG_HMAC = 16,
278 HASH_FLAG_UPDATE_KEY = 32,
279 };
280
281 struct artpec6_crypto_req_common {
282 struct list_head list;
283 struct list_head complete_in_progress;
284 struct artpec6_crypto_dma_descriptors *dma;
285 struct crypto_async_request *req;
286 void (*complete)(struct crypto_async_request *req);
287 gfp_t gfp_flags;
288 };
289
290 struct artpec6_hash_request_context {
291 char partial_buffer[SHA256_BLOCK_SIZE];
292 char partial_buffer_out[SHA256_BLOCK_SIZE];
293 char key_buffer[SHA256_BLOCK_SIZE];
294 char pad_buffer[SHA256_BLOCK_SIZE + 32];
295 unsigned char digeststate[SHA256_DIGEST_SIZE];
296 size_t partial_bytes;
297 u64 digcnt;
298 u32 key_md;
299 u32 hash_md;
300 enum artpec6_crypto_hash_flags hash_flags;
301 struct artpec6_crypto_req_common common;
302 };
303
304 struct artpec6_hash_export_state {
305 char partial_buffer[SHA256_BLOCK_SIZE];
306 unsigned char digeststate[SHA256_DIGEST_SIZE];
307 size_t partial_bytes;
308 u64 digcnt;
309 int oper;
310 unsigned int hash_flags;
311 };
312
313 struct artpec6_hashalg_context {
314 char hmac_key[SHA256_BLOCK_SIZE];
315 size_t hmac_key_length;
316 struct crypto_shash *child_hash;
317 };
318
319 struct artpec6_crypto_request_context {
320 u32 cipher_md;
321 bool decrypt;
322 struct artpec6_crypto_req_common common;
323 };
324
325 struct artpec6_cryptotfm_context {
326 unsigned char aes_key[2*AES_MAX_KEY_SIZE];
327 size_t key_length;
328 u32 key_md;
329 int crypto_type;
330 struct crypto_sync_skcipher *fallback;
331 };
332
333 struct artpec6_crypto_aead_hw_ctx {
334 __be64 aad_length_bits;
335 __be64 text_length_bits;
336 __u8 J0[AES_BLOCK_SIZE];
337 };
338
339 struct artpec6_crypto_aead_req_ctx {
340 struct artpec6_crypto_aead_hw_ctx hw_ctx;
341 u32 cipher_md;
342 bool decrypt;
343 struct artpec6_crypto_req_common common;
344 __u8 decryption_tag[AES_BLOCK_SIZE] ____cacheline_aligned;
345 };
346
347 /* The crypto framework makes it hard to avoid this global. */
348 static struct device *artpec6_crypto_dev;
349
350 #ifdef CONFIG_FAULT_INJECTION
351 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_status_read);
352 static DECLARE_FAULT_ATTR(artpec6_crypto_fail_dma_array_full);
353 #endif
354
355 enum {
356 ARTPEC6_CRYPTO_PREPARE_HASH_NO_START,
357 ARTPEC6_CRYPTO_PREPARE_HASH_START,
358 };
359
360 static int artpec6_crypto_prepare_aead(struct aead_request *areq);
361 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq);
362 static int artpec6_crypto_prepare_hash(struct ahash_request *areq);
363
364 static void
365 artpec6_crypto_complete_crypto(struct crypto_async_request *req);
366 static void
367 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req);
368 static void
369 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req);
370 static void
371 artpec6_crypto_complete_aead(struct crypto_async_request *req);
372 static void
373 artpec6_crypto_complete_hash(struct crypto_async_request *req);
374
375 static int
376 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common);
377
378 static void
379 artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common);
380
381 struct artpec6_crypto_walk {
382 struct scatterlist *sg;
383 size_t offset;
384 };
385
386 static void artpec6_crypto_walk_init(struct artpec6_crypto_walk *awalk,
387 struct scatterlist *sg)
388 {
389 awalk->sg = sg;
390 awalk->offset = 0;
391 }
392
393 static size_t artpec6_crypto_walk_advance(struct artpec6_crypto_walk *awalk,
394 size_t nbytes)
395 {
396 while (nbytes && awalk->sg) {
397 size_t piece;
398
399 WARN_ON(awalk->offset > awalk->sg->length);
400
401 piece = min(nbytes, (size_t)awalk->sg->length - awalk->offset);
402 nbytes -= piece;
403 awalk->offset += piece;
404 if (awalk->offset == awalk->sg->length) {
405 awalk->sg = sg_next(awalk->sg);
406 awalk->offset = 0;
407 }
408
409 }
410
411 return nbytes;
412 }
413
414 static size_t
415 artpec6_crypto_walk_chunklen(const struct artpec6_crypto_walk *awalk)
416 {
417 WARN_ON(awalk->sg->length == awalk->offset);
418
419 return awalk->sg->length - awalk->offset;
420 }
421
422 static dma_addr_t
423 artpec6_crypto_walk_chunk_phys(const struct artpec6_crypto_walk *awalk)
424 {
425 return sg_phys(awalk->sg) + awalk->offset;
426 }
427
428 static void
429 artpec6_crypto_copy_bounce_buffers(struct artpec6_crypto_req_common *common)
430 {
431 struct artpec6_crypto_dma_descriptors *dma = common->dma;
432 struct artpec6_crypto_bounce_buffer *b;
433 struct artpec6_crypto_bounce_buffer *next;
434
435 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
436 pr_debug("bounce entry %p: %zu bytes @ %zu from %p\n",
437 b, b->length, b->offset, b->buf);
438 sg_pcopy_from_buffer(b->sg,
439 1,
440 b->buf,
441 b->length,
442 b->offset);
443
444 list_del(&b->list);
445 kfree(b);
446 }
447 }
448
449 static inline bool artpec6_crypto_busy(void)
450 {
451 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
452 int fifo_count = ac->pending_count;
453
454 return fifo_count > 6;
455 }
456
457 static int artpec6_crypto_submit(struct artpec6_crypto_req_common *req)
458 {
459 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
460 int ret = -EBUSY;
461
462 spin_lock_bh(&ac->queue_lock);
463
464 if (!artpec6_crypto_busy()) {
465 list_add_tail(&req->list, &ac->pending);
466 artpec6_crypto_start_dma(req);
467 ret = -EINPROGRESS;
468 } else if (req->req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG) {
469 list_add_tail(&req->list, &ac->queue);
470 } else {
471 artpec6_crypto_common_destroy(req);
472 }
473
474 spin_unlock_bh(&ac->queue_lock);
475
476 return ret;
477 }
478
479 static void artpec6_crypto_start_dma(struct artpec6_crypto_req_common *common)
480 {
481 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
482 enum artpec6_crypto_variant variant = ac->variant;
483 void __iomem *base = ac->base;
484 struct artpec6_crypto_dma_descriptors *dma = common->dma;
485 u32 ind, statd, outd;
486
487 /* Make descriptor content visible to the DMA before starting it. */
488 wmb();
489
490 ind = FIELD_PREP(PDMA_IN_DESCRQ_PUSH_LEN, dma->in_cnt - 1) |
491 FIELD_PREP(PDMA_IN_DESCRQ_PUSH_ADDR, dma->in_dma_addr >> 6);
492
493 statd = FIELD_PREP(PDMA_IN_STATQ_PUSH_LEN, dma->in_cnt - 1) |
494 FIELD_PREP(PDMA_IN_STATQ_PUSH_ADDR, dma->stat_dma_addr >> 6);
495
496 outd = FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_LEN, dma->out_cnt - 1) |
497 FIELD_PREP(PDMA_OUT_DESCRQ_PUSH_ADDR, dma->out_dma_addr >> 6);
498
499 if (variant == ARTPEC6_CRYPTO) {
500 writel_relaxed(ind, base + A6_PDMA_IN_DESCRQ_PUSH);
501 writel_relaxed(statd, base + A6_PDMA_IN_STATQ_PUSH);
502 writel_relaxed(PDMA_IN_CMD_START, base + A6_PDMA_IN_CMD);
503 } else {
504 writel_relaxed(ind, base + A7_PDMA_IN_DESCRQ_PUSH);
505 writel_relaxed(statd, base + A7_PDMA_IN_STATQ_PUSH);
506 writel_relaxed(PDMA_IN_CMD_START, base + A7_PDMA_IN_CMD);
507 }
508
509 writel_relaxed(outd, base + PDMA_OUT_DESCRQ_PUSH);
510 writel_relaxed(PDMA_OUT_CMD_START, base + PDMA_OUT_CMD);
511
512 ac->pending_count++;
513 }
514
515 static void
516 artpec6_crypto_init_dma_operation(struct artpec6_crypto_req_common *common)
517 {
518 struct artpec6_crypto_dma_descriptors *dma = common->dma;
519
520 dma->out_cnt = 0;
521 dma->in_cnt = 0;
522 dma->map_count = 0;
523 INIT_LIST_HEAD(&dma->bounce_buffers);
524 }
525
526 static bool fault_inject_dma_descr(void)
527 {
528 #ifdef CONFIG_FAULT_INJECTION
529 return should_fail(&artpec6_crypto_fail_dma_array_full, 1);
530 #else
531 return false;
532 #endif
533 }
534
535 /** artpec6_crypto_setup_out_descr_phys - Setup an out channel with a
536 * physical address
537 *
538 * @addr: The physical address of the data buffer
539 * @len: The length of the data buffer
540 * @eop: True if this is the last buffer in the packet
541 *
542 * @return 0 on success or -ENOSPC if there are no more descriptors available
543 */
544 static int
545 artpec6_crypto_setup_out_descr_phys(struct artpec6_crypto_req_common *common,
546 dma_addr_t addr, size_t len, bool eop)
547 {
548 struct artpec6_crypto_dma_descriptors *dma = common->dma;
549 struct pdma_descr *d;
550
551 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
552 fault_inject_dma_descr()) {
553 pr_err("No free OUT DMA descriptors available!\n");
554 return -ENOSPC;
555 }
556
557 d = &dma->out[dma->out_cnt++];
558 memset(d, 0, sizeof(*d));
559
560 d->ctrl.short_descr = 0;
561 d->ctrl.eop = eop;
562 d->data.len = len;
563 d->data.buf = addr;
564 return 0;
565 }
566
567 /** artpec6_crypto_setup_out_descr_short - Setup a short out descriptor
568 *
569 * @dst: The virtual address of the data
570 * @len: The length of the data, must be between 1 to 7 bytes
571 * @eop: True if this is the last buffer in the packet
572 *
573 * @return 0 on success
574 * -ENOSPC if no more descriptors are available
575 * -EINVAL if the data length exceeds 7 bytes
576 */
577 static int
578 artpec6_crypto_setup_out_descr_short(struct artpec6_crypto_req_common *common,
579 void *dst, unsigned int len, bool eop)
580 {
581 struct artpec6_crypto_dma_descriptors *dma = common->dma;
582 struct pdma_descr *d;
583
584 if (dma->out_cnt >= PDMA_DESCR_COUNT ||
585 fault_inject_dma_descr()) {
586 pr_err("No free OUT DMA descriptors available!\n");
587 return -ENOSPC;
588 } else if (len > 7 || len < 1) {
589 return -EINVAL;
590 }
591 d = &dma->out[dma->out_cnt++];
592 memset(d, 0, sizeof(*d));
593
594 d->ctrl.short_descr = 1;
595 d->ctrl.short_len = len;
596 d->ctrl.eop = eop;
597 memcpy(d->shrt.data, dst, len);
598 return 0;
599 }
600
601 static int artpec6_crypto_dma_map_page(struct artpec6_crypto_req_common *common,
602 struct page *page, size_t offset,
603 size_t size,
604 enum dma_data_direction dir,
605 dma_addr_t *dma_addr_out)
606 {
607 struct artpec6_crypto_dma_descriptors *dma = common->dma;
608 struct device *dev = artpec6_crypto_dev;
609 struct artpec6_crypto_dma_map *map;
610 dma_addr_t dma_addr;
611
612 *dma_addr_out = 0;
613
614 if (dma->map_count >= ARRAY_SIZE(dma->maps))
615 return -ENOMEM;
616
617 dma_addr = dma_map_page(dev, page, offset, size, dir);
618 if (dma_mapping_error(dev, dma_addr))
619 return -ENOMEM;
620
621 map = &dma->maps[dma->map_count++];
622 map->size = size;
623 map->dma_addr = dma_addr;
624 map->dir = dir;
625
626 *dma_addr_out = dma_addr;
627
628 return 0;
629 }
630
631 static int
632 artpec6_crypto_dma_map_single(struct artpec6_crypto_req_common *common,
633 void *ptr, size_t size,
634 enum dma_data_direction dir,
635 dma_addr_t *dma_addr_out)
636 {
637 struct page *page = virt_to_page(ptr);
638 size_t offset = (uintptr_t)ptr & ~PAGE_MASK;
639
640 return artpec6_crypto_dma_map_page(common, page, offset, size, dir,
641 dma_addr_out);
642 }
643
644 static int
645 artpec6_crypto_dma_map_descs(struct artpec6_crypto_req_common *common)
646 {
647 struct artpec6_crypto_dma_descriptors *dma = common->dma;
648 int ret;
649
650 ret = artpec6_crypto_dma_map_single(common, dma->in,
651 sizeof(dma->in[0]) * dma->in_cnt,
652 DMA_TO_DEVICE, &dma->in_dma_addr);
653 if (ret)
654 return ret;
655
656 ret = artpec6_crypto_dma_map_single(common, dma->out,
657 sizeof(dma->out[0]) * dma->out_cnt,
658 DMA_TO_DEVICE, &dma->out_dma_addr);
659 if (ret)
660 return ret;
661
662 /* We only read one stat descriptor */
663 dma->stat[dma->in_cnt - 1] = 0;
664
665 /*
666 * DMA_BIDIRECTIONAL since we need our zeroing of the stat descriptor
667 * to be written.
668 */
669 return artpec6_crypto_dma_map_single(common,
670 dma->stat,
671 sizeof(dma->stat[0]) * dma->in_cnt,
672 DMA_BIDIRECTIONAL,
673 &dma->stat_dma_addr);
674 }
675
676 static void
677 artpec6_crypto_dma_unmap_all(struct artpec6_crypto_req_common *common)
678 {
679 struct artpec6_crypto_dma_descriptors *dma = common->dma;
680 struct device *dev = artpec6_crypto_dev;
681 int i;
682
683 for (i = 0; i < dma->map_count; i++) {
684 struct artpec6_crypto_dma_map *map = &dma->maps[i];
685
686 dma_unmap_page(dev, map->dma_addr, map->size, map->dir);
687 }
688
689 dma->map_count = 0;
690 }
691
692 /** artpec6_crypto_setup_out_descr - Setup an out descriptor
693 *
694 * @dst: The virtual address of the data
695 * @len: The length of the data
696 * @eop: True if this is the last buffer in the packet
697 * @use_short: If this is true and the data length is 7 bytes or less then
698 * a short descriptor will be used
699 *
700 * @return 0 on success
701 * Any errors from artpec6_crypto_setup_out_descr_short() or
702 * setup_out_descr_phys()
703 */
704 static int
705 artpec6_crypto_setup_out_descr(struct artpec6_crypto_req_common *common,
706 void *dst, unsigned int len, bool eop,
707 bool use_short)
708 {
709 if (use_short && len < 7) {
710 return artpec6_crypto_setup_out_descr_short(common, dst, len,
711 eop);
712 } else {
713 int ret;
714 dma_addr_t dma_addr;
715
716 ret = artpec6_crypto_dma_map_single(common, dst, len,
717 DMA_TO_DEVICE,
718 &dma_addr);
719 if (ret)
720 return ret;
721
722 return artpec6_crypto_setup_out_descr_phys(common, dma_addr,
723 len, eop);
724 }
725 }
726
727 /** artpec6_crypto_setup_in_descr_phys - Setup an in channel with a
728 * physical address
729 *
730 * @addr: The physical address of the data buffer
731 * @len: The length of the data buffer
732 * @intr: True if an interrupt should be fired after HW processing of this
733 * descriptor
734 *
735 */
736 static int
737 artpec6_crypto_setup_in_descr_phys(struct artpec6_crypto_req_common *common,
738 dma_addr_t addr, unsigned int len, bool intr)
739 {
740 struct artpec6_crypto_dma_descriptors *dma = common->dma;
741 struct pdma_descr *d;
742
743 if (dma->in_cnt >= PDMA_DESCR_COUNT ||
744 fault_inject_dma_descr()) {
745 pr_err("No free IN DMA descriptors available!\n");
746 return -ENOSPC;
747 }
748 d = &dma->in[dma->in_cnt++];
749 memset(d, 0, sizeof(*d));
750
751 d->ctrl.intr = intr;
752 d->data.len = len;
753 d->data.buf = addr;
754 return 0;
755 }
756
757 /** artpec6_crypto_setup_in_descr - Setup an in channel descriptor
758 *
759 * @buffer: The virtual address to of the data buffer
760 * @len: The length of the data buffer
761 * @last: If this is the last data buffer in the request (i.e. an interrupt
762 * is needed
763 *
764 * Short descriptors are not used for the in channel
765 */
766 static int
767 artpec6_crypto_setup_in_descr(struct artpec6_crypto_req_common *common,
768 void *buffer, unsigned int len, bool last)
769 {
770 dma_addr_t dma_addr;
771 int ret;
772
773 ret = artpec6_crypto_dma_map_single(common, buffer, len,
774 DMA_FROM_DEVICE, &dma_addr);
775 if (ret)
776 return ret;
777
778 return artpec6_crypto_setup_in_descr_phys(common, dma_addr, len, last);
779 }
780
781 static struct artpec6_crypto_bounce_buffer *
782 artpec6_crypto_alloc_bounce(gfp_t flags)
783 {
784 void *base;
785 size_t alloc_size = sizeof(struct artpec6_crypto_bounce_buffer) +
786 2 * ARTPEC_CACHE_LINE_MAX;
787 struct artpec6_crypto_bounce_buffer *bbuf = kzalloc(alloc_size, flags);
788
789 if (!bbuf)
790 return NULL;
791
792 base = bbuf + 1;
793 bbuf->buf = PTR_ALIGN(base, ARTPEC_CACHE_LINE_MAX);
794 return bbuf;
795 }
796
797 static int setup_bounce_buffer_in(struct artpec6_crypto_req_common *common,
798 struct artpec6_crypto_walk *walk, size_t size)
799 {
800 struct artpec6_crypto_bounce_buffer *bbuf;
801 int ret;
802
803 bbuf = artpec6_crypto_alloc_bounce(common->gfp_flags);
804 if (!bbuf)
805 return -ENOMEM;
806
807 bbuf->length = size;
808 bbuf->sg = walk->sg;
809 bbuf->offset = walk->offset;
810
811 ret = artpec6_crypto_setup_in_descr(common, bbuf->buf, size, false);
812 if (ret) {
813 kfree(bbuf);
814 return ret;
815 }
816
817 pr_debug("BOUNCE %zu offset %zu\n", size, walk->offset);
818 list_add_tail(&bbuf->list, &common->dma->bounce_buffers);
819 return 0;
820 }
821
822 static int
823 artpec6_crypto_setup_sg_descrs_in(struct artpec6_crypto_req_common *common,
824 struct artpec6_crypto_walk *walk,
825 size_t count)
826 {
827 size_t chunk;
828 int ret;
829 dma_addr_t addr;
830
831 while (walk->sg && count) {
832 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
833 addr = artpec6_crypto_walk_chunk_phys(walk);
834
835 /* When destination buffers are not aligned to the cache line
836 * size we need bounce buffers. The DMA-API requires that the
837 * entire line is owned by the DMA buffer and this holds also
838 * for the case when coherent DMA is used.
839 */
840 if (!IS_ALIGNED(addr, ARTPEC_CACHE_LINE_MAX)) {
841 chunk = min_t(dma_addr_t, chunk,
842 ALIGN(addr, ARTPEC_CACHE_LINE_MAX) -
843 addr);
844
845 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
846 ret = setup_bounce_buffer_in(common, walk, chunk);
847 } else if (chunk < ARTPEC_CACHE_LINE_MAX) {
848 pr_debug("CHUNK-b %pad:%zu\n", &addr, chunk);
849 ret = setup_bounce_buffer_in(common, walk, chunk);
850 } else {
851 dma_addr_t dma_addr;
852
853 chunk = chunk & ~(ARTPEC_CACHE_LINE_MAX-1);
854
855 pr_debug("CHUNK %pad:%zu\n", &addr, chunk);
856
857 ret = artpec6_crypto_dma_map_page(common,
858 sg_page(walk->sg),
859 walk->sg->offset +
860 walk->offset,
861 chunk,
862 DMA_FROM_DEVICE,
863 &dma_addr);
864 if (ret)
865 return ret;
866
867 ret = artpec6_crypto_setup_in_descr_phys(common,
868 dma_addr,
869 chunk, false);
870 }
871
872 if (ret)
873 return ret;
874
875 count = count - chunk;
876 artpec6_crypto_walk_advance(walk, chunk);
877 }
878
879 if (count)
880 pr_err("EOL unexpected %zu bytes left\n", count);
881
882 return count ? -EINVAL : 0;
883 }
884
885 static int
886 artpec6_crypto_setup_sg_descrs_out(struct artpec6_crypto_req_common *common,
887 struct artpec6_crypto_walk *walk,
888 size_t count)
889 {
890 size_t chunk;
891 int ret;
892 dma_addr_t addr;
893
894 while (walk->sg && count) {
895 chunk = min(count, artpec6_crypto_walk_chunklen(walk));
896 addr = artpec6_crypto_walk_chunk_phys(walk);
897
898 pr_debug("OUT-CHUNK %pad:%zu\n", &addr, chunk);
899
900 if (addr & 3) {
901 char buf[3];
902
903 chunk = min_t(size_t, chunk, (4-(addr&3)));
904
905 sg_pcopy_to_buffer(walk->sg, 1, buf, chunk,
906 walk->offset);
907
908 ret = artpec6_crypto_setup_out_descr_short(common, buf,
909 chunk,
910 false);
911 } else {
912 dma_addr_t dma_addr;
913
914 ret = artpec6_crypto_dma_map_page(common,
915 sg_page(walk->sg),
916 walk->sg->offset +
917 walk->offset,
918 chunk,
919 DMA_TO_DEVICE,
920 &dma_addr);
921 if (ret)
922 return ret;
923
924 ret = artpec6_crypto_setup_out_descr_phys(common,
925 dma_addr,
926 chunk, false);
927 }
928
929 if (ret)
930 return ret;
931
932 count = count - chunk;
933 artpec6_crypto_walk_advance(walk, chunk);
934 }
935
936 if (count)
937 pr_err("EOL unexpected %zu bytes left\n", count);
938
939 return count ? -EINVAL : 0;
940 }
941
942
943 /** artpec6_crypto_terminate_out_descrs - Set the EOP on the last out descriptor
944 *
945 * If the out descriptor list is non-empty, then the eop flag on the
946 * last used out descriptor will be set.
947 *
948 * @return 0 on success
949 * -EINVAL if the out descriptor is empty or has overflown
950 */
951 static int
952 artpec6_crypto_terminate_out_descrs(struct artpec6_crypto_req_common *common)
953 {
954 struct artpec6_crypto_dma_descriptors *dma = common->dma;
955 struct pdma_descr *d;
956
957 if (!dma->out_cnt || dma->out_cnt > PDMA_DESCR_COUNT) {
958 pr_err("%s: OUT descriptor list is %s\n",
959 MODULE_NAME, dma->out_cnt ? "empty" : "full");
960 return -EINVAL;
961
962 }
963
964 d = &dma->out[dma->out_cnt-1];
965 d->ctrl.eop = 1;
966
967 return 0;
968 }
969
970 /** artpec6_crypto_terminate_in_descrs - Set the interrupt flag on the last
971 * in descriptor
972 *
973 * See artpec6_crypto_terminate_out_descrs() for return values
974 */
975 static int
976 artpec6_crypto_terminate_in_descrs(struct artpec6_crypto_req_common *common)
977 {
978 struct artpec6_crypto_dma_descriptors *dma = common->dma;
979 struct pdma_descr *d;
980
981 if (!dma->in_cnt || dma->in_cnt > PDMA_DESCR_COUNT) {
982 pr_err("%s: IN descriptor list is %s\n",
983 MODULE_NAME, dma->in_cnt ? "empty" : "full");
984 return -EINVAL;
985 }
986
987 d = &dma->in[dma->in_cnt-1];
988 d->ctrl.intr = 1;
989 return 0;
990 }
991
992 /** create_hash_pad - Create a Secure Hash conformant pad
993 *
994 * @dst: The destination buffer to write the pad. Must be at least 64 bytes
995 * @dgstlen: The total length of the hash digest in bytes
996 * @bitcount: The total length of the digest in bits
997 *
998 * @return The total number of padding bytes written to @dst
999 */
1000 static size_t
1001 create_hash_pad(int oper, unsigned char *dst, u64 dgstlen, u64 bitcount)
1002 {
1003 unsigned int mod, target, diff, pad_bytes, size_bytes;
1004 __be64 bits = __cpu_to_be64(bitcount);
1005
1006 switch (oper) {
1007 case regk_crypto_sha1:
1008 case regk_crypto_sha256:
1009 case regk_crypto_hmac_sha1:
1010 case regk_crypto_hmac_sha256:
1011 target = 448 / 8;
1012 mod = 512 / 8;
1013 size_bytes = 8;
1014 break;
1015 default:
1016 target = 896 / 8;
1017 mod = 1024 / 8;
1018 size_bytes = 16;
1019 break;
1020 }
1021
1022 target -= 1;
1023 diff = dgstlen & (mod - 1);
1024 pad_bytes = diff > target ? target + mod - diff : target - diff;
1025
1026 memset(dst + 1, 0, pad_bytes);
1027 dst[0] = 0x80;
1028
1029 if (size_bytes == 16) {
1030 memset(dst + 1 + pad_bytes, 0, 8);
1031 memcpy(dst + 1 + pad_bytes + 8, &bits, 8);
1032 } else {
1033 memcpy(dst + 1 + pad_bytes, &bits, 8);
1034 }
1035
1036 return pad_bytes + size_bytes + 1;
1037 }
1038
1039 static int artpec6_crypto_common_init(struct artpec6_crypto_req_common *common,
1040 struct crypto_async_request *parent,
1041 void (*complete)(struct crypto_async_request *req),
1042 struct scatterlist *dstsg, unsigned int nbytes)
1043 {
1044 gfp_t flags;
1045 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1046
1047 flags = (parent->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
1048 GFP_KERNEL : GFP_ATOMIC;
1049
1050 common->gfp_flags = flags;
1051 common->dma = kmem_cache_alloc(ac->dma_cache, flags);
1052 if (!common->dma)
1053 return -ENOMEM;
1054
1055 common->req = parent;
1056 common->complete = complete;
1057 return 0;
1058 }
1059
1060 static void
1061 artpec6_crypto_bounce_destroy(struct artpec6_crypto_dma_descriptors *dma)
1062 {
1063 struct artpec6_crypto_bounce_buffer *b;
1064 struct artpec6_crypto_bounce_buffer *next;
1065
1066 list_for_each_entry_safe(b, next, &dma->bounce_buffers, list) {
1067 kfree(b);
1068 }
1069 }
1070
1071 static int
1072 artpec6_crypto_common_destroy(struct artpec6_crypto_req_common *common)
1073 {
1074 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1075
1076 artpec6_crypto_dma_unmap_all(common);
1077 artpec6_crypto_bounce_destroy(common->dma);
1078 kmem_cache_free(ac->dma_cache, common->dma);
1079 common->dma = NULL;
1080 return 0;
1081 }
1082
1083 /*
1084 * Ciphering functions.
1085 */
1086 static int artpec6_crypto_encrypt(struct skcipher_request *req)
1087 {
1088 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1089 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1090 struct artpec6_crypto_request_context *req_ctx = NULL;
1091 void (*complete)(struct crypto_async_request *req);
1092 int ret;
1093
1094 req_ctx = skcipher_request_ctx(req);
1095
1096 switch (ctx->crypto_type) {
1097 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1098 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1099 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1100 req_ctx->decrypt = 0;
1101 break;
1102 default:
1103 break;
1104 }
1105
1106 switch (ctx->crypto_type) {
1107 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1108 complete = artpec6_crypto_complete_cbc_encrypt;
1109 break;
1110 default:
1111 complete = artpec6_crypto_complete_crypto;
1112 break;
1113 }
1114
1115 ret = artpec6_crypto_common_init(&req_ctx->common,
1116 &req->base,
1117 complete,
1118 req->dst, req->cryptlen);
1119 if (ret)
1120 return ret;
1121
1122 ret = artpec6_crypto_prepare_crypto(req);
1123 if (ret) {
1124 artpec6_crypto_common_destroy(&req_ctx->common);
1125 return ret;
1126 }
1127
1128 return artpec6_crypto_submit(&req_ctx->common);
1129 }
1130
1131 static int artpec6_crypto_decrypt(struct skcipher_request *req)
1132 {
1133 int ret;
1134 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1135 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1136 struct artpec6_crypto_request_context *req_ctx = NULL;
1137 void (*complete)(struct crypto_async_request *req);
1138
1139 req_ctx = skcipher_request_ctx(req);
1140
1141 switch (ctx->crypto_type) {
1142 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1143 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1144 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1145 req_ctx->decrypt = 1;
1146 break;
1147 default:
1148 break;
1149 }
1150
1151
1152 switch (ctx->crypto_type) {
1153 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1154 complete = artpec6_crypto_complete_cbc_decrypt;
1155 break;
1156 default:
1157 complete = artpec6_crypto_complete_crypto;
1158 break;
1159 }
1160
1161 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1162 complete,
1163 req->dst, req->cryptlen);
1164 if (ret)
1165 return ret;
1166
1167 ret = artpec6_crypto_prepare_crypto(req);
1168 if (ret) {
1169 artpec6_crypto_common_destroy(&req_ctx->common);
1170 return ret;
1171 }
1172
1173 return artpec6_crypto_submit(&req_ctx->common);
1174 }
1175
1176 static int
1177 artpec6_crypto_ctr_crypt(struct skcipher_request *req, bool encrypt)
1178 {
1179 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req);
1180 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1181 size_t iv_len = crypto_skcipher_ivsize(cipher);
1182 unsigned int counter = be32_to_cpup((__be32 *)
1183 (req->iv + iv_len - 4));
1184 unsigned int nblks = ALIGN(req->cryptlen, AES_BLOCK_SIZE) /
1185 AES_BLOCK_SIZE;
1186
1187 /*
1188 * The hardware uses only the last 32-bits as the counter while the
1189 * kernel tests (aes_ctr_enc_tv_template[4] for example) expect that
1190 * the whole IV is a counter. So fallback if the counter is going to
1191 * overlow.
1192 */
1193 if (counter + nblks < counter) {
1194 int ret;
1195
1196 pr_debug("counter %x will overflow (nblks %u), falling back\n",
1197 counter, counter + nblks);
1198
1199 ret = crypto_sync_skcipher_setkey(ctx->fallback, ctx->aes_key,
1200 ctx->key_length);
1201 if (ret)
1202 return ret;
1203
1204 {
1205 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->fallback);
1206
1207 skcipher_request_set_sync_tfm(subreq, ctx->fallback);
1208 skcipher_request_set_callback(subreq, req->base.flags,
1209 NULL, NULL);
1210 skcipher_request_set_crypt(subreq, req->src, req->dst,
1211 req->cryptlen, req->iv);
1212 ret = encrypt ? crypto_skcipher_encrypt(subreq)
1213 : crypto_skcipher_decrypt(subreq);
1214 skcipher_request_zero(subreq);
1215 }
1216 return ret;
1217 }
1218
1219 return encrypt ? artpec6_crypto_encrypt(req)
1220 : artpec6_crypto_decrypt(req);
1221 }
1222
1223 static int artpec6_crypto_ctr_encrypt(struct skcipher_request *req)
1224 {
1225 return artpec6_crypto_ctr_crypt(req, true);
1226 }
1227
1228 static int artpec6_crypto_ctr_decrypt(struct skcipher_request *req)
1229 {
1230 return artpec6_crypto_ctr_crypt(req, false);
1231 }
1232
1233 /*
1234 * AEAD functions
1235 */
1236 static int artpec6_crypto_aead_init(struct crypto_aead *tfm)
1237 {
1238 struct artpec6_cryptotfm_context *tfm_ctx = crypto_aead_ctx(tfm);
1239
1240 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
1241
1242 crypto_aead_set_reqsize(tfm,
1243 sizeof(struct artpec6_crypto_aead_req_ctx));
1244
1245 return 0;
1246 }
1247
1248 static int artpec6_crypto_aead_set_key(struct crypto_aead *tfm, const u8 *key,
1249 unsigned int len)
1250 {
1251 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(&tfm->base);
1252
1253 if (len != 16 && len != 24 && len != 32)
1254 return -EINVAL;
1255
1256 ctx->key_length = len;
1257
1258 memcpy(ctx->aes_key, key, len);
1259 return 0;
1260 }
1261
1262 static int artpec6_crypto_aead_encrypt(struct aead_request *req)
1263 {
1264 int ret;
1265 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1266
1267 req_ctx->decrypt = false;
1268 ret = artpec6_crypto_common_init(&req_ctx->common, &req->base,
1269 artpec6_crypto_complete_aead,
1270 NULL, 0);
1271 if (ret)
1272 return ret;
1273
1274 ret = artpec6_crypto_prepare_aead(req);
1275 if (ret) {
1276 artpec6_crypto_common_destroy(&req_ctx->common);
1277 return ret;
1278 }
1279
1280 return artpec6_crypto_submit(&req_ctx->common);
1281 }
1282
1283 static int artpec6_crypto_aead_decrypt(struct aead_request *req)
1284 {
1285 int ret;
1286 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(req);
1287
1288 req_ctx->decrypt = true;
1289 if (req->cryptlen < AES_BLOCK_SIZE)
1290 return -EINVAL;
1291
1292 ret = artpec6_crypto_common_init(&req_ctx->common,
1293 &req->base,
1294 artpec6_crypto_complete_aead,
1295 NULL, 0);
1296 if (ret)
1297 return ret;
1298
1299 ret = artpec6_crypto_prepare_aead(req);
1300 if (ret) {
1301 artpec6_crypto_common_destroy(&req_ctx->common);
1302 return ret;
1303 }
1304
1305 return artpec6_crypto_submit(&req_ctx->common);
1306 }
1307
1308 static int artpec6_crypto_prepare_hash(struct ahash_request *areq)
1309 {
1310 struct artpec6_hashalg_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1311 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(areq);
1312 size_t digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(areq));
1313 size_t contextsize = digestsize;
1314 size_t blocksize = crypto_tfm_alg_blocksize(
1315 crypto_ahash_tfm(crypto_ahash_reqtfm(areq)));
1316 struct artpec6_crypto_req_common *common = &req_ctx->common;
1317 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1318 enum artpec6_crypto_variant variant = ac->variant;
1319 u32 sel_ctx;
1320 bool ext_ctx = false;
1321 bool run_hw = false;
1322 int error = 0;
1323
1324 artpec6_crypto_init_dma_operation(common);
1325
1326 /* Upload HMAC key, must be first the first packet */
1327 if (req_ctx->hash_flags & HASH_FLAG_HMAC) {
1328 if (variant == ARTPEC6_CRYPTO) {
1329 req_ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1330 a6_regk_crypto_dlkey);
1331 } else {
1332 req_ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1333 a7_regk_crypto_dlkey);
1334 }
1335
1336 /* Copy and pad up the key */
1337 memcpy(req_ctx->key_buffer, ctx->hmac_key,
1338 ctx->hmac_key_length);
1339 memset(req_ctx->key_buffer + ctx->hmac_key_length, 0,
1340 blocksize - ctx->hmac_key_length);
1341
1342 error = artpec6_crypto_setup_out_descr(common,
1343 (void *)&req_ctx->key_md,
1344 sizeof(req_ctx->key_md), false, false);
1345 if (error)
1346 return error;
1347
1348 error = artpec6_crypto_setup_out_descr(common,
1349 req_ctx->key_buffer, blocksize,
1350 true, false);
1351 if (error)
1352 return error;
1353 }
1354
1355 if (!(req_ctx->hash_flags & HASH_FLAG_INIT_CTX)) {
1356 /* Restore context */
1357 sel_ctx = regk_crypto_ext;
1358 ext_ctx = true;
1359 } else {
1360 sel_ctx = regk_crypto_init;
1361 }
1362
1363 if (variant == ARTPEC6_CRYPTO) {
1364 req_ctx->hash_md &= ~A6_CRY_MD_HASH_SEL_CTX;
1365 req_ctx->hash_md |= FIELD_PREP(A6_CRY_MD_HASH_SEL_CTX, sel_ctx);
1366
1367 /* If this is the final round, set the final flag */
1368 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1369 req_ctx->hash_md |= A6_CRY_MD_HASH_HMAC_FIN;
1370 } else {
1371 req_ctx->hash_md &= ~A7_CRY_MD_HASH_SEL_CTX;
1372 req_ctx->hash_md |= FIELD_PREP(A7_CRY_MD_HASH_SEL_CTX, sel_ctx);
1373
1374 /* If this is the final round, set the final flag */
1375 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE)
1376 req_ctx->hash_md |= A7_CRY_MD_HASH_HMAC_FIN;
1377 }
1378
1379 /* Setup up metadata descriptors */
1380 error = artpec6_crypto_setup_out_descr(common,
1381 (void *)&req_ctx->hash_md,
1382 sizeof(req_ctx->hash_md), false, false);
1383 if (error)
1384 return error;
1385
1386 error = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1387 if (error)
1388 return error;
1389
1390 if (ext_ctx) {
1391 error = artpec6_crypto_setup_out_descr(common,
1392 req_ctx->digeststate,
1393 contextsize, false, false);
1394
1395 if (error)
1396 return error;
1397 }
1398
1399 if (req_ctx->hash_flags & HASH_FLAG_UPDATE) {
1400 size_t done_bytes = 0;
1401 size_t total_bytes = areq->nbytes + req_ctx->partial_bytes;
1402 size_t ready_bytes = round_down(total_bytes, blocksize);
1403 struct artpec6_crypto_walk walk;
1404
1405 run_hw = ready_bytes > 0;
1406 if (req_ctx->partial_bytes && ready_bytes) {
1407 /* We have a partial buffer and will at least some bytes
1408 * to the HW. Empty this partial buffer before tackling
1409 * the SG lists
1410 */
1411 memcpy(req_ctx->partial_buffer_out,
1412 req_ctx->partial_buffer,
1413 req_ctx->partial_bytes);
1414
1415 error = artpec6_crypto_setup_out_descr(common,
1416 req_ctx->partial_buffer_out,
1417 req_ctx->partial_bytes,
1418 false, true);
1419 if (error)
1420 return error;
1421
1422 /* Reset partial buffer */
1423 done_bytes += req_ctx->partial_bytes;
1424 req_ctx->partial_bytes = 0;
1425 }
1426
1427 artpec6_crypto_walk_init(&walk, areq->src);
1428
1429 error = artpec6_crypto_setup_sg_descrs_out(common, &walk,
1430 ready_bytes -
1431 done_bytes);
1432 if (error)
1433 return error;
1434
1435 if (walk.sg) {
1436 size_t sg_skip = ready_bytes - done_bytes;
1437 size_t sg_rem = areq->nbytes - sg_skip;
1438
1439 sg_pcopy_to_buffer(areq->src, sg_nents(areq->src),
1440 req_ctx->partial_buffer +
1441 req_ctx->partial_bytes,
1442 sg_rem, sg_skip);
1443
1444 req_ctx->partial_bytes += sg_rem;
1445 }
1446
1447 req_ctx->digcnt += ready_bytes;
1448 req_ctx->hash_flags &= ~(HASH_FLAG_UPDATE);
1449 }
1450
1451 /* Finalize */
1452 if (req_ctx->hash_flags & HASH_FLAG_FINALIZE) {
1453 size_t hash_pad_len;
1454 u64 digest_bits;
1455 u32 oper;
1456
1457 if (variant == ARTPEC6_CRYPTO)
1458 oper = FIELD_GET(A6_CRY_MD_OPER, req_ctx->hash_md);
1459 else
1460 oper = FIELD_GET(A7_CRY_MD_OPER, req_ctx->hash_md);
1461
1462 /* Write out the partial buffer if present */
1463 if (req_ctx->partial_bytes) {
1464 memcpy(req_ctx->partial_buffer_out,
1465 req_ctx->partial_buffer,
1466 req_ctx->partial_bytes);
1467 error = artpec6_crypto_setup_out_descr(common,
1468 req_ctx->partial_buffer_out,
1469 req_ctx->partial_bytes,
1470 false, true);
1471 if (error)
1472 return error;
1473
1474 req_ctx->digcnt += req_ctx->partial_bytes;
1475 req_ctx->partial_bytes = 0;
1476 }
1477
1478 if (req_ctx->hash_flags & HASH_FLAG_HMAC)
1479 digest_bits = 8 * (req_ctx->digcnt + blocksize);
1480 else
1481 digest_bits = 8 * req_ctx->digcnt;
1482
1483 /* Add the hash pad */
1484 hash_pad_len = create_hash_pad(oper, req_ctx->pad_buffer,
1485 req_ctx->digcnt, digest_bits);
1486 error = artpec6_crypto_setup_out_descr(common,
1487 req_ctx->pad_buffer,
1488 hash_pad_len, false,
1489 true);
1490 req_ctx->digcnt = 0;
1491
1492 if (error)
1493 return error;
1494
1495 /* Descriptor for the final result */
1496 error = artpec6_crypto_setup_in_descr(common, areq->result,
1497 digestsize,
1498 true);
1499 if (error)
1500 return error;
1501
1502 } else { /* This is not the final operation for this request */
1503 if (!run_hw)
1504 return ARTPEC6_CRYPTO_PREPARE_HASH_NO_START;
1505
1506 /* Save the result to the context */
1507 error = artpec6_crypto_setup_in_descr(common,
1508 req_ctx->digeststate,
1509 contextsize, false);
1510 if (error)
1511 return error;
1512 /* fall through */
1513 }
1514
1515 req_ctx->hash_flags &= ~(HASH_FLAG_INIT_CTX | HASH_FLAG_UPDATE |
1516 HASH_FLAG_FINALIZE);
1517
1518 error = artpec6_crypto_terminate_in_descrs(common);
1519 if (error)
1520 return error;
1521
1522 error = artpec6_crypto_terminate_out_descrs(common);
1523 if (error)
1524 return error;
1525
1526 error = artpec6_crypto_dma_map_descs(common);
1527 if (error)
1528 return error;
1529
1530 return ARTPEC6_CRYPTO_PREPARE_HASH_START;
1531 }
1532
1533
1534 static int artpec6_crypto_aes_ecb_init(struct crypto_skcipher *tfm)
1535 {
1536 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1537
1538 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1539 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_ECB;
1540
1541 return 0;
1542 }
1543
1544 static int artpec6_crypto_aes_ctr_init(struct crypto_skcipher *tfm)
1545 {
1546 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1547
1548 ctx->fallback =
1549 crypto_alloc_sync_skcipher(crypto_tfm_alg_name(&tfm->base),
1550 0, CRYPTO_ALG_NEED_FALLBACK);
1551 if (IS_ERR(ctx->fallback))
1552 return PTR_ERR(ctx->fallback);
1553
1554 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1555 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CTR;
1556
1557 return 0;
1558 }
1559
1560 static int artpec6_crypto_aes_cbc_init(struct crypto_skcipher *tfm)
1561 {
1562 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1563
1564 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1565 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_CBC;
1566
1567 return 0;
1568 }
1569
1570 static int artpec6_crypto_aes_xts_init(struct crypto_skcipher *tfm)
1571 {
1572 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1573
1574 tfm->reqsize = sizeof(struct artpec6_crypto_request_context);
1575 ctx->crypto_type = ARTPEC6_CRYPTO_CIPHER_AES_XTS;
1576
1577 return 0;
1578 }
1579
1580 static void artpec6_crypto_aes_exit(struct crypto_skcipher *tfm)
1581 {
1582 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1583
1584 memset(ctx, 0, sizeof(*ctx));
1585 }
1586
1587 static void artpec6_crypto_aes_ctr_exit(struct crypto_skcipher *tfm)
1588 {
1589 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(tfm);
1590
1591 crypto_free_sync_skcipher(ctx->fallback);
1592 artpec6_crypto_aes_exit(tfm);
1593 }
1594
1595 static int
1596 artpec6_crypto_cipher_set_key(struct crypto_skcipher *cipher, const u8 *key,
1597 unsigned int keylen)
1598 {
1599 struct artpec6_cryptotfm_context *ctx =
1600 crypto_skcipher_ctx(cipher);
1601
1602 switch (keylen) {
1603 case 16:
1604 case 24:
1605 case 32:
1606 break;
1607 default:
1608 return -EINVAL;
1609 }
1610
1611 memcpy(ctx->aes_key, key, keylen);
1612 ctx->key_length = keylen;
1613 return 0;
1614 }
1615
1616 static int
1617 artpec6_crypto_xts_set_key(struct crypto_skcipher *cipher, const u8 *key,
1618 unsigned int keylen)
1619 {
1620 struct artpec6_cryptotfm_context *ctx =
1621 crypto_skcipher_ctx(cipher);
1622 int ret;
1623
1624 ret = xts_check_key(&cipher->base, key, keylen);
1625 if (ret)
1626 return ret;
1627
1628 switch (keylen) {
1629 case 32:
1630 case 48:
1631 case 64:
1632 break;
1633 default:
1634 return -EINVAL;
1635 }
1636
1637 memcpy(ctx->aes_key, key, keylen);
1638 ctx->key_length = keylen;
1639 return 0;
1640 }
1641
1642 /** artpec6_crypto_process_crypto - Prepare an async block cipher crypto request
1643 *
1644 * @req: The asynch request to process
1645 *
1646 * @return 0 if the dma job was successfully prepared
1647 * <0 on error
1648 *
1649 * This function sets up the PDMA descriptors for a block cipher request.
1650 *
1651 * The required padding is added for AES-CTR using a statically defined
1652 * buffer.
1653 *
1654 * The PDMA descriptor list will be as follows:
1655 *
1656 * OUT: [KEY_MD][KEY][EOP]<CIPHER_MD>[IV]<data_0>...[data_n][AES-CTR_pad]<eop>
1657 * IN: <CIPHER_MD><data_0>...[data_n]<intr>
1658 *
1659 */
1660 static int artpec6_crypto_prepare_crypto(struct skcipher_request *areq)
1661 {
1662 int ret;
1663 struct artpec6_crypto_walk walk;
1664 struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(areq);
1665 struct artpec6_cryptotfm_context *ctx = crypto_skcipher_ctx(cipher);
1666 struct artpec6_crypto_request_context *req_ctx = NULL;
1667 size_t iv_len = crypto_skcipher_ivsize(cipher);
1668 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1669 enum artpec6_crypto_variant variant = ac->variant;
1670 struct artpec6_crypto_req_common *common;
1671 bool cipher_decr = false;
1672 size_t cipher_klen;
1673 u32 cipher_len = 0; /* Same as regk_crypto_key_128 for NULL crypto */
1674 u32 oper;
1675
1676 req_ctx = skcipher_request_ctx(areq);
1677 common = &req_ctx->common;
1678
1679 artpec6_crypto_init_dma_operation(common);
1680
1681 if (variant == ARTPEC6_CRYPTO)
1682 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER, a6_regk_crypto_dlkey);
1683 else
1684 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER, a7_regk_crypto_dlkey);
1685
1686 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1687 sizeof(ctx->key_md), false, false);
1688 if (ret)
1689 return ret;
1690
1691 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1692 ctx->key_length, true, false);
1693 if (ret)
1694 return ret;
1695
1696 req_ctx->cipher_md = 0;
1697
1698 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS)
1699 cipher_klen = ctx->key_length/2;
1700 else
1701 cipher_klen = ctx->key_length;
1702
1703 /* Metadata */
1704 switch (cipher_klen) {
1705 case 16:
1706 cipher_len = regk_crypto_key_128;
1707 break;
1708 case 24:
1709 cipher_len = regk_crypto_key_192;
1710 break;
1711 case 32:
1712 cipher_len = regk_crypto_key_256;
1713 break;
1714 default:
1715 pr_err("%s: Invalid key length %d!\n",
1716 MODULE_NAME, ctx->key_length);
1717 return -EINVAL;
1718 }
1719
1720 switch (ctx->crypto_type) {
1721 case ARTPEC6_CRYPTO_CIPHER_AES_ECB:
1722 oper = regk_crypto_aes_ecb;
1723 cipher_decr = req_ctx->decrypt;
1724 break;
1725
1726 case ARTPEC6_CRYPTO_CIPHER_AES_CBC:
1727 oper = regk_crypto_aes_cbc;
1728 cipher_decr = req_ctx->decrypt;
1729 break;
1730
1731 case ARTPEC6_CRYPTO_CIPHER_AES_CTR:
1732 oper = regk_crypto_aes_ctr;
1733 cipher_decr = false;
1734 break;
1735
1736 case ARTPEC6_CRYPTO_CIPHER_AES_XTS:
1737 oper = regk_crypto_aes_xts;
1738 cipher_decr = req_ctx->decrypt;
1739
1740 if (variant == ARTPEC6_CRYPTO)
1741 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DSEQ;
1742 else
1743 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DSEQ;
1744 break;
1745
1746 default:
1747 pr_err("%s: Invalid cipher mode %d!\n",
1748 MODULE_NAME, ctx->crypto_type);
1749 return -EINVAL;
1750 }
1751
1752 if (variant == ARTPEC6_CRYPTO) {
1753 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER, oper);
1754 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1755 cipher_len);
1756 if (cipher_decr)
1757 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1758 } else {
1759 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER, oper);
1760 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1761 cipher_len);
1762 if (cipher_decr)
1763 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1764 }
1765
1766 ret = artpec6_crypto_setup_out_descr(common,
1767 &req_ctx->cipher_md,
1768 sizeof(req_ctx->cipher_md),
1769 false, false);
1770 if (ret)
1771 return ret;
1772
1773 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1774 if (ret)
1775 return ret;
1776
1777 if (iv_len) {
1778 ret = artpec6_crypto_setup_out_descr(common, areq->iv, iv_len,
1779 false, false);
1780 if (ret)
1781 return ret;
1782 }
1783 /* Data out */
1784 artpec6_crypto_walk_init(&walk, areq->src);
1785 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, areq->cryptlen);
1786 if (ret)
1787 return ret;
1788
1789 /* Data in */
1790 artpec6_crypto_walk_init(&walk, areq->dst);
1791 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, areq->cryptlen);
1792 if (ret)
1793 return ret;
1794
1795 /* CTR-mode padding required by the HW. */
1796 if (ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_CTR ||
1797 ctx->crypto_type == ARTPEC6_CRYPTO_CIPHER_AES_XTS) {
1798 size_t pad = ALIGN(areq->cryptlen, AES_BLOCK_SIZE) -
1799 areq->cryptlen;
1800
1801 if (pad) {
1802 ret = artpec6_crypto_setup_out_descr(common,
1803 ac->pad_buffer,
1804 pad, false, false);
1805 if (ret)
1806 return ret;
1807
1808 ret = artpec6_crypto_setup_in_descr(common,
1809 ac->pad_buffer, pad,
1810 false);
1811 if (ret)
1812 return ret;
1813 }
1814 }
1815
1816 ret = artpec6_crypto_terminate_out_descrs(common);
1817 if (ret)
1818 return ret;
1819
1820 ret = artpec6_crypto_terminate_in_descrs(common);
1821 if (ret)
1822 return ret;
1823
1824 return artpec6_crypto_dma_map_descs(common);
1825 }
1826
1827 static int artpec6_crypto_prepare_aead(struct aead_request *areq)
1828 {
1829 size_t count;
1830 int ret;
1831 size_t input_length;
1832 struct artpec6_cryptotfm_context *ctx = crypto_tfm_ctx(areq->base.tfm);
1833 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
1834 struct crypto_aead *cipher = crypto_aead_reqtfm(areq);
1835 struct artpec6_crypto_req_common *common = &req_ctx->common;
1836 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
1837 enum artpec6_crypto_variant variant = ac->variant;
1838 u32 md_cipher_len;
1839
1840 artpec6_crypto_init_dma_operation(common);
1841
1842 /* Key */
1843 if (variant == ARTPEC6_CRYPTO) {
1844 ctx->key_md = FIELD_PREP(A6_CRY_MD_OPER,
1845 a6_regk_crypto_dlkey);
1846 } else {
1847 ctx->key_md = FIELD_PREP(A7_CRY_MD_OPER,
1848 a7_regk_crypto_dlkey);
1849 }
1850 ret = artpec6_crypto_setup_out_descr(common, (void *)&ctx->key_md,
1851 sizeof(ctx->key_md), false, false);
1852 if (ret)
1853 return ret;
1854
1855 ret = artpec6_crypto_setup_out_descr(common, ctx->aes_key,
1856 ctx->key_length, true, false);
1857 if (ret)
1858 return ret;
1859
1860 req_ctx->cipher_md = 0;
1861
1862 switch (ctx->key_length) {
1863 case 16:
1864 md_cipher_len = regk_crypto_key_128;
1865 break;
1866 case 24:
1867 md_cipher_len = regk_crypto_key_192;
1868 break;
1869 case 32:
1870 md_cipher_len = regk_crypto_key_256;
1871 break;
1872 default:
1873 return -EINVAL;
1874 }
1875
1876 if (variant == ARTPEC6_CRYPTO) {
1877 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_OPER,
1878 regk_crypto_aes_gcm);
1879 req_ctx->cipher_md |= FIELD_PREP(A6_CRY_MD_CIPHER_LEN,
1880 md_cipher_len);
1881 if (req_ctx->decrypt)
1882 req_ctx->cipher_md |= A6_CRY_MD_CIPHER_DECR;
1883 } else {
1884 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_OPER,
1885 regk_crypto_aes_gcm);
1886 req_ctx->cipher_md |= FIELD_PREP(A7_CRY_MD_CIPHER_LEN,
1887 md_cipher_len);
1888 if (req_ctx->decrypt)
1889 req_ctx->cipher_md |= A7_CRY_MD_CIPHER_DECR;
1890 }
1891
1892 ret = artpec6_crypto_setup_out_descr(common,
1893 (void *) &req_ctx->cipher_md,
1894 sizeof(req_ctx->cipher_md), false,
1895 false);
1896 if (ret)
1897 return ret;
1898
1899 ret = artpec6_crypto_setup_in_descr(common, ac->pad_buffer, 4, false);
1900 if (ret)
1901 return ret;
1902
1903 /* For the decryption, cryptlen includes the tag. */
1904 input_length = areq->cryptlen;
1905 if (req_ctx->decrypt)
1906 input_length -= crypto_aead_authsize(cipher);
1907
1908 /* Prepare the context buffer */
1909 req_ctx->hw_ctx.aad_length_bits =
1910 __cpu_to_be64(8*areq->assoclen);
1911
1912 req_ctx->hw_ctx.text_length_bits =
1913 __cpu_to_be64(8*input_length);
1914
1915 memcpy(req_ctx->hw_ctx.J0, areq->iv, crypto_aead_ivsize(cipher));
1916 // The HW omits the initial increment of the counter field.
1917 memcpy(req_ctx->hw_ctx.J0 + GCM_AES_IV_SIZE, "\x00\x00\x00\x01", 4);
1918
1919 ret = artpec6_crypto_setup_out_descr(common, &req_ctx->hw_ctx,
1920 sizeof(struct artpec6_crypto_aead_hw_ctx), false, false);
1921 if (ret)
1922 return ret;
1923
1924 {
1925 struct artpec6_crypto_walk walk;
1926
1927 artpec6_crypto_walk_init(&walk, areq->src);
1928
1929 /* Associated data */
1930 count = areq->assoclen;
1931 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1932 if (ret)
1933 return ret;
1934
1935 if (!IS_ALIGNED(areq->assoclen, 16)) {
1936 size_t assoc_pad = 16 - (areq->assoclen % 16);
1937 /* The HW mandates zero padding here */
1938 ret = artpec6_crypto_setup_out_descr(common,
1939 ac->zero_buffer,
1940 assoc_pad, false,
1941 false);
1942 if (ret)
1943 return ret;
1944 }
1945
1946 /* Data to crypto */
1947 count = input_length;
1948 ret = artpec6_crypto_setup_sg_descrs_out(common, &walk, count);
1949 if (ret)
1950 return ret;
1951
1952 if (!IS_ALIGNED(input_length, 16)) {
1953 size_t crypto_pad = 16 - (input_length % 16);
1954 /* The HW mandates zero padding here */
1955 ret = artpec6_crypto_setup_out_descr(common,
1956 ac->zero_buffer,
1957 crypto_pad,
1958 false,
1959 false);
1960 if (ret)
1961 return ret;
1962 }
1963 }
1964
1965 /* Data from crypto */
1966 {
1967 struct artpec6_crypto_walk walk;
1968 size_t output_len = areq->cryptlen;
1969
1970 if (req_ctx->decrypt)
1971 output_len -= crypto_aead_authsize(cipher);
1972
1973 artpec6_crypto_walk_init(&walk, areq->dst);
1974
1975 /* skip associated data in the output */
1976 count = artpec6_crypto_walk_advance(&walk, areq->assoclen);
1977 if (count)
1978 return -EINVAL;
1979
1980 count = output_len;
1981 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk, count);
1982 if (ret)
1983 return ret;
1984
1985 /* Put padding between the cryptotext and the auth tag */
1986 if (!IS_ALIGNED(output_len, 16)) {
1987 size_t crypto_pad = 16 - (output_len % 16);
1988
1989 ret = artpec6_crypto_setup_in_descr(common,
1990 ac->pad_buffer,
1991 crypto_pad, false);
1992 if (ret)
1993 return ret;
1994 }
1995
1996 /* The authentication tag shall follow immediately after
1997 * the output ciphertext. For decryption it is put in a context
1998 * buffer for later compare against the input tag.
1999 */
2000
2001 if (req_ctx->decrypt) {
2002 ret = artpec6_crypto_setup_in_descr(common,
2003 req_ctx->decryption_tag, AES_BLOCK_SIZE, false);
2004 if (ret)
2005 return ret;
2006
2007 } else {
2008 /* For encryption the requested tag size may be smaller
2009 * than the hardware's generated tag.
2010 */
2011 size_t authsize = crypto_aead_authsize(cipher);
2012
2013 ret = artpec6_crypto_setup_sg_descrs_in(common, &walk,
2014 authsize);
2015 if (ret)
2016 return ret;
2017
2018 if (authsize < AES_BLOCK_SIZE) {
2019 count = AES_BLOCK_SIZE - authsize;
2020 ret = artpec6_crypto_setup_in_descr(common,
2021 ac->pad_buffer,
2022 count, false);
2023 if (ret)
2024 return ret;
2025 }
2026 }
2027
2028 }
2029
2030 ret = artpec6_crypto_terminate_in_descrs(common);
2031 if (ret)
2032 return ret;
2033
2034 ret = artpec6_crypto_terminate_out_descrs(common);
2035 if (ret)
2036 return ret;
2037
2038 return artpec6_crypto_dma_map_descs(common);
2039 }
2040
2041 static void artpec6_crypto_process_queue(struct artpec6_crypto *ac,
2042 struct list_head *completions)
2043 {
2044 struct artpec6_crypto_req_common *req;
2045
2046 while (!list_empty(&ac->queue) && !artpec6_crypto_busy()) {
2047 req = list_first_entry(&ac->queue,
2048 struct artpec6_crypto_req_common,
2049 list);
2050 list_move_tail(&req->list, &ac->pending);
2051 artpec6_crypto_start_dma(req);
2052
2053 list_add_tail(&req->complete_in_progress, completions);
2054 }
2055
2056 /*
2057 * In some cases, the hardware can raise an in_eop_flush interrupt
2058 * before actually updating the status, so we have an timer which will
2059 * recheck the status on timeout. Since the cases are expected to be
2060 * very rare, we use a relatively large timeout value. There should be
2061 * no noticeable negative effect if we timeout spuriously.
2062 */
2063 if (ac->pending_count)
2064 mod_timer(&ac->timer, jiffies + msecs_to_jiffies(100));
2065 else
2066 del_timer(&ac->timer);
2067 }
2068
2069 static void artpec6_crypto_timeout(struct timer_list *t)
2070 {
2071 struct artpec6_crypto *ac = from_timer(ac, t, timer);
2072
2073 dev_info_ratelimited(artpec6_crypto_dev, "timeout\n");
2074
2075 tasklet_schedule(&ac->task);
2076 }
2077
2078 static void artpec6_crypto_task(unsigned long data)
2079 {
2080 struct artpec6_crypto *ac = (struct artpec6_crypto *)data;
2081 struct artpec6_crypto_req_common *req;
2082 struct artpec6_crypto_req_common *n;
2083 struct list_head complete_done;
2084 struct list_head complete_in_progress;
2085
2086 INIT_LIST_HEAD(&complete_done);
2087 INIT_LIST_HEAD(&complete_in_progress);
2088
2089 if (list_empty(&ac->pending)) {
2090 pr_debug("Spurious IRQ\n");
2091 return;
2092 }
2093
2094 spin_lock_bh(&ac->queue_lock);
2095
2096 list_for_each_entry_safe(req, n, &ac->pending, list) {
2097 struct artpec6_crypto_dma_descriptors *dma = req->dma;
2098 u32 stat;
2099 dma_addr_t stataddr;
2100
2101 stataddr = dma->stat_dma_addr + 4 * (req->dma->in_cnt - 1);
2102 dma_sync_single_for_cpu(artpec6_crypto_dev,
2103 stataddr,
2104 4,
2105 DMA_BIDIRECTIONAL);
2106
2107 stat = req->dma->stat[req->dma->in_cnt-1];
2108
2109 /* A non-zero final status descriptor indicates
2110 * this job has finished.
2111 */
2112 pr_debug("Request %p status is %X\n", req, stat);
2113 if (!stat)
2114 break;
2115
2116 /* Allow testing of timeout handling with fault injection */
2117 #ifdef CONFIG_FAULT_INJECTION
2118 if (should_fail(&artpec6_crypto_fail_status_read, 1))
2119 continue;
2120 #endif
2121
2122 pr_debug("Completing request %p\n", req);
2123
2124 list_move_tail(&req->list, &complete_done);
2125
2126 ac->pending_count--;
2127 }
2128
2129 artpec6_crypto_process_queue(ac, &complete_in_progress);
2130
2131 spin_unlock_bh(&ac->queue_lock);
2132
2133 /* Perform the completion callbacks without holding the queue lock
2134 * to allow new request submissions from the callbacks.
2135 */
2136 list_for_each_entry_safe(req, n, &complete_done, list) {
2137 artpec6_crypto_dma_unmap_all(req);
2138 artpec6_crypto_copy_bounce_buffers(req);
2139 artpec6_crypto_common_destroy(req);
2140
2141 req->complete(req->req);
2142 }
2143
2144 list_for_each_entry_safe(req, n, &complete_in_progress,
2145 complete_in_progress) {
2146 req->req->complete(req->req, -EINPROGRESS);
2147 }
2148 }
2149
2150 static void artpec6_crypto_complete_crypto(struct crypto_async_request *req)
2151 {
2152 req->complete(req, 0);
2153 }
2154
2155 static void
2156 artpec6_crypto_complete_cbc_decrypt(struct crypto_async_request *req)
2157 {
2158 struct skcipher_request *cipher_req = container_of(req,
2159 struct skcipher_request, base);
2160
2161 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->src,
2162 cipher_req->cryptlen - AES_BLOCK_SIZE,
2163 AES_BLOCK_SIZE, 0);
2164 req->complete(req, 0);
2165 }
2166
2167 static void
2168 artpec6_crypto_complete_cbc_encrypt(struct crypto_async_request *req)
2169 {
2170 struct skcipher_request *cipher_req = container_of(req,
2171 struct skcipher_request, base);
2172
2173 scatterwalk_map_and_copy(cipher_req->iv, cipher_req->dst,
2174 cipher_req->cryptlen - AES_BLOCK_SIZE,
2175 AES_BLOCK_SIZE, 0);
2176 req->complete(req, 0);
2177 }
2178
2179 static void artpec6_crypto_complete_aead(struct crypto_async_request *req)
2180 {
2181 int result = 0;
2182
2183 /* Verify GCM hashtag. */
2184 struct aead_request *areq = container_of(req,
2185 struct aead_request, base);
2186 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
2187 struct artpec6_crypto_aead_req_ctx *req_ctx = aead_request_ctx(areq);
2188
2189 if (req_ctx->decrypt) {
2190 u8 input_tag[AES_BLOCK_SIZE];
2191 unsigned int authsize = crypto_aead_authsize(aead);
2192
2193 sg_pcopy_to_buffer(areq->src,
2194 sg_nents(areq->src),
2195 input_tag,
2196 authsize,
2197 areq->assoclen + areq->cryptlen -
2198 authsize);
2199
2200 if (crypto_memneq(req_ctx->decryption_tag,
2201 input_tag,
2202 authsize)) {
2203 pr_debug("***EBADMSG:\n");
2204 print_hex_dump_debug("ref:", DUMP_PREFIX_ADDRESS, 32, 1,
2205 input_tag, authsize, true);
2206 print_hex_dump_debug("out:", DUMP_PREFIX_ADDRESS, 32, 1,
2207 req_ctx->decryption_tag,
2208 authsize, true);
2209
2210 result = -EBADMSG;
2211 }
2212 }
2213
2214 req->complete(req, result);
2215 }
2216
2217 static void artpec6_crypto_complete_hash(struct crypto_async_request *req)
2218 {
2219 req->complete(req, 0);
2220 }
2221
2222
2223 /*------------------- Hash functions -----------------------------------------*/
2224 static int
2225 artpec6_crypto_hash_set_key(struct crypto_ahash *tfm,
2226 const u8 *key, unsigned int keylen)
2227 {
2228 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(&tfm->base);
2229 size_t blocksize;
2230 int ret;
2231
2232 if (!keylen) {
2233 pr_err("Invalid length (%d) of HMAC key\n",
2234 keylen);
2235 return -EINVAL;
2236 }
2237
2238 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2239
2240 blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2241
2242 if (keylen > blocksize) {
2243 tfm_ctx->hmac_key_length = blocksize;
2244
2245 ret = crypto_shash_tfm_digest(tfm_ctx->child_hash, key, keylen,
2246 tfm_ctx->hmac_key);
2247 if (ret)
2248 return ret;
2249 } else {
2250 memcpy(tfm_ctx->hmac_key, key, keylen);
2251 tfm_ctx->hmac_key_length = keylen;
2252 }
2253
2254 return 0;
2255 }
2256
2257 static int
2258 artpec6_crypto_init_hash(struct ahash_request *req, u8 type, int hmac)
2259 {
2260 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2261 enum artpec6_crypto_variant variant = ac->variant;
2262 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2263 u32 oper;
2264
2265 memset(req_ctx, 0, sizeof(*req_ctx));
2266
2267 req_ctx->hash_flags = HASH_FLAG_INIT_CTX;
2268 if (hmac)
2269 req_ctx->hash_flags |= (HASH_FLAG_HMAC | HASH_FLAG_UPDATE_KEY);
2270
2271 switch (type) {
2272 case ARTPEC6_CRYPTO_HASH_SHA1:
2273 oper = hmac ? regk_crypto_hmac_sha1 : regk_crypto_sha1;
2274 break;
2275 case ARTPEC6_CRYPTO_HASH_SHA256:
2276 oper = hmac ? regk_crypto_hmac_sha256 : regk_crypto_sha256;
2277 break;
2278 default:
2279 pr_err("%s: Unsupported hash type 0x%x\n", MODULE_NAME, type);
2280 return -EINVAL;
2281 }
2282
2283 if (variant == ARTPEC6_CRYPTO)
2284 req_ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, oper);
2285 else
2286 req_ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, oper);
2287
2288 return 0;
2289 }
2290
2291 static int artpec6_crypto_prepare_submit_hash(struct ahash_request *req)
2292 {
2293 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2294 int ret;
2295
2296 if (!req_ctx->common.dma) {
2297 ret = artpec6_crypto_common_init(&req_ctx->common,
2298 &req->base,
2299 artpec6_crypto_complete_hash,
2300 NULL, 0);
2301
2302 if (ret)
2303 return ret;
2304 }
2305
2306 ret = artpec6_crypto_prepare_hash(req);
2307 switch (ret) {
2308 case ARTPEC6_CRYPTO_PREPARE_HASH_START:
2309 ret = artpec6_crypto_submit(&req_ctx->common);
2310 break;
2311
2312 case ARTPEC6_CRYPTO_PREPARE_HASH_NO_START:
2313 ret = 0;
2314 fallthrough;
2315
2316 default:
2317 artpec6_crypto_common_destroy(&req_ctx->common);
2318 break;
2319 }
2320
2321 return ret;
2322 }
2323
2324 static int artpec6_crypto_hash_final(struct ahash_request *req)
2325 {
2326 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2327
2328 req_ctx->hash_flags |= HASH_FLAG_FINALIZE;
2329
2330 return artpec6_crypto_prepare_submit_hash(req);
2331 }
2332
2333 static int artpec6_crypto_hash_update(struct ahash_request *req)
2334 {
2335 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2336
2337 req_ctx->hash_flags |= HASH_FLAG_UPDATE;
2338
2339 return artpec6_crypto_prepare_submit_hash(req);
2340 }
2341
2342 static int artpec6_crypto_sha1_init(struct ahash_request *req)
2343 {
2344 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2345 }
2346
2347 static int artpec6_crypto_sha1_digest(struct ahash_request *req)
2348 {
2349 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2350
2351 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA1, 0);
2352
2353 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2354
2355 return artpec6_crypto_prepare_submit_hash(req);
2356 }
2357
2358 static int artpec6_crypto_sha256_init(struct ahash_request *req)
2359 {
2360 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2361 }
2362
2363 static int artpec6_crypto_sha256_digest(struct ahash_request *req)
2364 {
2365 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2366
2367 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 0);
2368 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2369
2370 return artpec6_crypto_prepare_submit_hash(req);
2371 }
2372
2373 static int artpec6_crypto_hmac_sha256_init(struct ahash_request *req)
2374 {
2375 return artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2376 }
2377
2378 static int artpec6_crypto_hmac_sha256_digest(struct ahash_request *req)
2379 {
2380 struct artpec6_hash_request_context *req_ctx = ahash_request_ctx(req);
2381
2382 artpec6_crypto_init_hash(req, ARTPEC6_CRYPTO_HASH_SHA256, 1);
2383 req_ctx->hash_flags |= HASH_FLAG_UPDATE | HASH_FLAG_FINALIZE;
2384
2385 return artpec6_crypto_prepare_submit_hash(req);
2386 }
2387
2388 static int artpec6_crypto_ahash_init_common(struct crypto_tfm *tfm,
2389 const char *base_hash_name)
2390 {
2391 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2392
2393 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2394 sizeof(struct artpec6_hash_request_context));
2395 memset(tfm_ctx, 0, sizeof(*tfm_ctx));
2396
2397 if (base_hash_name) {
2398 struct crypto_shash *child;
2399
2400 child = crypto_alloc_shash(base_hash_name, 0,
2401 CRYPTO_ALG_NEED_FALLBACK);
2402
2403 if (IS_ERR(child))
2404 return PTR_ERR(child);
2405
2406 tfm_ctx->child_hash = child;
2407 }
2408
2409 return 0;
2410 }
2411
2412 static int artpec6_crypto_ahash_init(struct crypto_tfm *tfm)
2413 {
2414 return artpec6_crypto_ahash_init_common(tfm, NULL);
2415 }
2416
2417 static int artpec6_crypto_ahash_init_hmac_sha256(struct crypto_tfm *tfm)
2418 {
2419 return artpec6_crypto_ahash_init_common(tfm, "sha256");
2420 }
2421
2422 static void artpec6_crypto_ahash_exit(struct crypto_tfm *tfm)
2423 {
2424 struct artpec6_hashalg_context *tfm_ctx = crypto_tfm_ctx(tfm);
2425
2426 if (tfm_ctx->child_hash)
2427 crypto_free_shash(tfm_ctx->child_hash);
2428
2429 memset(tfm_ctx->hmac_key, 0, sizeof(tfm_ctx->hmac_key));
2430 tfm_ctx->hmac_key_length = 0;
2431 }
2432
2433 static int artpec6_crypto_hash_export(struct ahash_request *req, void *out)
2434 {
2435 const struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2436 struct artpec6_hash_export_state *state = out;
2437 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2438 enum artpec6_crypto_variant variant = ac->variant;
2439
2440 BUILD_BUG_ON(sizeof(state->partial_buffer) !=
2441 sizeof(ctx->partial_buffer));
2442 BUILD_BUG_ON(sizeof(state->digeststate) != sizeof(ctx->digeststate));
2443
2444 state->digcnt = ctx->digcnt;
2445 state->partial_bytes = ctx->partial_bytes;
2446 state->hash_flags = ctx->hash_flags;
2447
2448 if (variant == ARTPEC6_CRYPTO)
2449 state->oper = FIELD_GET(A6_CRY_MD_OPER, ctx->hash_md);
2450 else
2451 state->oper = FIELD_GET(A7_CRY_MD_OPER, ctx->hash_md);
2452
2453 memcpy(state->partial_buffer, ctx->partial_buffer,
2454 sizeof(state->partial_buffer));
2455 memcpy(state->digeststate, ctx->digeststate,
2456 sizeof(state->digeststate));
2457
2458 return 0;
2459 }
2460
2461 static int artpec6_crypto_hash_import(struct ahash_request *req, const void *in)
2462 {
2463 struct artpec6_hash_request_context *ctx = ahash_request_ctx(req);
2464 const struct artpec6_hash_export_state *state = in;
2465 struct artpec6_crypto *ac = dev_get_drvdata(artpec6_crypto_dev);
2466 enum artpec6_crypto_variant variant = ac->variant;
2467
2468 memset(ctx, 0, sizeof(*ctx));
2469
2470 ctx->digcnt = state->digcnt;
2471 ctx->partial_bytes = state->partial_bytes;
2472 ctx->hash_flags = state->hash_flags;
2473
2474 if (variant == ARTPEC6_CRYPTO)
2475 ctx->hash_md = FIELD_PREP(A6_CRY_MD_OPER, state->oper);
2476 else
2477 ctx->hash_md = FIELD_PREP(A7_CRY_MD_OPER, state->oper);
2478
2479 memcpy(ctx->partial_buffer, state->partial_buffer,
2480 sizeof(state->partial_buffer));
2481 memcpy(ctx->digeststate, state->digeststate,
2482 sizeof(state->digeststate));
2483
2484 return 0;
2485 }
2486
2487 static int init_crypto_hw(struct artpec6_crypto *ac)
2488 {
2489 enum artpec6_crypto_variant variant = ac->variant;
2490 void __iomem *base = ac->base;
2491 u32 out_descr_buf_size;
2492 u32 out_data_buf_size;
2493 u32 in_data_buf_size;
2494 u32 in_descr_buf_size;
2495 u32 in_stat_buf_size;
2496 u32 in, out;
2497
2498 /*
2499 * The PDMA unit contains 1984 bytes of internal memory for the OUT
2500 * channels and 1024 bytes for the IN channel. This is an elastic
2501 * memory used to internally store the descriptors and data. The values
2502 * ares specified in 64 byte incremements. Trustzone buffers are not
2503 * used at this stage.
2504 */
2505 out_data_buf_size = 16; /* 1024 bytes for data */
2506 out_descr_buf_size = 15; /* 960 bytes for descriptors */
2507 in_data_buf_size = 8; /* 512 bytes for data */
2508 in_descr_buf_size = 4; /* 256 bytes for descriptors */
2509 in_stat_buf_size = 4; /* 256 bytes for stat descrs */
2510
2511 BUILD_BUG_ON_MSG((out_data_buf_size
2512 + out_descr_buf_size) * 64 > 1984,
2513 "Invalid OUT configuration");
2514
2515 BUILD_BUG_ON_MSG((in_data_buf_size
2516 + in_descr_buf_size
2517 + in_stat_buf_size) * 64 > 1024,
2518 "Invalid IN configuration");
2519
2520 in = FIELD_PREP(PDMA_IN_BUF_CFG_DATA_BUF_SIZE, in_data_buf_size) |
2521 FIELD_PREP(PDMA_IN_BUF_CFG_DESCR_BUF_SIZE, in_descr_buf_size) |
2522 FIELD_PREP(PDMA_IN_BUF_CFG_STAT_BUF_SIZE, in_stat_buf_size);
2523
2524 out = FIELD_PREP(PDMA_OUT_BUF_CFG_DATA_BUF_SIZE, out_data_buf_size) |
2525 FIELD_PREP(PDMA_OUT_BUF_CFG_DESCR_BUF_SIZE, out_descr_buf_size);
2526
2527 writel_relaxed(out, base + PDMA_OUT_BUF_CFG);
2528 writel_relaxed(PDMA_OUT_CFG_EN, base + PDMA_OUT_CFG);
2529
2530 if (variant == ARTPEC6_CRYPTO) {
2531 writel_relaxed(in, base + A6_PDMA_IN_BUF_CFG);
2532 writel_relaxed(PDMA_IN_CFG_EN, base + A6_PDMA_IN_CFG);
2533 writel_relaxed(A6_PDMA_INTR_MASK_IN_DATA |
2534 A6_PDMA_INTR_MASK_IN_EOP_FLUSH,
2535 base + A6_PDMA_INTR_MASK);
2536 } else {
2537 writel_relaxed(in, base + A7_PDMA_IN_BUF_CFG);
2538 writel_relaxed(PDMA_IN_CFG_EN, base + A7_PDMA_IN_CFG);
2539 writel_relaxed(A7_PDMA_INTR_MASK_IN_DATA |
2540 A7_PDMA_INTR_MASK_IN_EOP_FLUSH,
2541 base + A7_PDMA_INTR_MASK);
2542 }
2543
2544 return 0;
2545 }
2546
2547 static void artpec6_crypto_disable_hw(struct artpec6_crypto *ac)
2548 {
2549 enum artpec6_crypto_variant variant = ac->variant;
2550 void __iomem *base = ac->base;
2551
2552 if (variant == ARTPEC6_CRYPTO) {
2553 writel_relaxed(A6_PDMA_IN_CMD_STOP, base + A6_PDMA_IN_CMD);
2554 writel_relaxed(0, base + A6_PDMA_IN_CFG);
2555 writel_relaxed(A6_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2556 } else {
2557 writel_relaxed(A7_PDMA_IN_CMD_STOP, base + A7_PDMA_IN_CMD);
2558 writel_relaxed(0, base + A7_PDMA_IN_CFG);
2559 writel_relaxed(A7_PDMA_OUT_CMD_STOP, base + PDMA_OUT_CMD);
2560 }
2561
2562 writel_relaxed(0, base + PDMA_OUT_CFG);
2563
2564 }
2565
2566 static irqreturn_t artpec6_crypto_irq(int irq, void *dev_id)
2567 {
2568 struct artpec6_crypto *ac = dev_id;
2569 enum artpec6_crypto_variant variant = ac->variant;
2570 void __iomem *base = ac->base;
2571 u32 mask_in_data, mask_in_eop_flush;
2572 u32 in_cmd_flush_stat, in_cmd_reg;
2573 u32 ack_intr_reg;
2574 u32 ack = 0;
2575 u32 intr;
2576
2577 if (variant == ARTPEC6_CRYPTO) {
2578 intr = readl_relaxed(base + A6_PDMA_MASKED_INTR);
2579 mask_in_data = A6_PDMA_INTR_MASK_IN_DATA;
2580 mask_in_eop_flush = A6_PDMA_INTR_MASK_IN_EOP_FLUSH;
2581 in_cmd_flush_stat = A6_PDMA_IN_CMD_FLUSH_STAT;
2582 in_cmd_reg = A6_PDMA_IN_CMD;
2583 ack_intr_reg = A6_PDMA_ACK_INTR;
2584 } else {
2585 intr = readl_relaxed(base + A7_PDMA_MASKED_INTR);
2586 mask_in_data = A7_PDMA_INTR_MASK_IN_DATA;
2587 mask_in_eop_flush = A7_PDMA_INTR_MASK_IN_EOP_FLUSH;
2588 in_cmd_flush_stat = A7_PDMA_IN_CMD_FLUSH_STAT;
2589 in_cmd_reg = A7_PDMA_IN_CMD;
2590 ack_intr_reg = A7_PDMA_ACK_INTR;
2591 }
2592
2593 /* We get two interrupt notifications from each job.
2594 * The in_data means all data was sent to memory and then
2595 * we request a status flush command to write the per-job
2596 * status to its status vector. This ensures that the
2597 * tasklet can detect exactly how many submitted jobs
2598 * that have finished.
2599 */
2600 if (intr & mask_in_data)
2601 ack |= mask_in_data;
2602
2603 if (intr & mask_in_eop_flush)
2604 ack |= mask_in_eop_flush;
2605 else
2606 writel_relaxed(in_cmd_flush_stat, base + in_cmd_reg);
2607
2608 writel_relaxed(ack, base + ack_intr_reg);
2609
2610 if (intr & mask_in_eop_flush)
2611 tasklet_schedule(&ac->task);
2612
2613 return IRQ_HANDLED;
2614 }
2615
2616 /*------------------- Algorithm definitions ----------------------------------*/
2617
2618 /* Hashes */
2619 static struct ahash_alg hash_algos[] = {
2620 /* SHA-1 */
2621 {
2622 .init = artpec6_crypto_sha1_init,
2623 .update = artpec6_crypto_hash_update,
2624 .final = artpec6_crypto_hash_final,
2625 .digest = artpec6_crypto_sha1_digest,
2626 .import = artpec6_crypto_hash_import,
2627 .export = artpec6_crypto_hash_export,
2628 .halg.digestsize = SHA1_DIGEST_SIZE,
2629 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2630 .halg.base = {
2631 .cra_name = "sha1",
2632 .cra_driver_name = "artpec-sha1",
2633 .cra_priority = 300,
2634 .cra_flags = CRYPTO_ALG_ASYNC |
2635 CRYPTO_ALG_ALLOCATES_MEMORY,
2636 .cra_blocksize = SHA1_BLOCK_SIZE,
2637 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2638 .cra_alignmask = 3,
2639 .cra_module = THIS_MODULE,
2640 .cra_init = artpec6_crypto_ahash_init,
2641 .cra_exit = artpec6_crypto_ahash_exit,
2642 }
2643 },
2644 /* SHA-256 */
2645 {
2646 .init = artpec6_crypto_sha256_init,
2647 .update = artpec6_crypto_hash_update,
2648 .final = artpec6_crypto_hash_final,
2649 .digest = artpec6_crypto_sha256_digest,
2650 .import = artpec6_crypto_hash_import,
2651 .export = artpec6_crypto_hash_export,
2652 .halg.digestsize = SHA256_DIGEST_SIZE,
2653 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2654 .halg.base = {
2655 .cra_name = "sha256",
2656 .cra_driver_name = "artpec-sha256",
2657 .cra_priority = 300,
2658 .cra_flags = CRYPTO_ALG_ASYNC |
2659 CRYPTO_ALG_ALLOCATES_MEMORY,
2660 .cra_blocksize = SHA256_BLOCK_SIZE,
2661 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2662 .cra_alignmask = 3,
2663 .cra_module = THIS_MODULE,
2664 .cra_init = artpec6_crypto_ahash_init,
2665 .cra_exit = artpec6_crypto_ahash_exit,
2666 }
2667 },
2668 /* HMAC SHA-256 */
2669 {
2670 .init = artpec6_crypto_hmac_sha256_init,
2671 .update = artpec6_crypto_hash_update,
2672 .final = artpec6_crypto_hash_final,
2673 .digest = artpec6_crypto_hmac_sha256_digest,
2674 .import = artpec6_crypto_hash_import,
2675 .export = artpec6_crypto_hash_export,
2676 .setkey = artpec6_crypto_hash_set_key,
2677 .halg.digestsize = SHA256_DIGEST_SIZE,
2678 .halg.statesize = sizeof(struct artpec6_hash_export_state),
2679 .halg.base = {
2680 .cra_name = "hmac(sha256)",
2681 .cra_driver_name = "artpec-hmac-sha256",
2682 .cra_priority = 300,
2683 .cra_flags = CRYPTO_ALG_ASYNC |
2684 CRYPTO_ALG_ALLOCATES_MEMORY,
2685 .cra_blocksize = SHA256_BLOCK_SIZE,
2686 .cra_ctxsize = sizeof(struct artpec6_hashalg_context),
2687 .cra_alignmask = 3,
2688 .cra_module = THIS_MODULE,
2689 .cra_init = artpec6_crypto_ahash_init_hmac_sha256,
2690 .cra_exit = artpec6_crypto_ahash_exit,
2691 }
2692 },
2693 };
2694
2695 /* Crypto */
2696 static struct skcipher_alg crypto_algos[] = {
2697 /* AES - ECB */
2698 {
2699 .base = {
2700 .cra_name = "ecb(aes)",
2701 .cra_driver_name = "artpec6-ecb-aes",
2702 .cra_priority = 300,
2703 .cra_flags = CRYPTO_ALG_ASYNC |
2704 CRYPTO_ALG_ALLOCATES_MEMORY,
2705 .cra_blocksize = AES_BLOCK_SIZE,
2706 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2707 .cra_alignmask = 3,
2708 .cra_module = THIS_MODULE,
2709 },
2710 .min_keysize = AES_MIN_KEY_SIZE,
2711 .max_keysize = AES_MAX_KEY_SIZE,
2712 .setkey = artpec6_crypto_cipher_set_key,
2713 .encrypt = artpec6_crypto_encrypt,
2714 .decrypt = artpec6_crypto_decrypt,
2715 .init = artpec6_crypto_aes_ecb_init,
2716 .exit = artpec6_crypto_aes_exit,
2717 },
2718 /* AES - CTR */
2719 {
2720 .base = {
2721 .cra_name = "ctr(aes)",
2722 .cra_driver_name = "artpec6-ctr-aes",
2723 .cra_priority = 300,
2724 .cra_flags = CRYPTO_ALG_ASYNC |
2725 CRYPTO_ALG_ALLOCATES_MEMORY |
2726 CRYPTO_ALG_NEED_FALLBACK,
2727 .cra_blocksize = 1,
2728 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2729 .cra_alignmask = 3,
2730 .cra_module = THIS_MODULE,
2731 },
2732 .min_keysize = AES_MIN_KEY_SIZE,
2733 .max_keysize = AES_MAX_KEY_SIZE,
2734 .ivsize = AES_BLOCK_SIZE,
2735 .setkey = artpec6_crypto_cipher_set_key,
2736 .encrypt = artpec6_crypto_ctr_encrypt,
2737 .decrypt = artpec6_crypto_ctr_decrypt,
2738 .init = artpec6_crypto_aes_ctr_init,
2739 .exit = artpec6_crypto_aes_ctr_exit,
2740 },
2741 /* AES - CBC */
2742 {
2743 .base = {
2744 .cra_name = "cbc(aes)",
2745 .cra_driver_name = "artpec6-cbc-aes",
2746 .cra_priority = 300,
2747 .cra_flags = CRYPTO_ALG_ASYNC |
2748 CRYPTO_ALG_ALLOCATES_MEMORY,
2749 .cra_blocksize = AES_BLOCK_SIZE,
2750 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2751 .cra_alignmask = 3,
2752 .cra_module = THIS_MODULE,
2753 },
2754 .min_keysize = AES_MIN_KEY_SIZE,
2755 .max_keysize = AES_MAX_KEY_SIZE,
2756 .ivsize = AES_BLOCK_SIZE,
2757 .setkey = artpec6_crypto_cipher_set_key,
2758 .encrypt = artpec6_crypto_encrypt,
2759 .decrypt = artpec6_crypto_decrypt,
2760 .init = artpec6_crypto_aes_cbc_init,
2761 .exit = artpec6_crypto_aes_exit
2762 },
2763 /* AES - XTS */
2764 {
2765 .base = {
2766 .cra_name = "xts(aes)",
2767 .cra_driver_name = "artpec6-xts-aes",
2768 .cra_priority = 300,
2769 .cra_flags = CRYPTO_ALG_ASYNC |
2770 CRYPTO_ALG_ALLOCATES_MEMORY,
2771 .cra_blocksize = 1,
2772 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2773 .cra_alignmask = 3,
2774 .cra_module = THIS_MODULE,
2775 },
2776 .min_keysize = 2*AES_MIN_KEY_SIZE,
2777 .max_keysize = 2*AES_MAX_KEY_SIZE,
2778 .ivsize = 16,
2779 .setkey = artpec6_crypto_xts_set_key,
2780 .encrypt = artpec6_crypto_encrypt,
2781 .decrypt = artpec6_crypto_decrypt,
2782 .init = artpec6_crypto_aes_xts_init,
2783 .exit = artpec6_crypto_aes_exit,
2784 },
2785 };
2786
2787 static struct aead_alg aead_algos[] = {
2788 {
2789 .init = artpec6_crypto_aead_init,
2790 .setkey = artpec6_crypto_aead_set_key,
2791 .encrypt = artpec6_crypto_aead_encrypt,
2792 .decrypt = artpec6_crypto_aead_decrypt,
2793 .ivsize = GCM_AES_IV_SIZE,
2794 .maxauthsize = AES_BLOCK_SIZE,
2795
2796 .base = {
2797 .cra_name = "gcm(aes)",
2798 .cra_driver_name = "artpec-gcm-aes",
2799 .cra_priority = 300,
2800 .cra_flags = CRYPTO_ALG_ASYNC |
2801 CRYPTO_ALG_ALLOCATES_MEMORY |
2802 CRYPTO_ALG_KERN_DRIVER_ONLY,
2803 .cra_blocksize = 1,
2804 .cra_ctxsize = sizeof(struct artpec6_cryptotfm_context),
2805 .cra_alignmask = 3,
2806 .cra_module = THIS_MODULE,
2807 },
2808 }
2809 };
2810
2811 #ifdef CONFIG_DEBUG_FS
2812
2813 struct dbgfs_u32 {
2814 char *name;
2815 mode_t mode;
2816 u32 *flag;
2817 char *desc;
2818 };
2819
2820 static struct dentry *dbgfs_root;
2821
2822 static void artpec6_crypto_init_debugfs(void)
2823 {
2824 dbgfs_root = debugfs_create_dir("artpec6_crypto", NULL);
2825
2826 #ifdef CONFIG_FAULT_INJECTION
2827 fault_create_debugfs_attr("fail_status_read", dbgfs_root,
2828 &artpec6_crypto_fail_status_read);
2829
2830 fault_create_debugfs_attr("fail_dma_array_full", dbgfs_root,
2831 &artpec6_crypto_fail_dma_array_full);
2832 #endif
2833 }
2834
2835 static void artpec6_crypto_free_debugfs(void)
2836 {
2837 debugfs_remove_recursive(dbgfs_root);
2838 dbgfs_root = NULL;
2839 }
2840 #endif
2841
2842 static const struct of_device_id artpec6_crypto_of_match[] = {
2843 { .compatible = "axis,artpec6-crypto", .data = (void *)ARTPEC6_CRYPTO },
2844 { .compatible = "axis,artpec7-crypto", .data = (void *)ARTPEC7_CRYPTO },
2845 {}
2846 };
2847 MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match);
2848
2849 static int artpec6_crypto_probe(struct platform_device *pdev)
2850 {
2851 const struct of_device_id *match;
2852 enum artpec6_crypto_variant variant;
2853 struct artpec6_crypto *ac;
2854 struct device *dev = &pdev->dev;
2855 void __iomem *base;
2856 int irq;
2857 int err;
2858
2859 if (artpec6_crypto_dev)
2860 return -ENODEV;
2861
2862 match = of_match_node(artpec6_crypto_of_match, dev->of_node);
2863 if (!match)
2864 return -EINVAL;
2865
2866 variant = (enum artpec6_crypto_variant)match->data;
2867
2868 base = devm_platform_ioremap_resource(pdev, 0);
2869 if (IS_ERR(base))
2870 return PTR_ERR(base);
2871
2872 irq = platform_get_irq(pdev, 0);
2873 if (irq < 0)
2874 return -ENODEV;
2875
2876 ac = devm_kzalloc(&pdev->dev, sizeof(struct artpec6_crypto),
2877 GFP_KERNEL);
2878 if (!ac)
2879 return -ENOMEM;
2880
2881 platform_set_drvdata(pdev, ac);
2882 ac->variant = variant;
2883
2884 spin_lock_init(&ac->queue_lock);
2885 INIT_LIST_HEAD(&ac->queue);
2886 INIT_LIST_HEAD(&ac->pending);
2887 timer_setup(&ac->timer, artpec6_crypto_timeout, 0);
2888
2889 ac->base = base;
2890
2891 ac->dma_cache = kmem_cache_create("artpec6_crypto_dma",
2892 sizeof(struct artpec6_crypto_dma_descriptors),
2893 64,
2894 0,
2895 NULL);
2896 if (!ac->dma_cache)
2897 return -ENOMEM;
2898
2899 #ifdef CONFIG_DEBUG_FS
2900 artpec6_crypto_init_debugfs();
2901 #endif
2902
2903 tasklet_init(&ac->task, artpec6_crypto_task,
2904 (unsigned long)ac);
2905
2906 ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
2907 GFP_KERNEL);
2908 if (!ac->pad_buffer)
2909 return -ENOMEM;
2910 ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX);
2911
2912 ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX,
2913 GFP_KERNEL);
2914 if (!ac->zero_buffer)
2915 return -ENOMEM;
2916 ac->zero_buffer = PTR_ALIGN(ac->zero_buffer, ARTPEC_CACHE_LINE_MAX);
2917
2918 err = init_crypto_hw(ac);
2919 if (err)
2920 goto free_cache;
2921
2922 err = devm_request_irq(&pdev->dev, irq, artpec6_crypto_irq, 0,
2923 "artpec6-crypto", ac);
2924 if (err)
2925 goto disable_hw;
2926
2927 artpec6_crypto_dev = &pdev->dev;
2928
2929 err = crypto_register_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
2930 if (err) {
2931 dev_err(dev, "Failed to register ahashes\n");
2932 goto disable_hw;
2933 }
2934
2935 err = crypto_register_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
2936 if (err) {
2937 dev_err(dev, "Failed to register ciphers\n");
2938 goto unregister_ahashes;
2939 }
2940
2941 err = crypto_register_aeads(aead_algos, ARRAY_SIZE(aead_algos));
2942 if (err) {
2943 dev_err(dev, "Failed to register aeads\n");
2944 goto unregister_algs;
2945 }
2946
2947 return 0;
2948
2949 unregister_algs:
2950 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
2951 unregister_ahashes:
2952 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
2953 disable_hw:
2954 artpec6_crypto_disable_hw(ac);
2955 free_cache:
2956 kmem_cache_destroy(ac->dma_cache);
2957 return err;
2958 }
2959
2960 static int artpec6_crypto_remove(struct platform_device *pdev)
2961 {
2962 struct artpec6_crypto *ac = platform_get_drvdata(pdev);
2963 int irq = platform_get_irq(pdev, 0);
2964
2965 crypto_unregister_ahashes(hash_algos, ARRAY_SIZE(hash_algos));
2966 crypto_unregister_skciphers(crypto_algos, ARRAY_SIZE(crypto_algos));
2967 crypto_unregister_aeads(aead_algos, ARRAY_SIZE(aead_algos));
2968
2969 tasklet_disable(&ac->task);
2970 devm_free_irq(&pdev->dev, irq, ac);
2971 tasklet_kill(&ac->task);
2972 del_timer_sync(&ac->timer);
2973
2974 artpec6_crypto_disable_hw(ac);
2975
2976 kmem_cache_destroy(ac->dma_cache);
2977 #ifdef CONFIG_DEBUG_FS
2978 artpec6_crypto_free_debugfs();
2979 #endif
2980 return 0;
2981 }
2982
2983 static struct platform_driver artpec6_crypto_driver = {
2984 .probe = artpec6_crypto_probe,
2985 .remove = artpec6_crypto_remove,
2986 .driver = {
2987 .name = "artpec6-crypto",
2988 .of_match_table = artpec6_crypto_of_match,
2989 },
2990 };
2991
2992 module_platform_driver(artpec6_crypto_driver);
2993
2994 MODULE_AUTHOR("Axis Communications AB");
2995 MODULE_DESCRIPTION("ARTPEC-6 Crypto driver");
2996 MODULE_LICENSE("GPL");
Linux with added FPC-III support