?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'io_uring-5.11-2021-01-16' of git://git.kernel.dk/linux-block
[linux/fpc-iii.git] / drivers / crypto / sa2ul.c
blobf300b0a5958a595f4a40f1637fb4534343d667e9
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * K3 SA2UL crypto accelerator driver
4 *
5 * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
6 *
7 * Authors: Keerthy
8 * Vitaly Andrianov
9 * Tero Kristo
10 */
11 #include <linux/clk.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmaengine.h>
14 #include <linux/dmapool.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/of_device.h>
18 #include <linux/platform_device.h>
19 #include <linux/pm_runtime.h>
20
21 #include <crypto/aes.h>
22 #include <crypto/authenc.h>
23 #include <crypto/des.h>
24 #include <crypto/internal/aead.h>
25 #include <crypto/internal/hash.h>
26 #include <crypto/internal/skcipher.h>
27 #include <crypto/scatterwalk.h>
28 #include <crypto/sha1.h>
29 #include <crypto/sha2.h>
30
31 #include "sa2ul.h"
32
33 /* Byte offset for key in encryption security context */
34 #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
35 /* Byte offset for Aux-1 in encryption security context */
36 #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
37
38 #define SA_CMDL_UPD_ENC 0x0001
39 #define SA_CMDL_UPD_AUTH 0x0002
40 #define SA_CMDL_UPD_ENC_IV 0x0004
41 #define SA_CMDL_UPD_AUTH_IV 0x0008
42 #define SA_CMDL_UPD_AUX_KEY 0x0010
43
44 #define SA_AUTH_SUBKEY_LEN 16
45 #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF
46 #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000
47
48 #define MODE_CONTROL_BYTES 27
49 #define SA_HASH_PROCESSING 0
50 #define SA_CRYPTO_PROCESSING 0
51 #define SA_UPLOAD_HASH_TO_TLR BIT(6)
52
53 #define SA_SW0_FLAGS_MASK 0xF0000
54 #define SA_SW0_CMDL_INFO_MASK 0x1F00000
55 #define SA_SW0_CMDL_PRESENT BIT(4)
56 #define SA_SW0_ENG_ID_MASK 0x3E000000
57 #define SA_SW0_DEST_INFO_PRESENT BIT(30)
58 #define SA_SW2_EGRESS_LENGTH 0xFF000000
59 #define SA_BASIC_HASH 0x10
60
61 #define SHA256_DIGEST_WORDS 8
62 /* Make 32-bit word from 4 bytes */
63 #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
64 ((b2) << 8) | (b3))
65
66 /* size of SCCTL structure in bytes */
67 #define SA_SCCTL_SZ 16
68
69 /* Max Authentication tag size */
70 #define SA_MAX_AUTH_TAG_SZ 64
71
72 #define PRIV_ID 0x1
73 #define PRIV 0x1
74
75 static struct device *sa_k3_dev;
76
77 /**
78 * struct sa_cmdl_cfg - Command label configuration descriptor
79 * @aalg: authentication algorithm ID
80 * @enc_eng_id: Encryption Engine ID supported by the SA hardware
81 * @auth_eng_id: Authentication Engine ID
82 * @iv_size: Initialization Vector size
83 * @akey: Authentication key
84 * @akey_len: Authentication key length
85 * @enc: True, if this is an encode request
86 */
87 struct sa_cmdl_cfg {
88 int aalg;
89 u8 enc_eng_id;
90 u8 auth_eng_id;
91 u8 iv_size;
92 const u8 *akey;
93 u16 akey_len;
94 bool enc;
95 };
96
97 /**
98 * struct algo_data - Crypto algorithm specific data
99 * @enc_eng: Encryption engine info structure
100 * @auth_eng: Authentication engine info structure
101 * @auth_ctrl: Authentication control word
102 * @hash_size: Size of digest
103 * @iv_idx: iv index in psdata
104 * @iv_out_size: iv out size
105 * @ealg_id: Encryption Algorithm ID
106 * @aalg_id: Authentication algorithm ID
107 * @mci_enc: Mode Control Instruction for Encryption algorithm
108 * @mci_dec: Mode Control Instruction for Decryption
109 * @inv_key: Whether the encryption algorithm demands key inversion
110 * @ctx: Pointer to the algorithm context
111 * @keyed_mac: Whether the authentication algorithm has key
112 * @prep_iopad: Function pointer to generate intermediate ipad/opad
113 */
114 struct algo_data {
115 struct sa_eng_info enc_eng;
116 struct sa_eng_info auth_eng;
117 u8 auth_ctrl;
118 u8 hash_size;
119 u8 iv_idx;
120 u8 iv_out_size;
121 u8 ealg_id;
122 u8 aalg_id;
123 u8 *mci_enc;
124 u8 *mci_dec;
125 bool inv_key;
126 struct sa_tfm_ctx *ctx;
127 bool keyed_mac;
128 void (*prep_iopad)(struct algo_data *algo, const u8 *key,
129 u16 key_sz, __be32 *ipad, __be32 *opad);
130 };
131
132 /**
133 * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
134 * @type: Type of the crypto algorithm.
135 * @alg: Union of crypto algorithm definitions.
136 * @registered: Flag indicating if the crypto algorithm is already registered
137 */
138 struct sa_alg_tmpl {
139 u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
140 union {
141 struct skcipher_alg skcipher;
142 struct ahash_alg ahash;
143 struct aead_alg aead;
144 } alg;
145 bool registered;
146 };
147
148 /**
149 * struct sa_mapped_sg: scatterlist information for tx and rx
150 * @mapped: Set to true if the @sgt is mapped
151 * @dir: mapping direction used for @sgt
152 * @split_sg: Set if the sg is split and needs to be freed up
153 * @static_sg: Static scatterlist entry for overriding data
154 * @sgt: scatterlist table for DMA API use
155 */
156 struct sa_mapped_sg {
157 bool mapped;
158 enum dma_data_direction dir;
159 struct scatterlist static_sg;
160 struct scatterlist *split_sg;
161 struct sg_table sgt;
162 };
163 /**
164 * struct sa_rx_data: RX Packet miscellaneous data place holder
165 * @req: crypto request data pointer
166 * @ddev: pointer to the DMA device
167 * @tx_in: dma_async_tx_descriptor pointer for rx channel
168 * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
169 * @enc: Flag indicating either encryption or decryption
170 * @enc_iv_size: Initialisation vector size
171 * @iv_idx: Initialisation vector index
172 */
173 struct sa_rx_data {
174 void *req;
175 struct device *ddev;
176 struct dma_async_tx_descriptor *tx_in;
177 struct sa_mapped_sg mapped_sg[2];
178 u8 enc;
179 u8 enc_iv_size;
180 u8 iv_idx;
181 };
182
183 /**
184 * struct sa_req: SA request definition
185 * @dev: device for the request
186 * @size: total data to the xmitted via DMA
187 * @enc_offset: offset of cipher data
188 * @enc_size: data to be passed to cipher engine
189 * @enc_iv: cipher IV
190 * @auth_offset: offset of the authentication data
191 * @auth_size: size of the authentication data
192 * @auth_iv: authentication IV
193 * @type: algorithm type for the request
194 * @cmdl: command label pointer
195 * @base: pointer to the base request
196 * @ctx: pointer to the algorithm context data
197 * @enc: true if this is an encode request
198 * @src: source data
199 * @dst: destination data
200 * @callback: DMA callback for the request
201 * @mdata_size: metadata size passed to DMA
202 */
203 struct sa_req {
204 struct device *dev;
205 u16 size;
206 u8 enc_offset;
207 u16 enc_size;
208 u8 *enc_iv;
209 u8 auth_offset;
210 u16 auth_size;
211 u8 *auth_iv;
212 u32 type;
213 u32 *cmdl;
214 struct crypto_async_request *base;
215 struct sa_tfm_ctx *ctx;
216 bool enc;
217 struct scatterlist *src;
218 struct scatterlist *dst;
219 dma_async_tx_callback callback;
220 u16 mdata_size;
221 };
222
223 /*
224 * Mode Control Instructions for various Key lengths 128, 192, 256
225 * For CBC (Cipher Block Chaining) mode for encryption
226 */
227 static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
228 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
229 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
230 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
231 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
232 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
233 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
234 { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
235 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
236 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
237 };
238
239 /*
240 * Mode Control Instructions for various Key lengths 128, 192, 256
241 * For CBC (Cipher Block Chaining) mode for decryption
242 */
243 static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
244 { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
245 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
246 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
247 { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
248 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
249 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
250 { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
251 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
252 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
253 };
254
255 /*
256 * Mode Control Instructions for various Key lengths 128, 192, 256
257 * For CBC (Cipher Block Chaining) mode for encryption
258 */
259 static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
260 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
261 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
262 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
263 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
264 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
265 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
266 { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
267 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
268 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
269 };
270
271 /*
272 * Mode Control Instructions for various Key lengths 128, 192, 256
273 * For CBC (Cipher Block Chaining) mode for decryption
274 */
275 static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
276 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
277 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
278 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
279 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
280 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
281 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
282 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
283 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
284 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
285 };
286
287 /*
288 * Mode Control Instructions for various Key lengths 128, 192, 256
289 * For ECB (Electronic Code Book) mode for encryption
290 */
291 static u8 mci_ecb_enc_array[3][27] = {
292 { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
293 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
294 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
295 { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
296 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
297 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
298 { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
299 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
300 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
301 };
302
303 /*
304 * Mode Control Instructions for various Key lengths 128, 192, 256
305 * For ECB (Electronic Code Book) mode for decryption
306 */
307 static u8 mci_ecb_dec_array[3][27] = {
308 { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
309 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
310 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
311 { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
312 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
313 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
314 { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
315 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
316 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
317 };
318
319 /*
320 * Mode Control Instructions for DES algorithm
321 * For CBC (Cipher Block Chaining) mode and ECB mode
322 * encryption and for decryption respectively
323 */
324 static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
325 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
326 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
327 0x00, 0x00, 0x00,
328 };
329
330 static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
331 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
332 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
333 0x00, 0x00, 0x00,
334 };
335
336 static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
337 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
338 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
339 0x00, 0x00, 0x00,
340 };
341
342 static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
343 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
344 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
345 0x00, 0x00, 0x00,
346 };
347
348 /*
349 * Perform 16 byte or 128 bit swizzling
350 * The SA2UL Expects the security context to
351 * be in little Endian and the bus width is 128 bits or 16 bytes
352 * Hence swap 16 bytes at a time from higher to lower address
353 */
354 static void sa_swiz_128(u8 *in, u16 len)
355 {
356 u8 data[16];
357 int i, j;
358
359 for (i = 0; i < len; i += 16) {
360 memcpy(data, &in[i], 16);
361 for (j = 0; j < 16; j++)
362 in[i + j] = data[15 - j];
363 }
364 }
365
366 /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
367 static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz)
368 {
369 int i;
370
371 for (i = 0; i < key_sz; i++)
372 k_ipad[i] = key[i] ^ 0x36;
373
374 /* Instead of XOR with 0 */
375 for (; i < SHA1_BLOCK_SIZE; i++)
376 k_ipad[i] = 0x36;
377 }
378
379 static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz)
380 {
381 int i;
382
383 for (i = 0; i < key_sz; i++)
384 k_opad[i] = key[i] ^ 0x5c;
385
386 /* Instead of XOR with 0 */
387 for (; i < SHA1_BLOCK_SIZE; i++)
388 k_opad[i] = 0x5c;
389 }
390
391 static void sa_export_shash(void *state, struct shash_desc *hash,
392 int digest_size, __be32 *out)
393 {
394 struct sha1_state *sha1;
395 struct sha256_state *sha256;
396 u32 *result;
397
398 switch (digest_size) {
399 case SHA1_DIGEST_SIZE:
400 sha1 = state;
401 result = sha1->state;
402 break;
403 case SHA256_DIGEST_SIZE:
404 sha256 = state;
405 result = sha256->state;
406 break;
407 default:
408 dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
409 digest_size);
410 return;
411 }
412
413 crypto_shash_export(hash, state);
414
415 cpu_to_be32_array(out, result, digest_size / 4);
416 }
417
418 static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
419 u16 key_sz, __be32 *ipad, __be32 *opad)
420 {
421 SHASH_DESC_ON_STACK(shash, data->ctx->shash);
422 int block_size = crypto_shash_blocksize(data->ctx->shash);
423 int digest_size = crypto_shash_digestsize(data->ctx->shash);
424 union {
425 struct sha1_state sha1;
426 struct sha256_state sha256;
427 u8 k_pad[SHA1_BLOCK_SIZE];
428 } sha;
429
430 shash->tfm = data->ctx->shash;
431
432 prepare_kipad(sha.k_pad, key, key_sz);
433
434 crypto_shash_init(shash);
435 crypto_shash_update(shash, sha.k_pad, block_size);
436 sa_export_shash(&sha, shash, digest_size, ipad);
437
438 prepare_kopad(sha.k_pad, key, key_sz);
439
440 crypto_shash_init(shash);
441 crypto_shash_update(shash, sha.k_pad, block_size);
442
443 sa_export_shash(&sha, shash, digest_size, opad);
444
445 memzero_explicit(&sha, sizeof(sha));
446 }
447
448 /* Derive the inverse key used in AES-CBC decryption operation */
449 static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
450 {
451 struct crypto_aes_ctx ctx;
452 int key_pos;
453
454 if (aes_expandkey(&ctx, key, key_sz)) {
455 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
456 return -EINVAL;
457 }
458
459 /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
460 if (key_sz == AES_KEYSIZE_192) {
461 ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
462 ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
463 }
464
465 /* Based crypto_aes_expand_key logic */
466 switch (key_sz) {
467 case AES_KEYSIZE_128:
468 case AES_KEYSIZE_192:
469 key_pos = key_sz + 24;
470 break;
471
472 case AES_KEYSIZE_256:
473 key_pos = key_sz + 24 - 4;
474 break;
475
476 default:
477 dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
478 return -EINVAL;
479 }
480
481 memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
482 return 0;
483 }
484
485 /* Set Security context for the encryption engine */
486 static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
487 u8 enc, u8 *sc_buf)
488 {
489 const u8 *mci = NULL;
490
491 /* Set Encryption mode selector to crypto processing */
492 sc_buf[0] = SA_CRYPTO_PROCESSING;
493
494 if (enc)
495 mci = ad->mci_enc;
496 else
497 mci = ad->mci_dec;
498 /* Set the mode control instructions in security context */
499 if (mci)
500 memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
501
502 /* For AES-CBC decryption get the inverse key */
503 if (ad->inv_key && !enc) {
504 if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
505 return -EINVAL;
506 /* For all other cases: key is used */
507 } else {
508 memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
509 }
510
511 return 0;
512 }
513
514 /* Set Security context for the authentication engine */
515 static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
516 u8 *sc_buf)
517 {
518 __be32 *ipad = (void *)(sc_buf + 32);
519 __be32 *opad = (void *)(sc_buf + 64);
520
521 /* Set Authentication mode selector to hash processing */
522 sc_buf[0] = SA_HASH_PROCESSING;
523 /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
524 sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
525 sc_buf[1] |= ad->auth_ctrl;
526
527 /* Copy the keys or ipad/opad */
528 if (ad->keyed_mac)
529 ad->prep_iopad(ad, key, key_sz, ipad, opad);
530 else {
531 /* basic hash */
532 sc_buf[1] |= SA_BASIC_HASH;
533 }
534 }
535
536 static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
537 {
538 int j;
539
540 for (j = 0; j < ((size16) ? 4 : 2); j++) {
541 *out = cpu_to_be32(*((u32 *)iv));
542 iv += 4;
543 out++;
544 }
545 }
546
547 /* Format general command label */
548 static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
549 struct sa_cmdl_upd_info *upd_info)
550 {
551 u8 enc_offset = 0, auth_offset = 0, total = 0;
552 u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
553 u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
554 u32 *word_ptr = (u32 *)cmdl;
555 int i;
556
557 /* Clear the command label */
558 memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
559
560 /* Iniialize the command update structure */
561 memzero_explicit(upd_info, sizeof(*upd_info));
562
563 if (cfg->enc_eng_id && cfg->auth_eng_id) {
564 if (cfg->enc) {
565 auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
566 enc_next_eng = cfg->auth_eng_id;
567
568 if (cfg->iv_size)
569 auth_offset += cfg->iv_size;
570 } else {
571 enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
572 auth_next_eng = cfg->enc_eng_id;
573 }
574 }
575
576 if (cfg->enc_eng_id) {
577 upd_info->flags |= SA_CMDL_UPD_ENC;
578 upd_info->enc_size.index = enc_offset >> 2;
579 upd_info->enc_offset.index = upd_info->enc_size.index + 1;
580 /* Encryption command label */
581 cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
582
583 /* Encryption modes requiring IV */
584 if (cfg->iv_size) {
585 upd_info->flags |= SA_CMDL_UPD_ENC_IV;
586 upd_info->enc_iv.index =
587 (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
588 upd_info->enc_iv.size = cfg->iv_size;
589
590 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
591 SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
592
593 cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
594 (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
595 total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
596 } else {
597 cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
598 SA_CMDL_HEADER_SIZE_BYTES;
599 total += SA_CMDL_HEADER_SIZE_BYTES;
600 }
601 }
602
603 if (cfg->auth_eng_id) {
604 upd_info->flags |= SA_CMDL_UPD_AUTH;
605 upd_info->auth_size.index = auth_offset >> 2;
606 upd_info->auth_offset.index = upd_info->auth_size.index + 1;
607 cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
608 cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
609 SA_CMDL_HEADER_SIZE_BYTES;
610 total += SA_CMDL_HEADER_SIZE_BYTES;
611 }
612
613 total = roundup(total, 8);
614
615 for (i = 0; i < total / 4; i++)
616 word_ptr[i] = swab32(word_ptr[i]);
617
618 return total;
619 }
620
621 /* Update Command label */
622 static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
623 struct sa_cmdl_upd_info *upd_info)
624 {
625 int i = 0, j;
626
627 if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
628 cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
629 cmdl[upd_info->enc_size.index] |= req->enc_size;
630 cmdl[upd_info->enc_offset.index] &=
631 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
632 cmdl[upd_info->enc_offset.index] |=
633 ((u32)req->enc_offset <<
634 __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
635
636 if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
637 __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
638 u32 *enc_iv = (u32 *)req->enc_iv;
639
640 for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
641 data[j] = cpu_to_be32(*enc_iv);
642 enc_iv++;
643 }
644 }
645 }
646
647 if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
648 cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
649 cmdl[upd_info->auth_size.index] |= req->auth_size;
650 cmdl[upd_info->auth_offset.index] &=
651 ~SA_CMDL_SOP_BYPASS_LEN_MASK;
652 cmdl[upd_info->auth_offset.index] |=
653 ((u32)req->auth_offset <<
654 __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK));
655 if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
656 sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
657 req->auth_iv,
658 (upd_info->auth_iv.size > 8));
659 }
660 if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
661 int offset = (req->auth_size & 0xF) ? 4 : 0;
662
663 memcpy(&cmdl[upd_info->aux_key_info.index],
664 &upd_info->aux_key[offset], 16);
665 }
666 }
667 }
668
669 /* Format SWINFO words to be sent to SA */
670 static
671 void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
672 u8 cmdl_present, u8 cmdl_offset, u8 flags,
673 u8 hash_size, u32 *swinfo)
674 {
675 swinfo[0] = sc_id;
676 swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK));
677 if (likely(cmdl_present))
678 swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) <<
679 __ffs(SA_SW0_CMDL_INFO_MASK));
680 swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK));
681
682 swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
683 swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
684 swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
685 swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH));
686 }
687
688 /* Dump the security context */
689 static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
690 {
691 #ifdef DEBUG
692 dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
693 print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
694 16, 1, buf, SA_CTX_MAX_SZ, false);
695 #endif
696 }
697
698 static
699 int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key,
700 u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz,
701 struct algo_data *ad, u8 enc, u32 *swinfo)
702 {
703 int enc_sc_offset = 0;
704 int auth_sc_offset = 0;
705 u8 *sc_buf = ctx->sc;
706 u16 sc_id = ctx->sc_id;
707 u8 first_engine = 0;
708
709 memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
710
711 if (ad->auth_eng.eng_id) {
712 if (enc)
713 first_engine = ad->enc_eng.eng_id;
714 else
715 first_engine = ad->auth_eng.eng_id;
716
717 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
718 auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
719 sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
720 if (!ad->hash_size)
721 return -EINVAL;
722 ad->hash_size = roundup(ad->hash_size, 8);
723
724 } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
725 enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
726 first_engine = ad->enc_eng.eng_id;
727 sc_buf[1] = SA_SCCTL_FE_ENC;
728 ad->hash_size = ad->iv_out_size;
729 }
730
731 /* SCCTL Owner info: 0=host, 1=CP_ACE */
732 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
733 memcpy(&sc_buf[2], &sc_id, 2);
734 sc_buf[4] = 0x0;
735 sc_buf[5] = PRIV_ID;
736 sc_buf[6] = PRIV;
737 sc_buf[7] = 0x0;
738
739 /* Prepare context for encryption engine */
740 if (ad->enc_eng.sc_size) {
741 if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
742 &sc_buf[enc_sc_offset]))
743 return -EINVAL;
744 }
745
746 /* Prepare context for authentication engine */
747 if (ad->auth_eng.sc_size)
748 sa_set_sc_auth(ad, auth_key, auth_key_sz,
749 &sc_buf[auth_sc_offset]);
750
751 /* Set the ownership of context to CP_ACE */
752 sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
753
754 /* swizzle the security context */
755 sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
756
757 sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
758 SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
759
760 sa_dump_sc(sc_buf, ctx->sc_phys);
761
762 return 0;
763 }
764
765 /* Free the per direction context memory */
766 static void sa_free_ctx_info(struct sa_ctx_info *ctx,
767 struct sa_crypto_data *data)
768 {
769 unsigned long bn;
770
771 bn = ctx->sc_id - data->sc_id_start;
772 spin_lock(&data->scid_lock);
773 __clear_bit(bn, data->ctx_bm);
774 data->sc_id--;
775 spin_unlock(&data->scid_lock);
776
777 if (ctx->sc) {
778 dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
779 ctx->sc = NULL;
780 }
781 }
782
783 static int sa_init_ctx_info(struct sa_ctx_info *ctx,
784 struct sa_crypto_data *data)
785 {
786 unsigned long bn;
787 int err;
788
789 spin_lock(&data->scid_lock);
790 bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
791 __set_bit(bn, data->ctx_bm);
792 data->sc_id++;
793 spin_unlock(&data->scid_lock);
794
795 ctx->sc_id = (u16)(data->sc_id_start + bn);
796
797 ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
798 if (!ctx->sc) {
799 dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
800 err = -ENOMEM;
801 goto scid_rollback;
802 }
803
804 return 0;
805
806 scid_rollback:
807 spin_lock(&data->scid_lock);
808 __clear_bit(bn, data->ctx_bm);
809 data->sc_id--;
810 spin_unlock(&data->scid_lock);
811
812 return err;
813 }
814
815 static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
816 {
817 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
818 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
819
820 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
821 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
822 ctx->dec.sc_id, &ctx->dec.sc_phys);
823
824 sa_free_ctx_info(&ctx->enc, data);
825 sa_free_ctx_info(&ctx->dec, data);
826
827 crypto_free_skcipher(ctx->fallback.skcipher);
828 }
829
830 static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
831 {
832 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
833 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
834 const char *name = crypto_tfm_alg_name(&tfm->base);
835 struct crypto_skcipher *child;
836 int ret;
837
838 memzero_explicit(ctx, sizeof(*ctx));
839 ctx->dev_data = data;
840
841 ret = sa_init_ctx_info(&ctx->enc, data);
842 if (ret)
843 return ret;
844 ret = sa_init_ctx_info(&ctx->dec, data);
845 if (ret) {
846 sa_free_ctx_info(&ctx->enc, data);
847 return ret;
848 }
849
850 child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
851
852 if (IS_ERR(child)) {
853 dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
854 return PTR_ERR(child);
855 }
856
857 ctx->fallback.skcipher = child;
858 crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
859 sizeof(struct skcipher_request));
860
861 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
862 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
863 ctx->dec.sc_id, &ctx->dec.sc_phys);
864 return 0;
865 }
866
867 static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
868 unsigned int keylen, struct algo_data *ad)
869 {
870 struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
871 struct crypto_skcipher *child = ctx->fallback.skcipher;
872 int cmdl_len;
873 struct sa_cmdl_cfg cfg;
874 int ret;
875
876 if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
877 keylen != AES_KEYSIZE_256)
878 return -EINVAL;
879
880 ad->enc_eng.eng_id = SA_ENG_ID_EM1;
881 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
882
883 memzero_explicit(&cfg, sizeof(cfg));
884 cfg.enc_eng_id = ad->enc_eng.eng_id;
885 cfg.iv_size = crypto_skcipher_ivsize(tfm);
886
887 crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
888 crypto_skcipher_set_flags(child, tfm->base.crt_flags &
889 CRYPTO_TFM_REQ_MASK);
890 ret = crypto_skcipher_setkey(child, key, keylen);
891 if (ret)
892 return ret;
893
894 /* Setup Encryption Security Context & Command label template */
895 if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1,
896 &ctx->enc.epib[1]))
897 goto badkey;
898
899 cmdl_len = sa_format_cmdl_gen(&cfg,
900 (u8 *)ctx->enc.cmdl,
901 &ctx->enc.cmdl_upd_info);
902 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
903 goto badkey;
904
905 ctx->enc.cmdl_size = cmdl_len;
906
907 /* Setup Decryption Security Context & Command label template */
908 if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0,
909 &ctx->dec.epib[1]))
910 goto badkey;
911
912 cfg.enc_eng_id = ad->enc_eng.eng_id;
913 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
914 &ctx->dec.cmdl_upd_info);
915
916 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
917 goto badkey;
918
919 ctx->dec.cmdl_size = cmdl_len;
920 ctx->iv_idx = ad->iv_idx;
921
922 return 0;
923
924 badkey:
925 dev_err(sa_k3_dev, "%s: badkey\n", __func__);
926 return -EINVAL;
927 }
928
929 static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
930 unsigned int keylen)
931 {
932 struct algo_data ad = { 0 };
933 /* Convert the key size (16/24/32) to the key size index (0/1/2) */
934 int key_idx = (keylen >> 3) - 2;
935
936 if (key_idx >= 3)
937 return -EINVAL;
938
939 ad.mci_enc = mci_cbc_enc_array[key_idx];
940 ad.mci_dec = mci_cbc_dec_array[key_idx];
941 ad.inv_key = true;
942 ad.ealg_id = SA_EALG_ID_AES_CBC;
943 ad.iv_idx = 4;
944 ad.iv_out_size = 16;
945
946 return sa_cipher_setkey(tfm, key, keylen, &ad);
947 }
948
949 static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
950 unsigned int keylen)
951 {
952 struct algo_data ad = { 0 };
953 /* Convert the key size (16/24/32) to the key size index (0/1/2) */
954 int key_idx = (keylen >> 3) - 2;
955
956 if (key_idx >= 3)
957 return -EINVAL;
958
959 ad.mci_enc = mci_ecb_enc_array[key_idx];
960 ad.mci_dec = mci_ecb_dec_array[key_idx];
961 ad.inv_key = true;
962 ad.ealg_id = SA_EALG_ID_AES_ECB;
963
964 return sa_cipher_setkey(tfm, key, keylen, &ad);
965 }
966
967 static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
968 unsigned int keylen)
969 {
970 struct algo_data ad = { 0 };
971
972 ad.mci_enc = mci_cbc_3des_enc_array;
973 ad.mci_dec = mci_cbc_3des_dec_array;
974 ad.ealg_id = SA_EALG_ID_3DES_CBC;
975 ad.iv_idx = 6;
976 ad.iv_out_size = 8;
977
978 return sa_cipher_setkey(tfm, key, keylen, &ad);
979 }
980
981 static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
982 unsigned int keylen)
983 {
984 struct algo_data ad = { 0 };
985
986 ad.mci_enc = mci_ecb_3des_enc_array;
987 ad.mci_dec = mci_ecb_3des_dec_array;
988
989 return sa_cipher_setkey(tfm, key, keylen, &ad);
990 }
991
992 static void sa_sync_from_device(struct sa_rx_data *rxd)
993 {
994 struct sg_table *sgt;
995
996 if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
997 sgt = &rxd->mapped_sg[0].sgt;
998 else
999 sgt = &rxd->mapped_sg[1].sgt;
1000
1001 dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
1002 }
1003
1004 static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
1005 {
1006 int i;
1007
1008 for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
1009 struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
1010
1011 if (mapped_sg->mapped) {
1012 dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
1013 mapped_sg->dir, 0);
1014 kfree(mapped_sg->split_sg);
1015 }
1016 }
1017
1018 kfree(rxd);
1019 }
1020
1021 static void sa_aes_dma_in_callback(void *data)
1022 {
1023 struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1024 struct skcipher_request *req;
1025 u32 *result;
1026 __be32 *mdptr;
1027 size_t ml, pl;
1028 int i;
1029
1030 sa_sync_from_device(rxd);
1031 req = container_of(rxd->req, struct skcipher_request, base);
1032
1033 if (req->iv) {
1034 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
1035 &ml);
1036 result = (u32 *)req->iv;
1037
1038 for (i = 0; i < (rxd->enc_iv_size / 4); i++)
1039 result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
1040 }
1041
1042 sa_free_sa_rx_data(rxd);
1043
1044 skcipher_request_complete(req, 0);
1045 }
1046
1047 static void
1048 sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
1049 {
1050 u32 *out, *in;
1051 int i;
1052
1053 for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
1054 *out++ = *in++;
1055
1056 mdptr[4] = (0xFFFF << 16);
1057 for (out = &mdptr[5], in = psdata, i = 0;
1058 i < pslen / sizeof(u32); i++)
1059 *out++ = *in++;
1060 }
1061
1062 static int sa_run(struct sa_req *req)
1063 {
1064 struct sa_rx_data *rxd;
1065 gfp_t gfp_flags;
1066 u32 cmdl[SA_MAX_CMDL_WORDS];
1067 struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
1068 struct device *ddev;
1069 struct dma_chan *dma_rx;
1070 int sg_nents, src_nents, dst_nents;
1071 struct scatterlist *src, *dst;
1072 size_t pl, ml, split_size;
1073 struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
1074 int ret;
1075 struct dma_async_tx_descriptor *tx_out;
1076 u32 *mdptr;
1077 bool diff_dst;
1078 enum dma_data_direction dir_src;
1079 struct sa_mapped_sg *mapped_sg;
1080
1081 gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
1082 GFP_KERNEL : GFP_ATOMIC;
1083
1084 rxd = kzalloc(sizeof(*rxd), gfp_flags);
1085 if (!rxd)
1086 return -ENOMEM;
1087
1088 if (req->src != req->dst) {
1089 diff_dst = true;
1090 dir_src = DMA_TO_DEVICE;
1091 } else {
1092 diff_dst = false;
1093 dir_src = DMA_BIDIRECTIONAL;
1094 }
1095
1096 /*
1097 * SA2UL has an interesting feature where the receive DMA channel
1098 * is selected based on the data passed to the engine. Within the
1099 * transition range, there is also a space where it is impossible
1100 * to determine where the data will end up, and this should be
1101 * avoided. This will be handled by the SW fallback mechanism by
1102 * the individual algorithm implementations.
1103 */
1104 if (req->size >= 256)
1105 dma_rx = pdata->dma_rx2;
1106 else
1107 dma_rx = pdata->dma_rx1;
1108
1109 ddev = dma_rx->device->dev;
1110 rxd->ddev = ddev;
1111
1112 memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
1113
1114 sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
1115
1116 if (req->type != CRYPTO_ALG_TYPE_AHASH) {
1117 if (req->enc)
1118 req->type |=
1119 (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
1120 else
1121 req->type |=
1122 (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
1123 }
1124
1125 cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
1126
1127 /*
1128 * Map the packets, first we check if the data fits into a single
1129 * sg entry and use that if possible. If it does not fit, we check
1130 * if we need to do sg_split to align the scatterlist data on the
1131 * actual data size being processed by the crypto engine.
1132 */
1133 src = req->src;
1134 sg_nents = sg_nents_for_len(src, req->size);
1135
1136 split_size = req->size;
1137
1138 mapped_sg = &rxd->mapped_sg[0];
1139 if (sg_nents == 1 && split_size <= req->src->length) {
1140 src = &mapped_sg->static_sg;
1141 src_nents = 1;
1142 sg_init_table(src, 1);
1143 sg_set_page(src, sg_page(req->src), split_size,
1144 req->src->offset);
1145
1146 mapped_sg->sgt.sgl = src;
1147 mapped_sg->sgt.orig_nents = src_nents;
1148 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1149 if (ret)
1150 return ret;
1151
1152 mapped_sg->dir = dir_src;
1153 mapped_sg->mapped = true;
1154 } else {
1155 mapped_sg->sgt.sgl = req->src;
1156 mapped_sg->sgt.orig_nents = sg_nents;
1157 ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1158 if (ret)
1159 return ret;
1160
1161 mapped_sg->dir = dir_src;
1162 mapped_sg->mapped = true;
1163
1164 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
1165 &split_size, &src, &src_nents, gfp_flags);
1166 if (ret) {
1167 src_nents = mapped_sg->sgt.nents;
1168 src = mapped_sg->sgt.sgl;
1169 } else {
1170 mapped_sg->split_sg = src;
1171 }
1172 }
1173
1174 dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
1175
1176 if (!diff_dst) {
1177 dst_nents = src_nents;
1178 dst = src;
1179 } else {
1180 dst_nents = sg_nents_for_len(req->dst, req->size);
1181 mapped_sg = &rxd->mapped_sg[1];
1182
1183 if (dst_nents == 1 && split_size <= req->dst->length) {
1184 dst = &mapped_sg->static_sg;
1185 dst_nents = 1;
1186 sg_init_table(dst, 1);
1187 sg_set_page(dst, sg_page(req->dst), split_size,
1188 req->dst->offset);
1189
1190 mapped_sg->sgt.sgl = dst;
1191 mapped_sg->sgt.orig_nents = dst_nents;
1192 ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1193 DMA_FROM_DEVICE, 0);
1194 if (ret)
1195 goto err_cleanup;
1196
1197 mapped_sg->dir = DMA_FROM_DEVICE;
1198 mapped_sg->mapped = true;
1199 } else {
1200 mapped_sg->sgt.sgl = req->dst;
1201 mapped_sg->sgt.orig_nents = dst_nents;
1202 ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
1203 DMA_FROM_DEVICE, 0);
1204 if (ret)
1205 goto err_cleanup;
1206
1207 mapped_sg->dir = DMA_FROM_DEVICE;
1208 mapped_sg->mapped = true;
1209
1210 ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
1211 0, 1, &split_size, &dst, &dst_nents,
1212 gfp_flags);
1213 if (ret) {
1214 dst_nents = mapped_sg->sgt.nents;
1215 dst = mapped_sg->sgt.sgl;
1216 } else {
1217 mapped_sg->split_sg = dst;
1218 }
1219 }
1220 }
1221
1222 rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
1223 DMA_DEV_TO_MEM,
1224 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1225 if (!rxd->tx_in) {
1226 dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
1227 ret = -EINVAL;
1228 goto err_cleanup;
1229 }
1230
1231 rxd->req = (void *)req->base;
1232 rxd->enc = req->enc;
1233 rxd->iv_idx = req->ctx->iv_idx;
1234 rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
1235 rxd->tx_in->callback = req->callback;
1236 rxd->tx_in->callback_param = rxd;
1237
1238 tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
1239 src_nents, DMA_MEM_TO_DEV,
1240 DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1241
1242 if (!tx_out) {
1243 dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
1244 ret = -EINVAL;
1245 goto err_cleanup;
1246 }
1247
1248 /*
1249 * Prepare metadata for DMA engine. This essentially describes the
1250 * crypto algorithm to be used, data sizes, different keys etc.
1251 */
1252 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
1253
1254 sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
1255 sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
1256 sa_ctx->epib);
1257
1258 ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
1259 dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
1260
1261 dmaengine_submit(tx_out);
1262 dmaengine_submit(rxd->tx_in);
1263
1264 dma_async_issue_pending(dma_rx);
1265 dma_async_issue_pending(pdata->dma_tx);
1266
1267 return -EINPROGRESS;
1268
1269 err_cleanup:
1270 sa_free_sa_rx_data(rxd);
1271
1272 return ret;
1273 }
1274
1275 static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
1276 {
1277 struct sa_tfm_ctx *ctx =
1278 crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
1279 struct crypto_alg *alg = req->base.tfm->__crt_alg;
1280 struct sa_req sa_req = { 0 };
1281
1282 if (!req->cryptlen)
1283 return 0;
1284
1285 if (req->cryptlen % alg->cra_blocksize)
1286 return -EINVAL;
1287
1288 /* Use SW fallback if the data size is not supported */
1289 if (req->cryptlen > SA_MAX_DATA_SZ ||
1290 (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
1291 req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
1292 struct skcipher_request *subreq = skcipher_request_ctx(req);
1293
1294 skcipher_request_set_tfm(subreq, ctx->fallback.skcipher);
1295 skcipher_request_set_callback(subreq, req->base.flags,
1296 req->base.complete,
1297 req->base.data);
1298 skcipher_request_set_crypt(subreq, req->src, req->dst,
1299 req->cryptlen, req->iv);
1300 if (enc)
1301 return crypto_skcipher_encrypt(subreq);
1302 else
1303 return crypto_skcipher_decrypt(subreq);
1304 }
1305
1306 sa_req.size = req->cryptlen;
1307 sa_req.enc_size = req->cryptlen;
1308 sa_req.src = req->src;
1309 sa_req.dst = req->dst;
1310 sa_req.enc_iv = iv;
1311 sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
1312 sa_req.enc = enc;
1313 sa_req.callback = sa_aes_dma_in_callback;
1314 sa_req.mdata_size = 44;
1315 sa_req.base = &req->base;
1316 sa_req.ctx = ctx;
1317
1318 return sa_run(&sa_req);
1319 }
1320
1321 static int sa_encrypt(struct skcipher_request *req)
1322 {
1323 return sa_cipher_run(req, req->iv, 1);
1324 }
1325
1326 static int sa_decrypt(struct skcipher_request *req)
1327 {
1328 return sa_cipher_run(req, req->iv, 0);
1329 }
1330
1331 static void sa_sha_dma_in_callback(void *data)
1332 {
1333 struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1334 struct ahash_request *req;
1335 struct crypto_ahash *tfm;
1336 unsigned int authsize;
1337 int i;
1338 size_t ml, pl;
1339 u32 *result;
1340 __be32 *mdptr;
1341
1342 sa_sync_from_device(rxd);
1343 req = container_of(rxd->req, struct ahash_request, base);
1344 tfm = crypto_ahash_reqtfm(req);
1345 authsize = crypto_ahash_digestsize(tfm);
1346
1347 mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1348 result = (u32 *)req->result;
1349
1350 for (i = 0; i < (authsize / 4); i++)
1351 result[i] = be32_to_cpu(mdptr[i + 4]);
1352
1353 sa_free_sa_rx_data(rxd);
1354
1355 ahash_request_complete(req, 0);
1356 }
1357
1358 static int zero_message_process(struct ahash_request *req)
1359 {
1360 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1361 int sa_digest_size = crypto_ahash_digestsize(tfm);
1362
1363 switch (sa_digest_size) {
1364 case SHA1_DIGEST_SIZE:
1365 memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
1366 break;
1367 case SHA256_DIGEST_SIZE:
1368 memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
1369 break;
1370 case SHA512_DIGEST_SIZE:
1371 memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
1372 break;
1373 default:
1374 return -EINVAL;
1375 }
1376
1377 return 0;
1378 }
1379
1380 static int sa_sha_run(struct ahash_request *req)
1381 {
1382 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
1383 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1384 struct sa_req sa_req = { 0 };
1385 size_t auth_len;
1386
1387 auth_len = req->nbytes;
1388
1389 if (!auth_len)
1390 return zero_message_process(req);
1391
1392 if (auth_len > SA_MAX_DATA_SZ ||
1393 (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
1394 auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
1395 struct ahash_request *subreq = &rctx->fallback_req;
1396 int ret = 0;
1397
1398 ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1399 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1400
1401 crypto_ahash_init(subreq);
1402
1403 subreq->nbytes = auth_len;
1404 subreq->src = req->src;
1405 subreq->result = req->result;
1406
1407 ret |= crypto_ahash_update(subreq);
1408
1409 subreq->nbytes = 0;
1410
1411 ret |= crypto_ahash_final(subreq);
1412
1413 return ret;
1414 }
1415
1416 sa_req.size = auth_len;
1417 sa_req.auth_size = auth_len;
1418 sa_req.src = req->src;
1419 sa_req.dst = req->src;
1420 sa_req.enc = true;
1421 sa_req.type = CRYPTO_ALG_TYPE_AHASH;
1422 sa_req.callback = sa_sha_dma_in_callback;
1423 sa_req.mdata_size = 28;
1424 sa_req.ctx = ctx;
1425 sa_req.base = &req->base;
1426
1427 return sa_run(&sa_req);
1428 }
1429
1430 static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad)
1431 {
1432 int bs = crypto_shash_blocksize(ctx->shash);
1433 int cmdl_len;
1434 struct sa_cmdl_cfg cfg;
1435
1436 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1437 ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1438 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1439
1440 memset(ctx->authkey, 0, bs);
1441 memset(&cfg, 0, sizeof(cfg));
1442 cfg.aalg = ad->aalg_id;
1443 cfg.enc_eng_id = ad->enc_eng.eng_id;
1444 cfg.auth_eng_id = ad->auth_eng.eng_id;
1445 cfg.iv_size = 0;
1446 cfg.akey = NULL;
1447 cfg.akey_len = 0;
1448
1449 /* Setup Encryption Security Context & Command label template */
1450 if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0,
1451 &ctx->enc.epib[1]))
1452 goto badkey;
1453
1454 cmdl_len = sa_format_cmdl_gen(&cfg,
1455 (u8 *)ctx->enc.cmdl,
1456 &ctx->enc.cmdl_upd_info);
1457 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1458 goto badkey;
1459
1460 ctx->enc.cmdl_size = cmdl_len;
1461
1462 return 0;
1463
1464 badkey:
1465 dev_err(sa_k3_dev, "%s: badkey\n", __func__);
1466 return -EINVAL;
1467 }
1468
1469 static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
1470 {
1471 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1472 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1473 int ret;
1474
1475 memset(ctx, 0, sizeof(*ctx));
1476 ctx->dev_data = data;
1477 ret = sa_init_ctx_info(&ctx->enc, data);
1478 if (ret)
1479 return ret;
1480
1481 if (alg_base) {
1482 ctx->shash = crypto_alloc_shash(alg_base, 0,
1483 CRYPTO_ALG_NEED_FALLBACK);
1484 if (IS_ERR(ctx->shash)) {
1485 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
1486 alg_base);
1487 return PTR_ERR(ctx->shash);
1488 }
1489 /* for fallback */
1490 ctx->fallback.ahash =
1491 crypto_alloc_ahash(alg_base, 0,
1492 CRYPTO_ALG_NEED_FALLBACK);
1493 if (IS_ERR(ctx->fallback.ahash)) {
1494 dev_err(ctx->dev_data->dev,
1495 "Could not load fallback driver\n");
1496 return PTR_ERR(ctx->fallback.ahash);
1497 }
1498 }
1499
1500 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1501 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1502 ctx->dec.sc_id, &ctx->dec.sc_phys);
1503
1504 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1505 sizeof(struct sa_sha_req_ctx) +
1506 crypto_ahash_reqsize(ctx->fallback.ahash));
1507
1508 return 0;
1509 }
1510
1511 static int sa_sha_digest(struct ahash_request *req)
1512 {
1513 return sa_sha_run(req);
1514 }
1515
1516 static int sa_sha_init(struct ahash_request *req)
1517 {
1518 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1519 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1520 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1521
1522 dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
1523 crypto_ahash_digestsize(tfm), rctx);
1524
1525 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1526 rctx->fallback_req.base.flags =
1527 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1528
1529 return crypto_ahash_init(&rctx->fallback_req);
1530 }
1531
1532 static int sa_sha_update(struct ahash_request *req)
1533 {
1534 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1535 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1536 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1537
1538 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1539 rctx->fallback_req.base.flags =
1540 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1541 rctx->fallback_req.nbytes = req->nbytes;
1542 rctx->fallback_req.src = req->src;
1543
1544 return crypto_ahash_update(&rctx->fallback_req);
1545 }
1546
1547 static int sa_sha_final(struct ahash_request *req)
1548 {
1549 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1550 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1551 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1552
1553 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1554 rctx->fallback_req.base.flags =
1555 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1556 rctx->fallback_req.result = req->result;
1557
1558 return crypto_ahash_final(&rctx->fallback_req);
1559 }
1560
1561 static int sa_sha_finup(struct ahash_request *req)
1562 {
1563 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1564 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1565 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1566
1567 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1568 rctx->fallback_req.base.flags =
1569 req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1570
1571 rctx->fallback_req.nbytes = req->nbytes;
1572 rctx->fallback_req.src = req->src;
1573 rctx->fallback_req.result = req->result;
1574
1575 return crypto_ahash_finup(&rctx->fallback_req);
1576 }
1577
1578 static int sa_sha_import(struct ahash_request *req, const void *in)
1579 {
1580 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1581 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1582 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1583
1584 ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
1585 rctx->fallback_req.base.flags = req->base.flags &
1586 CRYPTO_TFM_REQ_MAY_SLEEP;
1587
1588 return crypto_ahash_import(&rctx->fallback_req, in);
1589 }
1590
1591 static int sa_sha_export(struct ahash_request *req, void *out)
1592 {
1593 struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
1594 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1595 struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
1596 struct ahash_request *subreq = &rctx->fallback_req;
1597
1598 ahash_request_set_tfm(subreq, ctx->fallback.ahash);
1599 subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
1600
1601 return crypto_ahash_export(subreq, out);
1602 }
1603
1604 static int sa_sha1_cra_init(struct crypto_tfm *tfm)
1605 {
1606 struct algo_data ad = { 0 };
1607 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1608
1609 sa_sha_cra_init_alg(tfm, "sha1");
1610
1611 ad.aalg_id = SA_AALG_ID_SHA1;
1612 ad.hash_size = SHA1_DIGEST_SIZE;
1613 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1614
1615 sa_sha_setup(ctx, &ad);
1616
1617 return 0;
1618 }
1619
1620 static int sa_sha256_cra_init(struct crypto_tfm *tfm)
1621 {
1622 struct algo_data ad = { 0 };
1623 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1624
1625 sa_sha_cra_init_alg(tfm, "sha256");
1626
1627 ad.aalg_id = SA_AALG_ID_SHA2_256;
1628 ad.hash_size = SHA256_DIGEST_SIZE;
1629 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1630
1631 sa_sha_setup(ctx, &ad);
1632
1633 return 0;
1634 }
1635
1636 static int sa_sha512_cra_init(struct crypto_tfm *tfm)
1637 {
1638 struct algo_data ad = { 0 };
1639 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1640
1641 sa_sha_cra_init_alg(tfm, "sha512");
1642
1643 ad.aalg_id = SA_AALG_ID_SHA2_512;
1644 ad.hash_size = SHA512_DIGEST_SIZE;
1645 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
1646
1647 sa_sha_setup(ctx, &ad);
1648
1649 return 0;
1650 }
1651
1652 static void sa_sha_cra_exit(struct crypto_tfm *tfm)
1653 {
1654 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
1655 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1656
1657 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1658 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1659 ctx->dec.sc_id, &ctx->dec.sc_phys);
1660
1661 if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
1662 sa_free_ctx_info(&ctx->enc, data);
1663
1664 crypto_free_shash(ctx->shash);
1665 crypto_free_ahash(ctx->fallback.ahash);
1666 }
1667
1668 static void sa_aead_dma_in_callback(void *data)
1669 {
1670 struct sa_rx_data *rxd = (struct sa_rx_data *)data;
1671 struct aead_request *req;
1672 struct crypto_aead *tfm;
1673 unsigned int start;
1674 unsigned int authsize;
1675 u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
1676 size_t pl, ml;
1677 int i;
1678 int err = 0;
1679 u16 auth_len;
1680 u32 *mdptr;
1681
1682 sa_sync_from_device(rxd);
1683 req = container_of(rxd->req, struct aead_request, base);
1684 tfm = crypto_aead_reqtfm(req);
1685 start = req->assoclen + req->cryptlen;
1686 authsize = crypto_aead_authsize(tfm);
1687
1688 mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1689 for (i = 0; i < (authsize / 4); i++)
1690 mdptr[i + 4] = swab32(mdptr[i + 4]);
1691
1692 auth_len = req->assoclen + req->cryptlen;
1693
1694 if (rxd->enc) {
1695 scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1696 1);
1697 } else {
1698 auth_len -= authsize;
1699 start -= authsize;
1700 scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
1701 0);
1702
1703 err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
1704 }
1705
1706 sa_free_sa_rx_data(rxd);
1707
1708 aead_request_complete(req, err);
1709 }
1710
1711 static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
1712 const char *fallback)
1713 {
1714 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1715 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1716 int ret;
1717
1718 memzero_explicit(ctx, sizeof(*ctx));
1719
1720 ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
1721 if (IS_ERR(ctx->shash)) {
1722 dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
1723 return PTR_ERR(ctx->shash);
1724 }
1725
1726 ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
1727 CRYPTO_ALG_NEED_FALLBACK);
1728
1729 if (IS_ERR(ctx->fallback.aead)) {
1730 dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
1731 fallback);
1732 return PTR_ERR(ctx->fallback.aead);
1733 }
1734
1735 crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
1736 crypto_aead_reqsize(ctx->fallback.aead));
1737
1738 ret = sa_init_ctx_info(&ctx->enc, data);
1739 if (ret)
1740 return ret;
1741
1742 ret = sa_init_ctx_info(&ctx->dec, data);
1743 if (ret) {
1744 sa_free_ctx_info(&ctx->enc, data);
1745 return ret;
1746 }
1747
1748 dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1749 __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1750 ctx->dec.sc_id, &ctx->dec.sc_phys);
1751
1752 return ret;
1753 }
1754
1755 static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
1756 {
1757 return sa_cra_init_aead(tfm, "sha1",
1758 "authenc(hmac(sha1-ce),cbc(aes-ce))");
1759 }
1760
1761 static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
1762 {
1763 return sa_cra_init_aead(tfm, "sha256",
1764 "authenc(hmac(sha256-ce),cbc(aes-ce))");
1765 }
1766
1767 static void sa_exit_tfm_aead(struct crypto_aead *tfm)
1768 {
1769 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1770 struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1771
1772 crypto_free_shash(ctx->shash);
1773 crypto_free_aead(ctx->fallback.aead);
1774
1775 sa_free_ctx_info(&ctx->enc, data);
1776 sa_free_ctx_info(&ctx->dec, data);
1777 }
1778
1779 /* AEAD algorithm configuration interface function */
1780 static int sa_aead_setkey(struct crypto_aead *authenc,
1781 const u8 *key, unsigned int keylen,
1782 struct algo_data *ad)
1783 {
1784 struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
1785 struct crypto_authenc_keys keys;
1786 int cmdl_len;
1787 struct sa_cmdl_cfg cfg;
1788 int key_idx;
1789
1790 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1791 return -EINVAL;
1792
1793 /* Convert the key size (16/24/32) to the key size index (0/1/2) */
1794 key_idx = (keys.enckeylen >> 3) - 2;
1795 if (key_idx >= 3)
1796 return -EINVAL;
1797
1798 ad->ctx = ctx;
1799 ad->enc_eng.eng_id = SA_ENG_ID_EM1;
1800 ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1801 ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1802 ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1803 ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
1804 ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
1805 ad->inv_key = true;
1806 ad->keyed_mac = true;
1807 ad->ealg_id = SA_EALG_ID_AES_CBC;
1808 ad->prep_iopad = sa_prepare_iopads;
1809
1810 memset(&cfg, 0, sizeof(cfg));
1811 cfg.enc = true;
1812 cfg.aalg = ad->aalg_id;
1813 cfg.enc_eng_id = ad->enc_eng.eng_id;
1814 cfg.auth_eng_id = ad->auth_eng.eng_id;
1815 cfg.iv_size = crypto_aead_ivsize(authenc);
1816 cfg.akey = keys.authkey;
1817 cfg.akey_len = keys.authkeylen;
1818
1819 /* Setup Encryption Security Context & Command label template */
1820 if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen,
1821 keys.authkey, keys.authkeylen,
1822 ad, 1, &ctx->enc.epib[1]))
1823 return -EINVAL;
1824
1825 cmdl_len = sa_format_cmdl_gen(&cfg,
1826 (u8 *)ctx->enc.cmdl,
1827 &ctx->enc.cmdl_upd_info);
1828 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1829 return -EINVAL;
1830
1831 ctx->enc.cmdl_size = cmdl_len;
1832
1833 /* Setup Decryption Security Context & Command label template */
1834 if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen,
1835 keys.authkey, keys.authkeylen,
1836 ad, 0, &ctx->dec.epib[1]))
1837 return -EINVAL;
1838
1839 cfg.enc = false;
1840 cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
1841 &ctx->dec.cmdl_upd_info);
1842
1843 if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1844 return -EINVAL;
1845
1846 ctx->dec.cmdl_size = cmdl_len;
1847
1848 crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
1849 crypto_aead_set_flags(ctx->fallback.aead,
1850 crypto_aead_get_flags(authenc) &
1851 CRYPTO_TFM_REQ_MASK);
1852 crypto_aead_setkey(ctx->fallback.aead, key, keylen);
1853
1854 return 0;
1855 }
1856
1857 static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1858 {
1859 struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
1860
1861 return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
1862 }
1863
1864 static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
1865 const u8 *key, unsigned int keylen)
1866 {
1867 struct algo_data ad = { 0 };
1868
1869 ad.ealg_id = SA_EALG_ID_AES_CBC;
1870 ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
1871 ad.hash_size = SHA1_DIGEST_SIZE;
1872 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1873
1874 return sa_aead_setkey(authenc, key, keylen, &ad);
1875 }
1876
1877 static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
1878 const u8 *key, unsigned int keylen)
1879 {
1880 struct algo_data ad = { 0 };
1881
1882 ad.ealg_id = SA_EALG_ID_AES_CBC;
1883 ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
1884 ad.hash_size = SHA256_DIGEST_SIZE;
1885 ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1886
1887 return sa_aead_setkey(authenc, key, keylen, &ad);
1888 }
1889
1890 static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
1891 {
1892 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1893 struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1894 struct sa_req sa_req = { 0 };
1895 size_t auth_size, enc_size;
1896
1897 enc_size = req->cryptlen;
1898 auth_size = req->assoclen + req->cryptlen;
1899
1900 if (!enc) {
1901 enc_size -= crypto_aead_authsize(tfm);
1902 auth_size -= crypto_aead_authsize(tfm);
1903 }
1904
1905 if (auth_size > SA_MAX_DATA_SZ ||
1906 (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
1907 auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
1908 struct aead_request *subreq = aead_request_ctx(req);
1909 int ret;
1910
1911 aead_request_set_tfm(subreq, ctx->fallback.aead);
1912 aead_request_set_callback(subreq, req->base.flags,
1913 req->base.complete, req->base.data);
1914 aead_request_set_crypt(subreq, req->src, req->dst,
1915 req->cryptlen, req->iv);
1916 aead_request_set_ad(subreq, req->assoclen);
1917
1918 ret = enc ? crypto_aead_encrypt(subreq) :
1919 crypto_aead_decrypt(subreq);
1920 return ret;
1921 }
1922
1923 sa_req.enc_offset = req->assoclen;
1924 sa_req.enc_size = enc_size;
1925 sa_req.auth_size = auth_size;
1926 sa_req.size = auth_size;
1927 sa_req.enc_iv = iv;
1928 sa_req.type = CRYPTO_ALG_TYPE_AEAD;
1929 sa_req.enc = enc;
1930 sa_req.callback = sa_aead_dma_in_callback;
1931 sa_req.mdata_size = 52;
1932 sa_req.base = &req->base;
1933 sa_req.ctx = ctx;
1934 sa_req.src = req->src;
1935 sa_req.dst = req->dst;
1936
1937 return sa_run(&sa_req);
1938 }
1939
1940 /* AEAD algorithm encrypt interface function */
1941 static int sa_aead_encrypt(struct aead_request *req)
1942 {
1943 return sa_aead_run(req, req->iv, 1);
1944 }
1945
1946 /* AEAD algorithm decrypt interface function */
1947 static int sa_aead_decrypt(struct aead_request *req)
1948 {
1949 return sa_aead_run(req, req->iv, 0);
1950 }
1951
1952 static struct sa_alg_tmpl sa_algs[] = {
1953 {
1954 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1955 .alg.skcipher = {
1956 .base.cra_name = "cbc(aes)",
1957 .base.cra_driver_name = "cbc-aes-sa2ul",
1958 .base.cra_priority = 30000,
1959 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
1960 CRYPTO_ALG_KERN_DRIVER_ONLY |
1961 CRYPTO_ALG_ASYNC |
1962 CRYPTO_ALG_NEED_FALLBACK,
1963 .base.cra_blocksize = AES_BLOCK_SIZE,
1964 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
1965 .base.cra_module = THIS_MODULE,
1966 .init = sa_cipher_cra_init,
1967 .exit = sa_cipher_cra_exit,
1968 .min_keysize = AES_MIN_KEY_SIZE,
1969 .max_keysize = AES_MAX_KEY_SIZE,
1970 .ivsize = AES_BLOCK_SIZE,
1971 .setkey = sa_aes_cbc_setkey,
1972 .encrypt = sa_encrypt,
1973 .decrypt = sa_decrypt,
1974 }
1975 },
1976 {
1977 .type = CRYPTO_ALG_TYPE_SKCIPHER,
1978 .alg.skcipher = {
1979 .base.cra_name = "ecb(aes)",
1980 .base.cra_driver_name = "ecb-aes-sa2ul",
1981 .base.cra_priority = 30000,
1982 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
1983 CRYPTO_ALG_KERN_DRIVER_ONLY |
1984 CRYPTO_ALG_ASYNC |
1985 CRYPTO_ALG_NEED_FALLBACK,
1986 .base.cra_blocksize = AES_BLOCK_SIZE,
1987 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
1988 .base.cra_module = THIS_MODULE,
1989 .init = sa_cipher_cra_init,
1990 .exit = sa_cipher_cra_exit,
1991 .min_keysize = AES_MIN_KEY_SIZE,
1992 .max_keysize = AES_MAX_KEY_SIZE,
1993 .setkey = sa_aes_ecb_setkey,
1994 .encrypt = sa_encrypt,
1995 .decrypt = sa_decrypt,
1996 }
1997 },
1998 {
1999 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2000 .alg.skcipher = {
2001 .base.cra_name = "cbc(des3_ede)",
2002 .base.cra_driver_name = "cbc-des3-sa2ul",
2003 .base.cra_priority = 30000,
2004 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2005 CRYPTO_ALG_KERN_DRIVER_ONLY |
2006 CRYPTO_ALG_ASYNC |
2007 CRYPTO_ALG_NEED_FALLBACK,
2008 .base.cra_blocksize = DES_BLOCK_SIZE,
2009 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
2010 .base.cra_module = THIS_MODULE,
2011 .init = sa_cipher_cra_init,
2012 .exit = sa_cipher_cra_exit,
2013 .min_keysize = 3 * DES_KEY_SIZE,
2014 .max_keysize = 3 * DES_KEY_SIZE,
2015 .ivsize = DES_BLOCK_SIZE,
2016 .setkey = sa_3des_cbc_setkey,
2017 .encrypt = sa_encrypt,
2018 .decrypt = sa_decrypt,
2019 }
2020 },
2021 {
2022 .type = CRYPTO_ALG_TYPE_SKCIPHER,
2023 .alg.skcipher = {
2024 .base.cra_name = "ecb(des3_ede)",
2025 .base.cra_driver_name = "ecb-des3-sa2ul",
2026 .base.cra_priority = 30000,
2027 .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
2028 CRYPTO_ALG_KERN_DRIVER_ONLY |
2029 CRYPTO_ALG_ASYNC |
2030 CRYPTO_ALG_NEED_FALLBACK,
2031 .base.cra_blocksize = DES_BLOCK_SIZE,
2032 .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
2033 .base.cra_module = THIS_MODULE,
2034 .init = sa_cipher_cra_init,
2035 .exit = sa_cipher_cra_exit,
2036 .min_keysize = 3 * DES_KEY_SIZE,
2037 .max_keysize = 3 * DES_KEY_SIZE,
2038 .setkey = sa_3des_ecb_setkey,
2039 .encrypt = sa_encrypt,
2040 .decrypt = sa_decrypt,
2041 }
2042 },
2043 {
2044 .type = CRYPTO_ALG_TYPE_AHASH,
2045 .alg.ahash = {
2046 .halg.base = {
2047 .cra_name = "sha1",
2048 .cra_driver_name = "sha1-sa2ul",
2049 .cra_priority = 400,
2050 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2051 CRYPTO_ALG_ASYNC |
2052 CRYPTO_ALG_KERN_DRIVER_ONLY |
2053 CRYPTO_ALG_NEED_FALLBACK,
2054 .cra_blocksize = SHA1_BLOCK_SIZE,
2055 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2056 .cra_module = THIS_MODULE,
2057 .cra_init = sa_sha1_cra_init,
2058 .cra_exit = sa_sha_cra_exit,
2059 },
2060 .halg.digestsize = SHA1_DIGEST_SIZE,
2061 .halg.statesize = sizeof(struct sa_sha_req_ctx) +
2062 sizeof(struct sha1_state),
2063 .init = sa_sha_init,
2064 .update = sa_sha_update,
2065 .final = sa_sha_final,
2066 .finup = sa_sha_finup,
2067 .digest = sa_sha_digest,
2068 .export = sa_sha_export,
2069 .import = sa_sha_import,
2070 },
2071 },
2072 {
2073 .type = CRYPTO_ALG_TYPE_AHASH,
2074 .alg.ahash = {
2075 .halg.base = {
2076 .cra_name = "sha256",
2077 .cra_driver_name = "sha256-sa2ul",
2078 .cra_priority = 400,
2079 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2080 CRYPTO_ALG_ASYNC |
2081 CRYPTO_ALG_KERN_DRIVER_ONLY |
2082 CRYPTO_ALG_NEED_FALLBACK,
2083 .cra_blocksize = SHA256_BLOCK_SIZE,
2084 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2085 .cra_module = THIS_MODULE,
2086 .cra_init = sa_sha256_cra_init,
2087 .cra_exit = sa_sha_cra_exit,
2088 },
2089 .halg.digestsize = SHA256_DIGEST_SIZE,
2090 .halg.statesize = sizeof(struct sa_sha_req_ctx) +
2091 sizeof(struct sha256_state),
2092 .init = sa_sha_init,
2093 .update = sa_sha_update,
2094 .final = sa_sha_final,
2095 .finup = sa_sha_finup,
2096 .digest = sa_sha_digest,
2097 .export = sa_sha_export,
2098 .import = sa_sha_import,
2099 },
2100 },
2101 {
2102 .type = CRYPTO_ALG_TYPE_AHASH,
2103 .alg.ahash = {
2104 .halg.base = {
2105 .cra_name = "sha512",
2106 .cra_driver_name = "sha512-sa2ul",
2107 .cra_priority = 400,
2108 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2109 CRYPTO_ALG_ASYNC |
2110 CRYPTO_ALG_KERN_DRIVER_ONLY |
2111 CRYPTO_ALG_NEED_FALLBACK,
2112 .cra_blocksize = SHA512_BLOCK_SIZE,
2113 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2114 .cra_module = THIS_MODULE,
2115 .cra_init = sa_sha512_cra_init,
2116 .cra_exit = sa_sha_cra_exit,
2117 },
2118 .halg.digestsize = SHA512_DIGEST_SIZE,
2119 .halg.statesize = sizeof(struct sa_sha_req_ctx) +
2120 sizeof(struct sha512_state),
2121 .init = sa_sha_init,
2122 .update = sa_sha_update,
2123 .final = sa_sha_final,
2124 .finup = sa_sha_finup,
2125 .digest = sa_sha_digest,
2126 .export = sa_sha_export,
2127 .import = sa_sha_import,
2128 },
2129 },
2130 {
2131 .type = CRYPTO_ALG_TYPE_AEAD,
2132 .alg.aead = {
2133 .base = {
2134 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2135 .cra_driver_name =
2136 "authenc(hmac(sha1),cbc(aes))-sa2ul",
2137 .cra_blocksize = AES_BLOCK_SIZE,
2138 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2139 CRYPTO_ALG_KERN_DRIVER_ONLY |
2140 CRYPTO_ALG_ASYNC |
2141 CRYPTO_ALG_NEED_FALLBACK,
2142 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2143 .cra_module = THIS_MODULE,
2144 .cra_priority = 3000,
2145 },
2146 .ivsize = AES_BLOCK_SIZE,
2147 .maxauthsize = SHA1_DIGEST_SIZE,
2148
2149 .init = sa_cra_init_aead_sha1,
2150 .exit = sa_exit_tfm_aead,
2151 .setkey = sa_aead_cbc_sha1_setkey,
2152 .setauthsize = sa_aead_setauthsize,
2153 .encrypt = sa_aead_encrypt,
2154 .decrypt = sa_aead_decrypt,
2155 },
2156 },
2157 {
2158 .type = CRYPTO_ALG_TYPE_AEAD,
2159 .alg.aead = {
2160 .base = {
2161 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2162 .cra_driver_name =
2163 "authenc(hmac(sha256),cbc(aes))-sa2ul",
2164 .cra_blocksize = AES_BLOCK_SIZE,
2165 .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2166 CRYPTO_ALG_KERN_DRIVER_ONLY |
2167 CRYPTO_ALG_ASYNC |
2168 CRYPTO_ALG_NEED_FALLBACK,
2169 .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2170 .cra_module = THIS_MODULE,
2171 .cra_alignmask = 0,
2172 .cra_priority = 3000,
2173 },
2174 .ivsize = AES_BLOCK_SIZE,
2175 .maxauthsize = SHA256_DIGEST_SIZE,
2176
2177 .init = sa_cra_init_aead_sha256,
2178 .exit = sa_exit_tfm_aead,
2179 .setkey = sa_aead_cbc_sha256_setkey,
2180 .setauthsize = sa_aead_setauthsize,
2181 .encrypt = sa_aead_encrypt,
2182 .decrypt = sa_aead_decrypt,
2183 },
2184 },
2185 };
2186
2187 /* Register the algorithms in crypto framework */
2188 static void sa_register_algos(const struct device *dev)
2189 {
2190 char *alg_name;
2191 u32 type;
2192 int i, err;
2193
2194 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2195 type = sa_algs[i].type;
2196 if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
2197 alg_name = sa_algs[i].alg.skcipher.base.cra_name;
2198 err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
2199 } else if (type == CRYPTO_ALG_TYPE_AHASH) {
2200 alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
2201 err = crypto_register_ahash(&sa_algs[i].alg.ahash);
2202 } else if (type == CRYPTO_ALG_TYPE_AEAD) {
2203 alg_name = sa_algs[i].alg.aead.base.cra_name;
2204 err = crypto_register_aead(&sa_algs[i].alg.aead);
2205 } else {
2206 dev_err(dev,
2207 "un-supported crypto algorithm (%d)",
2208 sa_algs[i].type);
2209 continue;
2210 }
2211
2212 if (err)
2213 dev_err(dev, "Failed to register '%s'\n", alg_name);
2214 else
2215 sa_algs[i].registered = true;
2216 }
2217 }
2218
2219 /* Unregister the algorithms in crypto framework */
2220 static void sa_unregister_algos(const struct device *dev)
2221 {
2222 u32 type;
2223 int i;
2224
2225 for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
2226 type = sa_algs[i].type;
2227 if (!sa_algs[i].registered)
2228 continue;
2229 if (type == CRYPTO_ALG_TYPE_SKCIPHER)
2230 crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
2231 else if (type == CRYPTO_ALG_TYPE_AHASH)
2232 crypto_unregister_ahash(&sa_algs[i].alg.ahash);
2233 else if (type == CRYPTO_ALG_TYPE_AEAD)
2234 crypto_unregister_aead(&sa_algs[i].alg.aead);
2235
2236 sa_algs[i].registered = false;
2237 }
2238 }
2239
2240 static int sa_init_mem(struct sa_crypto_data *dev_data)
2241 {
2242 struct device *dev = &dev_data->pdev->dev;
2243 /* Setup dma pool for security context buffers */
2244 dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
2245 SA_CTX_MAX_SZ, 64, 0);
2246 if (!dev_data->sc_pool) {
2247 dev_err(dev, "Failed to create dma pool");
2248 return -ENOMEM;
2249 }
2250
2251 return 0;
2252 }
2253
2254 static int sa_dma_init(struct sa_crypto_data *dd)
2255 {
2256 int ret;
2257 struct dma_slave_config cfg;
2258
2259 dd->dma_rx1 = NULL;
2260 dd->dma_tx = NULL;
2261 dd->dma_rx2 = NULL;
2262
2263 ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
2264 if (ret)
2265 return ret;
2266
2267 dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
2268 if (IS_ERR(dd->dma_rx1))
2269 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
2270 "Unable to request rx1 DMA channel\n");
2271
2272 dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
2273 if (IS_ERR(dd->dma_rx2)) {
2274 dma_release_channel(dd->dma_rx1);
2275 return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
2276 "Unable to request rx2 DMA channel\n");
2277 }
2278
2279 dd->dma_tx = dma_request_chan(dd->dev, "tx");
2280 if (IS_ERR(dd->dma_tx)) {
2281 ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
2282 "Unable to request tx DMA channel\n");
2283 goto err_dma_tx;
2284 }
2285
2286 memzero_explicit(&cfg, sizeof(cfg));
2287
2288 cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2289 cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2290 cfg.src_maxburst = 4;
2291 cfg.dst_maxburst = 4;
2292
2293 ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
2294 if (ret) {
2295 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2296 ret);
2297 return ret;
2298 }
2299
2300 ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
2301 if (ret) {
2302 dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
2303 ret);
2304 return ret;
2305 }
2306
2307 ret = dmaengine_slave_config(dd->dma_tx, &cfg);
2308 if (ret) {
2309 dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
2310 ret);
2311 return ret;
2312 }
2313
2314 return 0;
2315
2316 err_dma_tx:
2317 dma_release_channel(dd->dma_rx1);
2318 dma_release_channel(dd->dma_rx2);
2319
2320 return ret;
2321 }
2322
2323 static int sa_link_child(struct device *dev, void *data)
2324 {
2325 struct device *parent = data;
2326
2327 device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
2328
2329 return 0;
2330 }
2331
2332 static int sa_ul_probe(struct platform_device *pdev)
2333 {
2334 struct device *dev = &pdev->dev;
2335 struct device_node *node = dev->of_node;
2336 struct resource *res;
2337 static void __iomem *saul_base;
2338 struct sa_crypto_data *dev_data;
2339 u32 val;
2340 int ret;
2341
2342 dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
2343 if (!dev_data)
2344 return -ENOMEM;
2345
2346 sa_k3_dev = dev;
2347 dev_data->dev = dev;
2348 dev_data->pdev = pdev;
2349 platform_set_drvdata(pdev, dev_data);
2350 dev_set_drvdata(sa_k3_dev, dev_data);
2351
2352 pm_runtime_enable(dev);
2353 ret = pm_runtime_get_sync(dev);
2354 if (ret < 0) {
2355 dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__,
2356 ret);
2357 return ret;
2358 }
2359
2360 sa_init_mem(dev_data);
2361 ret = sa_dma_init(dev_data);
2362 if (ret)
2363 goto disable_pm_runtime;
2364
2365 spin_lock_init(&dev_data->scid_lock);
2366 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2367 saul_base = devm_ioremap_resource(dev, res);
2368
2369 dev_data->base = saul_base;
2370 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
2371 SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
2372 SA_EEC_TRNG_EN;
2373
2374 writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
2375
2376 sa_register_algos(dev);
2377
2378 ret = of_platform_populate(node, NULL, NULL, &pdev->dev);
2379 if (ret)
2380 goto release_dma;
2381
2382 device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child);
2383
2384 return 0;
2385
2386 release_dma:
2387 sa_unregister_algos(&pdev->dev);
2388
2389 dma_release_channel(dev_data->dma_rx2);
2390 dma_release_channel(dev_data->dma_rx1);
2391 dma_release_channel(dev_data->dma_tx);
2392
2393 dma_pool_destroy(dev_data->sc_pool);
2394
2395 disable_pm_runtime:
2396 pm_runtime_put_sync(&pdev->dev);
2397 pm_runtime_disable(&pdev->dev);
2398
2399 return ret;
2400 }
2401
2402 static int sa_ul_remove(struct platform_device *pdev)
2403 {
2404 struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
2405
2406 sa_unregister_algos(&pdev->dev);
2407
2408 dma_release_channel(dev_data->dma_rx2);
2409 dma_release_channel(dev_data->dma_rx1);
2410 dma_release_channel(dev_data->dma_tx);
2411
2412 dma_pool_destroy(dev_data->sc_pool);
2413
2414 platform_set_drvdata(pdev, NULL);
2415
2416 pm_runtime_put_sync(&pdev->dev);
2417 pm_runtime_disable(&pdev->dev);
2418
2419 return 0;
2420 }
2421
2422 static const struct of_device_id of_match[] = {
2423 {.compatible = "ti,j721e-sa2ul",},
2424 {.compatible = "ti,am654-sa2ul",},
2425 {},
2426 };
2427 MODULE_DEVICE_TABLE(of, of_match);
2428
2429 static struct platform_driver sa_ul_driver = {
2430 .probe = sa_ul_probe,
2431 .remove = sa_ul_remove,
2432 .driver = {
2433 .name = "saul-crypto",
2434 .of_match_table = of_match,
2435 },
2436 };
2437 module_platform_driver(sa_ul_driver);
2438 MODULE_LICENSE("GPL v2");
Linux with added FPC-III support