?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
blk: rq_data_dir() should not return a boolean
[cris-mirror.git] / arch / arm / crypto / aesbs-glue.c
blob6d685298690ed228b416c7cf67c58e8b6d6f2909
1 /*
2 * linux/arch/arm/crypto/aesbs-glue.c - glue code for NEON bit sliced AES
3 *
4 * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11 #include <asm/neon.h>
12 #include <crypto/aes.h>
13 #include <crypto/ablk_helper.h>
14 #include <crypto/algapi.h>
15 #include <linux/module.h>
16
17 #include "aes_glue.h"
18
19 #define BIT_SLICED_KEY_MAXSIZE (128 * (AES_MAXNR - 1) + 2 * AES_BLOCK_SIZE)
20
21 struct BS_KEY {
22 struct AES_KEY rk;
23 int converted;
24 u8 __aligned(8) bs[BIT_SLICED_KEY_MAXSIZE];
25 } __aligned(8);
26
27 asmlinkage void bsaes_enc_key_convert(u8 out[], struct AES_KEY const *in);
28 asmlinkage void bsaes_dec_key_convert(u8 out[], struct AES_KEY const *in);
29
30 asmlinkage void bsaes_cbc_encrypt(u8 const in[], u8 out[], u32 bytes,
31 struct BS_KEY *key, u8 iv[]);
32
33 asmlinkage void bsaes_ctr32_encrypt_blocks(u8 const in[], u8 out[], u32 blocks,
34 struct BS_KEY *key, u8 const iv[]);
35
36 asmlinkage void bsaes_xts_encrypt(u8 const in[], u8 out[], u32 bytes,
37 struct BS_KEY *key, u8 tweak[]);
38
39 asmlinkage void bsaes_xts_decrypt(u8 const in[], u8 out[], u32 bytes,
40 struct BS_KEY *key, u8 tweak[]);
41
42 struct aesbs_cbc_ctx {
43 struct AES_KEY enc;
44 struct BS_KEY dec;
45 };
46
47 struct aesbs_ctr_ctx {
48 struct BS_KEY enc;
49 };
50
51 struct aesbs_xts_ctx {
52 struct BS_KEY enc;
53 struct BS_KEY dec;
54 struct AES_KEY twkey;
55 };
56
57 static int aesbs_cbc_set_key(struct crypto_tfm *tfm, const u8 *in_key,
58 unsigned int key_len)
59 {
60 struct aesbs_cbc_ctx *ctx = crypto_tfm_ctx(tfm);
61 int bits = key_len * 8;
62
63 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc)) {
64 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
65 return -EINVAL;
66 }
67 ctx->dec.rk = ctx->enc;
68 private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
69 ctx->dec.converted = 0;
70 return 0;
71 }
72
73 static int aesbs_ctr_set_key(struct crypto_tfm *tfm, const u8 *in_key,
74 unsigned int key_len)
75 {
76 struct aesbs_ctr_ctx *ctx = crypto_tfm_ctx(tfm);
77 int bits = key_len * 8;
78
79 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
80 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
81 return -EINVAL;
82 }
83 ctx->enc.converted = 0;
84 return 0;
85 }
86
87 static int aesbs_xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
88 unsigned int key_len)
89 {
90 struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
91 int bits = key_len * 4;
92
93 if (private_AES_set_encrypt_key(in_key, bits, &ctx->enc.rk)) {
94 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
95 return -EINVAL;
96 }
97 ctx->dec.rk = ctx->enc.rk;
98 private_AES_set_decrypt_key(in_key, bits, &ctx->dec.rk);
99 private_AES_set_encrypt_key(in_key + key_len / 2, bits, &ctx->twkey);
100 ctx->enc.converted = ctx->dec.converted = 0;
101 return 0;
102 }
103
104 static int aesbs_cbc_encrypt(struct blkcipher_desc *desc,
105 struct scatterlist *dst,
106 struct scatterlist *src, unsigned int nbytes)
107 {
108 struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
109 struct blkcipher_walk walk;
110 int err;
111
112 blkcipher_walk_init(&walk, dst, src, nbytes);
113 err = blkcipher_walk_virt(desc, &walk);
114
115 while (walk.nbytes) {
116 u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
117 u8 *src = walk.src.virt.addr;
118
119 if (walk.dst.virt.addr == walk.src.virt.addr) {
120 u8 *iv = walk.iv;
121
122 do {
123 crypto_xor(src, iv, AES_BLOCK_SIZE);
124 AES_encrypt(src, src, &ctx->enc);
125 iv = src;
126 src += AES_BLOCK_SIZE;
127 } while (--blocks);
128 memcpy(walk.iv, iv, AES_BLOCK_SIZE);
129 } else {
130 u8 *dst = walk.dst.virt.addr;
131
132 do {
133 crypto_xor(walk.iv, src, AES_BLOCK_SIZE);
134 AES_encrypt(walk.iv, dst, &ctx->enc);
135 memcpy(walk.iv, dst, AES_BLOCK_SIZE);
136 src += AES_BLOCK_SIZE;
137 dst += AES_BLOCK_SIZE;
138 } while (--blocks);
139 }
140 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
141 }
142 return err;
143 }
144
145 static int aesbs_cbc_decrypt(struct blkcipher_desc *desc,
146 struct scatterlist *dst,
147 struct scatterlist *src, unsigned int nbytes)
148 {
149 struct aesbs_cbc_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
150 struct blkcipher_walk walk;
151 int err;
152
153 blkcipher_walk_init(&walk, dst, src, nbytes);
154 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
155
156 while ((walk.nbytes / AES_BLOCK_SIZE) >= 8) {
157 kernel_neon_begin();
158 bsaes_cbc_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
159 walk.nbytes, &ctx->dec, walk.iv);
160 kernel_neon_end();
161 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
162 }
163 while (walk.nbytes) {
164 u32 blocks = walk.nbytes / AES_BLOCK_SIZE;
165 u8 *dst = walk.dst.virt.addr;
166 u8 *src = walk.src.virt.addr;
167 u8 bk[2][AES_BLOCK_SIZE];
168 u8 *iv = walk.iv;
169
170 do {
171 if (walk.dst.virt.addr == walk.src.virt.addr)
172 memcpy(bk[blocks & 1], src, AES_BLOCK_SIZE);
173
174 AES_decrypt(src, dst, &ctx->dec.rk);
175 crypto_xor(dst, iv, AES_BLOCK_SIZE);
176
177 if (walk.dst.virt.addr == walk.src.virt.addr)
178 iv = bk[blocks & 1];
179 else
180 iv = src;
181
182 dst += AES_BLOCK_SIZE;
183 src += AES_BLOCK_SIZE;
184 } while (--blocks);
185 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
186 }
187 return err;
188 }
189
190 static void inc_be128_ctr(__be32 ctr[], u32 addend)
191 {
192 int i;
193
194 for (i = 3; i >= 0; i--, addend = 1) {
195 u32 n = be32_to_cpu(ctr[i]) + addend;
196
197 ctr[i] = cpu_to_be32(n);
198 if (n >= addend)
199 break;
200 }
201 }
202
203 static int aesbs_ctr_encrypt(struct blkcipher_desc *desc,
204 struct scatterlist *dst, struct scatterlist *src,
205 unsigned int nbytes)
206 {
207 struct aesbs_ctr_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
208 struct blkcipher_walk walk;
209 u32 blocks;
210 int err;
211
212 blkcipher_walk_init(&walk, dst, src, nbytes);
213 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
214
215 while ((blocks = walk.nbytes / AES_BLOCK_SIZE)) {
216 u32 tail = walk.nbytes % AES_BLOCK_SIZE;
217 __be32 *ctr = (__be32 *)walk.iv;
218 u32 headroom = UINT_MAX - be32_to_cpu(ctr[3]);
219
220 /* avoid 32 bit counter overflow in the NEON code */
221 if (unlikely(headroom < blocks)) {
222 blocks = headroom + 1;
223 tail = walk.nbytes - blocks * AES_BLOCK_SIZE;
224 }
225 kernel_neon_begin();
226 bsaes_ctr32_encrypt_blocks(walk.src.virt.addr,
227 walk.dst.virt.addr, blocks,
228 &ctx->enc, walk.iv);
229 kernel_neon_end();
230 inc_be128_ctr(ctr, blocks);
231
232 nbytes -= blocks * AES_BLOCK_SIZE;
233 if (nbytes && nbytes == tail && nbytes <= AES_BLOCK_SIZE)
234 break;
235
236 err = blkcipher_walk_done(desc, &walk, tail);
237 }
238 if (walk.nbytes) {
239 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
240 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
241 u8 ks[AES_BLOCK_SIZE];
242
243 AES_encrypt(walk.iv, ks, &ctx->enc.rk);
244 if (tdst != tsrc)
245 memcpy(tdst, tsrc, nbytes);
246 crypto_xor(tdst, ks, nbytes);
247 err = blkcipher_walk_done(desc, &walk, 0);
248 }
249 return err;
250 }
251
252 static int aesbs_xts_encrypt(struct blkcipher_desc *desc,
253 struct scatterlist *dst,
254 struct scatterlist *src, unsigned int nbytes)
255 {
256 struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
257 struct blkcipher_walk walk;
258 int err;
259
260 blkcipher_walk_init(&walk, dst, src, nbytes);
261 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
262
263 /* generate the initial tweak */
264 AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
265
266 while (walk.nbytes) {
267 kernel_neon_begin();
268 bsaes_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr,
269 walk.nbytes, &ctx->enc, walk.iv);
270 kernel_neon_end();
271 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
272 }
273 return err;
274 }
275
276 static int aesbs_xts_decrypt(struct blkcipher_desc *desc,
277 struct scatterlist *dst,
278 struct scatterlist *src, unsigned int nbytes)
279 {
280 struct aesbs_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
281 struct blkcipher_walk walk;
282 int err;
283
284 blkcipher_walk_init(&walk, dst, src, nbytes);
285 err = blkcipher_walk_virt_block(desc, &walk, 8 * AES_BLOCK_SIZE);
286
287 /* generate the initial tweak */
288 AES_encrypt(walk.iv, walk.iv, &ctx->twkey);
289
290 while (walk.nbytes) {
291 kernel_neon_begin();
292 bsaes_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr,
293 walk.nbytes, &ctx->dec, walk.iv);
294 kernel_neon_end();
295 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
296 }
297 return err;
298 }
299
300 static struct crypto_alg aesbs_algs[] = { {
301 .cra_name = "__cbc-aes-neonbs",
302 .cra_driver_name = "__driver-cbc-aes-neonbs",
303 .cra_priority = 0,
304 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
305 CRYPTO_ALG_INTERNAL,
306 .cra_blocksize = AES_BLOCK_SIZE,
307 .cra_ctxsize = sizeof(struct aesbs_cbc_ctx),
308 .cra_alignmask = 7,
309 .cra_type = &crypto_blkcipher_type,
310 .cra_module = THIS_MODULE,
311 .cra_blkcipher = {
312 .min_keysize = AES_MIN_KEY_SIZE,
313 .max_keysize = AES_MAX_KEY_SIZE,
314 .ivsize = AES_BLOCK_SIZE,
315 .setkey = aesbs_cbc_set_key,
316 .encrypt = aesbs_cbc_encrypt,
317 .decrypt = aesbs_cbc_decrypt,
318 },
319 }, {
320 .cra_name = "__ctr-aes-neonbs",
321 .cra_driver_name = "__driver-ctr-aes-neonbs",
322 .cra_priority = 0,
323 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
324 CRYPTO_ALG_INTERNAL,
325 .cra_blocksize = 1,
326 .cra_ctxsize = sizeof(struct aesbs_ctr_ctx),
327 .cra_alignmask = 7,
328 .cra_type = &crypto_blkcipher_type,
329 .cra_module = THIS_MODULE,
330 .cra_blkcipher = {
331 .min_keysize = AES_MIN_KEY_SIZE,
332 .max_keysize = AES_MAX_KEY_SIZE,
333 .ivsize = AES_BLOCK_SIZE,
334 .setkey = aesbs_ctr_set_key,
335 .encrypt = aesbs_ctr_encrypt,
336 .decrypt = aesbs_ctr_encrypt,
337 },
338 }, {
339 .cra_name = "__xts-aes-neonbs",
340 .cra_driver_name = "__driver-xts-aes-neonbs",
341 .cra_priority = 0,
342 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER |
343 CRYPTO_ALG_INTERNAL,
344 .cra_blocksize = AES_BLOCK_SIZE,
345 .cra_ctxsize = sizeof(struct aesbs_xts_ctx),
346 .cra_alignmask = 7,
347 .cra_type = &crypto_blkcipher_type,
348 .cra_module = THIS_MODULE,
349 .cra_blkcipher = {
350 .min_keysize = 2 * AES_MIN_KEY_SIZE,
351 .max_keysize = 2 * AES_MAX_KEY_SIZE,
352 .ivsize = AES_BLOCK_SIZE,
353 .setkey = aesbs_xts_set_key,
354 .encrypt = aesbs_xts_encrypt,
355 .decrypt = aesbs_xts_decrypt,
356 },
357 }, {
358 .cra_name = "cbc(aes)",
359 .cra_driver_name = "cbc-aes-neonbs",
360 .cra_priority = 300,
361 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
362 .cra_blocksize = AES_BLOCK_SIZE,
363 .cra_ctxsize = sizeof(struct async_helper_ctx),
364 .cra_alignmask = 7,
365 .cra_type = &crypto_ablkcipher_type,
366 .cra_module = THIS_MODULE,
367 .cra_init = ablk_init,
368 .cra_exit = ablk_exit,
369 .cra_ablkcipher = {
370 .min_keysize = AES_MIN_KEY_SIZE,
371 .max_keysize = AES_MAX_KEY_SIZE,
372 .ivsize = AES_BLOCK_SIZE,
373 .setkey = ablk_set_key,
374 .encrypt = __ablk_encrypt,
375 .decrypt = ablk_decrypt,
376 }
377 }, {
378 .cra_name = "ctr(aes)",
379 .cra_driver_name = "ctr-aes-neonbs",
380 .cra_priority = 300,
381 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
382 .cra_blocksize = 1,
383 .cra_ctxsize = sizeof(struct async_helper_ctx),
384 .cra_alignmask = 7,
385 .cra_type = &crypto_ablkcipher_type,
386 .cra_module = THIS_MODULE,
387 .cra_init = ablk_init,
388 .cra_exit = ablk_exit,
389 .cra_ablkcipher = {
390 .min_keysize = AES_MIN_KEY_SIZE,
391 .max_keysize = AES_MAX_KEY_SIZE,
392 .ivsize = AES_BLOCK_SIZE,
393 .setkey = ablk_set_key,
394 .encrypt = ablk_encrypt,
395 .decrypt = ablk_decrypt,
396 }
397 }, {
398 .cra_name = "xts(aes)",
399 .cra_driver_name = "xts-aes-neonbs",
400 .cra_priority = 300,
401 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
402 .cra_blocksize = AES_BLOCK_SIZE,
403 .cra_ctxsize = sizeof(struct async_helper_ctx),
404 .cra_alignmask = 7,
405 .cra_type = &crypto_ablkcipher_type,
406 .cra_module = THIS_MODULE,
407 .cra_init = ablk_init,
408 .cra_exit = ablk_exit,
409 .cra_ablkcipher = {
410 .min_keysize = 2 * AES_MIN_KEY_SIZE,
411 .max_keysize = 2 * AES_MAX_KEY_SIZE,
412 .ivsize = AES_BLOCK_SIZE,
413 .setkey = ablk_set_key,
414 .encrypt = ablk_encrypt,
415 .decrypt = ablk_decrypt,
416 }
417 } };
418
419 static int __init aesbs_mod_init(void)
420 {
421 if (!cpu_has_neon())
422 return -ENODEV;
423
424 return crypto_register_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
425 }
426
427 static void __exit aesbs_mod_exit(void)
428 {
429 crypto_unregister_algs(aesbs_algs, ARRAY_SIZE(aesbs_algs));
430 }
431
432 module_init(aesbs_mod_init);
433 module_exit(aesbs_mod_exit);
434
435 MODULE_DESCRIPTION("Bit sliced AES in CBC/CTR/XTS modes using NEON");
436 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
437 MODULE_LICENSE("GPL");
CRIS linux kernel tree