?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / arch / arm / crypto / chacha-glue.c
blob6fdb0ac62b3d85076824ecbb93eae1139bd79a57
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * ARM NEON accelerated ChaCha and XChaCha stream ciphers,
4 * including ChaCha20 (RFC7539)
5 *
6 * Copyright (C) 2016-2019 Linaro, Ltd. <ard.biesheuvel@linaro.org>
7 * Copyright (C) 2015 Martin Willi
8 */
9
10 #include <crypto/algapi.h>
11 #include <crypto/internal/chacha.h>
12 #include <crypto/internal/simd.h>
13 #include <crypto/internal/skcipher.h>
14 #include <linux/jump_label.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17
18 #include <asm/cputype.h>
19 #include <asm/hwcap.h>
20 #include <asm/neon.h>
21 #include <asm/simd.h>
22
23 asmlinkage void chacha_block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
24 int nrounds);
25 asmlinkage void chacha_4block_xor_neon(const u32 *state, u8 *dst, const u8 *src,
26 int nrounds);
27 asmlinkage void hchacha_block_arm(const u32 *state, u32 *out, int nrounds);
28 asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
29
30 asmlinkage void chacha_doarm(u8 *dst, const u8 *src, unsigned int bytes,
31 const u32 *state, int nrounds);
32
33 static __ro_after_init DEFINE_STATIC_KEY_FALSE(use_neon);
34
35 static inline bool neon_usable(void)
36 {
37 return static_branch_likely(&use_neon) && crypto_simd_usable();
38 }
39
40 static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
41 unsigned int bytes, int nrounds)
42 {
43 u8 buf[CHACHA_BLOCK_SIZE];
44
45 while (bytes >= CHACHA_BLOCK_SIZE * 4) {
46 chacha_4block_xor_neon(state, dst, src, nrounds);
47 bytes -= CHACHA_BLOCK_SIZE * 4;
48 src += CHACHA_BLOCK_SIZE * 4;
49 dst += CHACHA_BLOCK_SIZE * 4;
50 state[12] += 4;
51 }
52 while (bytes >= CHACHA_BLOCK_SIZE) {
53 chacha_block_xor_neon(state, dst, src, nrounds);
54 bytes -= CHACHA_BLOCK_SIZE;
55 src += CHACHA_BLOCK_SIZE;
56 dst += CHACHA_BLOCK_SIZE;
57 state[12]++;
58 }
59 if (bytes) {
60 memcpy(buf, src, bytes);
61 chacha_block_xor_neon(state, buf, buf, nrounds);
62 memcpy(dst, buf, bytes);
63 }
64 }
65
66 void hchacha_block_arch(const u32 *state, u32 *stream, int nrounds)
67 {
68 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable()) {
69 hchacha_block_arm(state, stream, nrounds);
70 } else {
71 kernel_neon_begin();
72 hchacha_block_neon(state, stream, nrounds);
73 kernel_neon_end();
74 }
75 }
76 EXPORT_SYMBOL(hchacha_block_arch);
77
78 void chacha_init_arch(u32 *state, const u32 *key, const u8 *iv)
79 {
80 chacha_init_generic(state, key, iv);
81 }
82 EXPORT_SYMBOL(chacha_init_arch);
83
84 void chacha_crypt_arch(u32 *state, u8 *dst, const u8 *src, unsigned int bytes,
85 int nrounds)
86 {
87 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon_usable() ||
88 bytes <= CHACHA_BLOCK_SIZE) {
89 chacha_doarm(dst, src, bytes, state, nrounds);
90 state[12] += DIV_ROUND_UP(bytes, CHACHA_BLOCK_SIZE);
91 return;
92 }
93
94 kernel_neon_begin();
95 chacha_doneon(state, dst, src, bytes, nrounds);
96 kernel_neon_end();
97 }
98 EXPORT_SYMBOL(chacha_crypt_arch);
99
100 static int chacha_stream_xor(struct skcipher_request *req,
101 const struct chacha_ctx *ctx, const u8 *iv,
102 bool neon)
103 {
104 struct skcipher_walk walk;
105 u32 state[16];
106 int err;
107
108 err = skcipher_walk_virt(&walk, req, false);
109
110 chacha_init_generic(state, ctx->key, iv);
111
112 while (walk.nbytes > 0) {
113 unsigned int nbytes = walk.nbytes;
114
115 if (nbytes < walk.total)
116 nbytes = round_down(nbytes, walk.stride);
117
118 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
119 chacha_doarm(walk.dst.virt.addr, walk.src.virt.addr,
120 nbytes, state, ctx->nrounds);
121 state[12] += DIV_ROUND_UP(nbytes, CHACHA_BLOCK_SIZE);
122 } else {
123 kernel_neon_begin();
124 chacha_doneon(state, walk.dst.virt.addr,
125 walk.src.virt.addr, nbytes, ctx->nrounds);
126 kernel_neon_end();
127 }
128 err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
129 }
130
131 return err;
132 }
133
134 static int do_chacha(struct skcipher_request *req, bool neon)
135 {
136 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
137 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
138
139 return chacha_stream_xor(req, ctx, req->iv, neon);
140 }
141
142 static int chacha_arm(struct skcipher_request *req)
143 {
144 return do_chacha(req, false);
145 }
146
147 static int chacha_neon(struct skcipher_request *req)
148 {
149 return do_chacha(req, neon_usable());
150 }
151
152 static int do_xchacha(struct skcipher_request *req, bool neon)
153 {
154 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
155 struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
156 struct chacha_ctx subctx;
157 u32 state[16];
158 u8 real_iv[16];
159
160 chacha_init_generic(state, ctx->key, req->iv);
161
162 if (!IS_ENABLED(CONFIG_KERNEL_MODE_NEON) || !neon) {
163 hchacha_block_arm(state, subctx.key, ctx->nrounds);
164 } else {
165 kernel_neon_begin();
166 hchacha_block_neon(state, subctx.key, ctx->nrounds);
167 kernel_neon_end();
168 }
169 subctx.nrounds = ctx->nrounds;
170
171 memcpy(&real_iv[0], req->iv + 24, 8);
172 memcpy(&real_iv[8], req->iv + 16, 8);
173 return chacha_stream_xor(req, &subctx, real_iv, neon);
174 }
175
176 static int xchacha_arm(struct skcipher_request *req)
177 {
178 return do_xchacha(req, false);
179 }
180
181 static int xchacha_neon(struct skcipher_request *req)
182 {
183 return do_xchacha(req, neon_usable());
184 }
185
186 static struct skcipher_alg arm_algs[] = {
187 {
188 .base.cra_name = "chacha20",
189 .base.cra_driver_name = "chacha20-arm",
190 .base.cra_priority = 200,
191 .base.cra_blocksize = 1,
192 .base.cra_ctxsize = sizeof(struct chacha_ctx),
193 .base.cra_module = THIS_MODULE,
194
195 .min_keysize = CHACHA_KEY_SIZE,
196 .max_keysize = CHACHA_KEY_SIZE,
197 .ivsize = CHACHA_IV_SIZE,
198 .chunksize = CHACHA_BLOCK_SIZE,
199 .setkey = chacha20_setkey,
200 .encrypt = chacha_arm,
201 .decrypt = chacha_arm,
202 }, {
203 .base.cra_name = "xchacha20",
204 .base.cra_driver_name = "xchacha20-arm",
205 .base.cra_priority = 200,
206 .base.cra_blocksize = 1,
207 .base.cra_ctxsize = sizeof(struct chacha_ctx),
208 .base.cra_module = THIS_MODULE,
209
210 .min_keysize = CHACHA_KEY_SIZE,
211 .max_keysize = CHACHA_KEY_SIZE,
212 .ivsize = XCHACHA_IV_SIZE,
213 .chunksize = CHACHA_BLOCK_SIZE,
214 .setkey = chacha20_setkey,
215 .encrypt = xchacha_arm,
216 .decrypt = xchacha_arm,
217 }, {
218 .base.cra_name = "xchacha12",
219 .base.cra_driver_name = "xchacha12-arm",
220 .base.cra_priority = 200,
221 .base.cra_blocksize = 1,
222 .base.cra_ctxsize = sizeof(struct chacha_ctx),
223 .base.cra_module = THIS_MODULE,
224
225 .min_keysize = CHACHA_KEY_SIZE,
226 .max_keysize = CHACHA_KEY_SIZE,
227 .ivsize = XCHACHA_IV_SIZE,
228 .chunksize = CHACHA_BLOCK_SIZE,
229 .setkey = chacha12_setkey,
230 .encrypt = xchacha_arm,
231 .decrypt = xchacha_arm,
232 },
233 };
234
235 static struct skcipher_alg neon_algs[] = {
236 {
237 .base.cra_name = "chacha20",
238 .base.cra_driver_name = "chacha20-neon",
239 .base.cra_priority = 300,
240 .base.cra_blocksize = 1,
241 .base.cra_ctxsize = sizeof(struct chacha_ctx),
242 .base.cra_module = THIS_MODULE,
243
244 .min_keysize = CHACHA_KEY_SIZE,
245 .max_keysize = CHACHA_KEY_SIZE,
246 .ivsize = CHACHA_IV_SIZE,
247 .chunksize = CHACHA_BLOCK_SIZE,
248 .walksize = 4 * CHACHA_BLOCK_SIZE,
249 .setkey = chacha20_setkey,
250 .encrypt = chacha_neon,
251 .decrypt = chacha_neon,
252 }, {
253 .base.cra_name = "xchacha20",
254 .base.cra_driver_name = "xchacha20-neon",
255 .base.cra_priority = 300,
256 .base.cra_blocksize = 1,
257 .base.cra_ctxsize = sizeof(struct chacha_ctx),
258 .base.cra_module = THIS_MODULE,
259
260 .min_keysize = CHACHA_KEY_SIZE,
261 .max_keysize = CHACHA_KEY_SIZE,
262 .ivsize = XCHACHA_IV_SIZE,
263 .chunksize = CHACHA_BLOCK_SIZE,
264 .walksize = 4 * CHACHA_BLOCK_SIZE,
265 .setkey = chacha20_setkey,
266 .encrypt = xchacha_neon,
267 .decrypt = xchacha_neon,
268 }, {
269 .base.cra_name = "xchacha12",
270 .base.cra_driver_name = "xchacha12-neon",
271 .base.cra_priority = 300,
272 .base.cra_blocksize = 1,
273 .base.cra_ctxsize = sizeof(struct chacha_ctx),
274 .base.cra_module = THIS_MODULE,
275
276 .min_keysize = CHACHA_KEY_SIZE,
277 .max_keysize = CHACHA_KEY_SIZE,
278 .ivsize = XCHACHA_IV_SIZE,
279 .chunksize = CHACHA_BLOCK_SIZE,
280 .walksize = 4 * CHACHA_BLOCK_SIZE,
281 .setkey = chacha12_setkey,
282 .encrypt = xchacha_neon,
283 .decrypt = xchacha_neon,
284 }
285 };
286
287 static int __init chacha_simd_mod_init(void)
288 {
289 int err = 0;
290
291 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
292 err = crypto_register_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
293 if (err)
294 return err;
295 }
296
297 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
298 int i;
299
300 switch (read_cpuid_part()) {
301 case ARM_CPU_PART_CORTEX_A7:
302 case ARM_CPU_PART_CORTEX_A5:
303 /*
304 * The Cortex-A7 and Cortex-A5 do not perform well with
305 * the NEON implementation but do incredibly with the
306 * scalar one and use less power.
307 */
308 for (i = 0; i < ARRAY_SIZE(neon_algs); i++)
309 neon_algs[i].base.cra_priority = 0;
310 break;
311 default:
312 static_branch_enable(&use_neon);
313 }
314
315 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
316 err = crypto_register_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
317 if (err)
318 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
319 }
320 }
321 return err;
322 }
323
324 static void __exit chacha_simd_mod_fini(void)
325 {
326 if (IS_REACHABLE(CONFIG_CRYPTO_SKCIPHER)) {
327 crypto_unregister_skciphers(arm_algs, ARRAY_SIZE(arm_algs));
328 if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON))
329 crypto_unregister_skciphers(neon_algs, ARRAY_SIZE(neon_algs));
330 }
331 }
332
333 module_init(chacha_simd_mod_init);
334 module_exit(chacha_simd_mod_fini);
335
336 MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (scalar and NEON accelerated)");
337 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
338 MODULE_LICENSE("GPL v2");
339 MODULE_ALIAS_CRYPTO("chacha20");
340 MODULE_ALIAS_CRYPTO("chacha20-arm");
341 MODULE_ALIAS_CRYPTO("xchacha20");
342 MODULE_ALIAS_CRYPTO("xchacha20-arm");
343 MODULE_ALIAS_CRYPTO("xchacha12");
344 MODULE_ALIAS_CRYPTO("xchacha12-arm");
345 #ifdef CONFIG_KERNEL_MODE_NEON
346 MODULE_ALIAS_CRYPTO("chacha20-neon");
347 MODULE_ALIAS_CRYPTO("xchacha20-neon");
348 MODULE_ALIAS_CRYPTO("xchacha12-neon");
349 #endif
Linux with added FPC-III support