4 * Glue code for the SHA1 Secure Hash Algorithm assembler implementation using
5 * Supplemental SSE3 instructions.
7 * This file is based on sha1_generic.c
9 * Copyright (c) Alan Smithee.
10 * Copyright (c) Andrew McDonald <andrew@mcdonald.org.uk>
11 * Copyright (c) Jean-Francois Dive <jef@linuxbe.org>
12 * Copyright (c) Mathias Krause <minipli@googlemail.com>
13 * Copyright (c) Chandramouli Narayanan <mouli@linux.intel.com>
15 * This program is free software; you can redistribute it and/or modify it
16 * under the terms of the GNU General Public License as published by the Free
17 * Software Foundation; either version 2 of the License, or (at your option)
22 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
24 #include <crypto/internal/hash.h>
25 #include <linux/init.h>
26 #include <linux/module.h>
28 #include <linux/cryptohash.h>
29 #include <linux/types.h>
30 #include <crypto/sha.h>
31 #include <crypto/sha1_base.h>
32 #include <asm/fpu/api.h>
34 typedef void (sha1_transform_fn
)(u32
*digest
, const char *data
,
37 static int sha1_update(struct shash_desc
*desc
, const u8
*data
,
38 unsigned int len
, sha1_transform_fn
*sha1_xform
)
40 struct sha1_state
*sctx
= shash_desc_ctx(desc
);
42 if (!irq_fpu_usable() ||
43 (sctx
->count
% SHA1_BLOCK_SIZE
) + len
< SHA1_BLOCK_SIZE
)
44 return crypto_sha1_update(desc
, data
, len
);
46 /* make sure casting to sha1_block_fn() is safe */
47 BUILD_BUG_ON(offsetof(struct sha1_state
, state
) != 0);
50 sha1_base_do_update(desc
, data
, len
,
51 (sha1_block_fn
*)sha1_xform
);
57 static int sha1_finup(struct shash_desc
*desc
, const u8
*data
,
58 unsigned int len
, u8
*out
, sha1_transform_fn
*sha1_xform
)
60 if (!irq_fpu_usable())
61 return crypto_sha1_finup(desc
, data
, len
, out
);
65 sha1_base_do_update(desc
, data
, len
,
66 (sha1_block_fn
*)sha1_xform
);
67 sha1_base_do_finalize(desc
, (sha1_block_fn
*)sha1_xform
);
70 return sha1_base_finish(desc
, out
);
73 asmlinkage
void sha1_transform_ssse3(u32
*digest
, const char *data
,
76 static int sha1_ssse3_update(struct shash_desc
*desc
, const u8
*data
,
79 return sha1_update(desc
, data
, len
,
80 (sha1_transform_fn
*) sha1_transform_ssse3
);
83 static int sha1_ssse3_finup(struct shash_desc
*desc
, const u8
*data
,
84 unsigned int len
, u8
*out
)
86 return sha1_finup(desc
, data
, len
, out
,
87 (sha1_transform_fn
*) sha1_transform_ssse3
);
90 /* Add padding and return the message digest. */
91 static int sha1_ssse3_final(struct shash_desc
*desc
, u8
*out
)
93 return sha1_ssse3_finup(desc
, NULL
, 0, out
);
96 static struct shash_alg sha1_ssse3_alg
= {
97 .digestsize
= SHA1_DIGEST_SIZE
,
98 .init
= sha1_base_init
,
99 .update
= sha1_ssse3_update
,
100 .final
= sha1_ssse3_final
,
101 .finup
= sha1_ssse3_finup
,
102 .descsize
= sizeof(struct sha1_state
),
105 .cra_driver_name
= "sha1-ssse3",
107 .cra_flags
= CRYPTO_ALG_TYPE_SHASH
,
108 .cra_blocksize
= SHA1_BLOCK_SIZE
,
109 .cra_module
= THIS_MODULE
,
113 static int register_sha1_ssse3(void)
115 if (boot_cpu_has(X86_FEATURE_SSSE3
))
116 return crypto_register_shash(&sha1_ssse3_alg
);
120 static void unregister_sha1_ssse3(void)
122 if (boot_cpu_has(X86_FEATURE_SSSE3
))
123 crypto_unregister_shash(&sha1_ssse3_alg
);
127 asmlinkage
void sha1_transform_avx(u32
*digest
, const char *data
,
128 unsigned int rounds
);
130 static int sha1_avx_update(struct shash_desc
*desc
, const u8
*data
,
133 return sha1_update(desc
, data
, len
,
134 (sha1_transform_fn
*) sha1_transform_avx
);
137 static int sha1_avx_finup(struct shash_desc
*desc
, const u8
*data
,
138 unsigned int len
, u8
*out
)
140 return sha1_finup(desc
, data
, len
, out
,
141 (sha1_transform_fn
*) sha1_transform_avx
);
144 static int sha1_avx_final(struct shash_desc
*desc
, u8
*out
)
146 return sha1_avx_finup(desc
, NULL
, 0, out
);
149 static struct shash_alg sha1_avx_alg
= {
150 .digestsize
= SHA1_DIGEST_SIZE
,
151 .init
= sha1_base_init
,
152 .update
= sha1_avx_update
,
153 .final
= sha1_avx_final
,
154 .finup
= sha1_avx_finup
,
155 .descsize
= sizeof(struct sha1_state
),
158 .cra_driver_name
= "sha1-avx",
160 .cra_flags
= CRYPTO_ALG_TYPE_SHASH
,
161 .cra_blocksize
= SHA1_BLOCK_SIZE
,
162 .cra_module
= THIS_MODULE
,
166 static bool avx_usable(void)
168 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE
| XFEATURE_MASK_YMM
, NULL
)) {
170 pr_info("AVX detected but unusable.\n");
177 static int register_sha1_avx(void)
180 return crypto_register_shash(&sha1_avx_alg
);
184 static void unregister_sha1_avx(void)
187 crypto_unregister_shash(&sha1_avx_alg
);
190 #else /* CONFIG_AS_AVX */
191 static inline int register_sha1_avx(void) { return 0; }
192 static inline void unregister_sha1_avx(void) { }
193 #endif /* CONFIG_AS_AVX */
196 #if defined(CONFIG_AS_AVX2) && (CONFIG_AS_AVX)
197 #define SHA1_AVX2_BLOCK_OPTSIZE 4 /* optimal 4*64 bytes of SHA1 blocks */
199 asmlinkage
void sha1_transform_avx2(u32
*digest
, const char *data
,
200 unsigned int rounds
);
202 static bool avx2_usable(void)
204 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2
)
205 && boot_cpu_has(X86_FEATURE_BMI1
)
206 && boot_cpu_has(X86_FEATURE_BMI2
))
212 static void sha1_apply_transform_avx2(u32
*digest
, const char *data
,
215 /* Select the optimal transform based on data block size */
216 if (rounds
>= SHA1_AVX2_BLOCK_OPTSIZE
)
217 sha1_transform_avx2(digest
, data
, rounds
);
219 sha1_transform_avx(digest
, data
, rounds
);
222 static int sha1_avx2_update(struct shash_desc
*desc
, const u8
*data
,
225 return sha1_update(desc
, data
, len
,
226 (sha1_transform_fn
*) sha1_apply_transform_avx2
);
229 static int sha1_avx2_finup(struct shash_desc
*desc
, const u8
*data
,
230 unsigned int len
, u8
*out
)
232 return sha1_finup(desc
, data
, len
, out
,
233 (sha1_transform_fn
*) sha1_apply_transform_avx2
);
236 static int sha1_avx2_final(struct shash_desc
*desc
, u8
*out
)
238 return sha1_avx2_finup(desc
, NULL
, 0, out
);
241 static struct shash_alg sha1_avx2_alg
= {
242 .digestsize
= SHA1_DIGEST_SIZE
,
243 .init
= sha1_base_init
,
244 .update
= sha1_avx2_update
,
245 .final
= sha1_avx2_final
,
246 .finup
= sha1_avx2_finup
,
247 .descsize
= sizeof(struct sha1_state
),
250 .cra_driver_name
= "sha1-avx2",
252 .cra_flags
= CRYPTO_ALG_TYPE_SHASH
,
253 .cra_blocksize
= SHA1_BLOCK_SIZE
,
254 .cra_module
= THIS_MODULE
,
258 static int register_sha1_avx2(void)
261 return crypto_register_shash(&sha1_avx2_alg
);
265 static void unregister_sha1_avx2(void)
268 crypto_unregister_shash(&sha1_avx2_alg
);
272 static inline int register_sha1_avx2(void) { return 0; }
273 static inline void unregister_sha1_avx2(void) { }
276 #ifdef CONFIG_AS_SHA1_NI
277 asmlinkage
void sha1_ni_transform(u32
*digest
, const char *data
,
278 unsigned int rounds
);
280 static int sha1_ni_update(struct shash_desc
*desc
, const u8
*data
,
283 return sha1_update(desc
, data
, len
,
284 (sha1_transform_fn
*) sha1_ni_transform
);
287 static int sha1_ni_finup(struct shash_desc
*desc
, const u8
*data
,
288 unsigned int len
, u8
*out
)
290 return sha1_finup(desc
, data
, len
, out
,
291 (sha1_transform_fn
*) sha1_ni_transform
);
294 static int sha1_ni_final(struct shash_desc
*desc
, u8
*out
)
296 return sha1_ni_finup(desc
, NULL
, 0, out
);
299 static struct shash_alg sha1_ni_alg
= {
300 .digestsize
= SHA1_DIGEST_SIZE
,
301 .init
= sha1_base_init
,
302 .update
= sha1_ni_update
,
303 .final
= sha1_ni_final
,
304 .finup
= sha1_ni_finup
,
305 .descsize
= sizeof(struct sha1_state
),
308 .cra_driver_name
= "sha1-ni",
310 .cra_flags
= CRYPTO_ALG_TYPE_SHASH
,
311 .cra_blocksize
= SHA1_BLOCK_SIZE
,
312 .cra_module
= THIS_MODULE
,
316 static int register_sha1_ni(void)
318 if (boot_cpu_has(X86_FEATURE_SHA_NI
))
319 return crypto_register_shash(&sha1_ni_alg
);
323 static void unregister_sha1_ni(void)
325 if (boot_cpu_has(X86_FEATURE_SHA_NI
))
326 crypto_unregister_shash(&sha1_ni_alg
);
330 static inline int register_sha1_ni(void) { return 0; }
331 static inline void unregister_sha1_ni(void) { }
334 static int __init
sha1_ssse3_mod_init(void)
336 if (register_sha1_ssse3())
339 if (register_sha1_avx()) {
340 unregister_sha1_ssse3();
344 if (register_sha1_avx2()) {
345 unregister_sha1_avx();
346 unregister_sha1_ssse3();
350 if (register_sha1_ni()) {
351 unregister_sha1_avx2();
352 unregister_sha1_avx();
353 unregister_sha1_ssse3();
362 static void __exit
sha1_ssse3_mod_fini(void)
364 unregister_sha1_ni();
365 unregister_sha1_avx2();
366 unregister_sha1_avx();
367 unregister_sha1_ssse3();
370 module_init(sha1_ssse3_mod_init
);
371 module_exit(sha1_ssse3_mod_fini
);
373 MODULE_LICENSE("GPL");
374 MODULE_DESCRIPTION("SHA1 Secure Hash Algorithm, Supplemental SSE3 accelerated");
376 MODULE_ALIAS_CRYPTO("sha1");