2 * Glue Code for assembler optimized version of 3DES
4 * Copyright © 2014 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
8 * CTR part based on code (crypto/ctr.c) by:
9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
23 #include <asm/processor.h>
24 #include <crypto/des.h>
25 #include <linux/crypto.h>
26 #include <linux/init.h>
27 #include <linux/module.h>
28 #include <linux/types.h>
29 #include <crypto/algapi.h>
31 struct des3_ede_x86_ctx
{
32 u32 enc_expkey
[DES3_EDE_EXPKEY_WORDS
];
33 u32 dec_expkey
[DES3_EDE_EXPKEY_WORDS
];
36 /* regular block cipher functions */
37 asmlinkage
void des3_ede_x86_64_crypt_blk(const u32
*expkey
, u8
*dst
,
40 /* 3-way parallel cipher functions */
41 asmlinkage
void des3_ede_x86_64_crypt_blk_3way(const u32
*expkey
, u8
*dst
,
44 static inline void des3_ede_enc_blk(struct des3_ede_x86_ctx
*ctx
, u8
*dst
,
47 u32
*enc_ctx
= ctx
->enc_expkey
;
49 des3_ede_x86_64_crypt_blk(enc_ctx
, dst
, src
);
52 static inline void des3_ede_dec_blk(struct des3_ede_x86_ctx
*ctx
, u8
*dst
,
55 u32
*dec_ctx
= ctx
->dec_expkey
;
57 des3_ede_x86_64_crypt_blk(dec_ctx
, dst
, src
);
60 static inline void des3_ede_enc_blk_3way(struct des3_ede_x86_ctx
*ctx
, u8
*dst
,
63 u32
*enc_ctx
= ctx
->enc_expkey
;
65 des3_ede_x86_64_crypt_blk_3way(enc_ctx
, dst
, src
);
68 static inline void des3_ede_dec_blk_3way(struct des3_ede_x86_ctx
*ctx
, u8
*dst
,
71 u32
*dec_ctx
= ctx
->dec_expkey
;
73 des3_ede_x86_64_crypt_blk_3way(dec_ctx
, dst
, src
);
76 static void des3_ede_x86_encrypt(struct crypto_tfm
*tfm
, u8
*dst
, const u8
*src
)
78 des3_ede_enc_blk(crypto_tfm_ctx(tfm
), dst
, src
);
81 static void des3_ede_x86_decrypt(struct crypto_tfm
*tfm
, u8
*dst
, const u8
*src
)
83 des3_ede_dec_blk(crypto_tfm_ctx(tfm
), dst
, src
);
86 static int ecb_crypt(struct blkcipher_desc
*desc
, struct blkcipher_walk
*walk
,
89 unsigned int bsize
= DES3_EDE_BLOCK_SIZE
;
93 err
= blkcipher_walk_virt(desc
, walk
);
95 while ((nbytes
= walk
->nbytes
)) {
96 u8
*wsrc
= walk
->src
.virt
.addr
;
97 u8
*wdst
= walk
->dst
.virt
.addr
;
99 /* Process four block batch */
100 if (nbytes
>= bsize
* 3) {
102 des3_ede_x86_64_crypt_blk_3way(expkey
, wdst
,
108 } while (nbytes
>= bsize
* 3);
114 /* Handle leftovers */
116 des3_ede_x86_64_crypt_blk(expkey
, wdst
, wsrc
);
121 } while (nbytes
>= bsize
);
124 err
= blkcipher_walk_done(desc
, walk
, nbytes
);
130 static int ecb_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
131 struct scatterlist
*src
, unsigned int nbytes
)
133 struct des3_ede_x86_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
134 struct blkcipher_walk walk
;
136 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
137 return ecb_crypt(desc
, &walk
, ctx
->enc_expkey
);
140 static int ecb_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
141 struct scatterlist
*src
, unsigned int nbytes
)
143 struct des3_ede_x86_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
144 struct blkcipher_walk walk
;
146 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
147 return ecb_crypt(desc
, &walk
, ctx
->dec_expkey
);
150 static unsigned int __cbc_encrypt(struct blkcipher_desc
*desc
,
151 struct blkcipher_walk
*walk
)
153 struct des3_ede_x86_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
154 unsigned int bsize
= DES3_EDE_BLOCK_SIZE
;
155 unsigned int nbytes
= walk
->nbytes
;
156 u64
*src
= (u64
*)walk
->src
.virt
.addr
;
157 u64
*dst
= (u64
*)walk
->dst
.virt
.addr
;
158 u64
*iv
= (u64
*)walk
->iv
;
162 des3_ede_enc_blk(ctx
, (u8
*)dst
, (u8
*)dst
);
168 } while (nbytes
>= bsize
);
170 *(u64
*)walk
->iv
= *iv
;
174 static int cbc_encrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
175 struct scatterlist
*src
, unsigned int nbytes
)
177 struct blkcipher_walk walk
;
180 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
181 err
= blkcipher_walk_virt(desc
, &walk
);
183 while ((nbytes
= walk
.nbytes
)) {
184 nbytes
= __cbc_encrypt(desc
, &walk
);
185 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
191 static unsigned int __cbc_decrypt(struct blkcipher_desc
*desc
,
192 struct blkcipher_walk
*walk
)
194 struct des3_ede_x86_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
195 unsigned int bsize
= DES3_EDE_BLOCK_SIZE
;
196 unsigned int nbytes
= walk
->nbytes
;
197 u64
*src
= (u64
*)walk
->src
.virt
.addr
;
198 u64
*dst
= (u64
*)walk
->dst
.virt
.addr
;
202 /* Start of the last block. */
203 src
+= nbytes
/ bsize
- 1;
204 dst
+= nbytes
/ bsize
- 1;
208 /* Process four block batch */
209 if (nbytes
>= bsize
* 3) {
211 nbytes
-= bsize
* 3 - bsize
;
218 des3_ede_dec_blk_3way(ctx
, (u8
*)dst
, (u8
*)src
);
230 } while (nbytes
>= bsize
* 3);
233 /* Handle leftovers */
235 des3_ede_dec_blk(ctx
, (u8
*)dst
, (u8
*)src
);
247 *dst
^= *(u64
*)walk
->iv
;
248 *(u64
*)walk
->iv
= last_iv
;
253 static int cbc_decrypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
254 struct scatterlist
*src
, unsigned int nbytes
)
256 struct blkcipher_walk walk
;
259 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
260 err
= blkcipher_walk_virt(desc
, &walk
);
262 while ((nbytes
= walk
.nbytes
)) {
263 nbytes
= __cbc_decrypt(desc
, &walk
);
264 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
270 static void ctr_crypt_final(struct des3_ede_x86_ctx
*ctx
,
271 struct blkcipher_walk
*walk
)
273 u8
*ctrblk
= walk
->iv
;
274 u8 keystream
[DES3_EDE_BLOCK_SIZE
];
275 u8
*src
= walk
->src
.virt
.addr
;
276 u8
*dst
= walk
->dst
.virt
.addr
;
277 unsigned int nbytes
= walk
->nbytes
;
279 des3_ede_enc_blk(ctx
, keystream
, ctrblk
);
280 crypto_xor(keystream
, src
, nbytes
);
281 memcpy(dst
, keystream
, nbytes
);
283 crypto_inc(ctrblk
, DES3_EDE_BLOCK_SIZE
);
286 static unsigned int __ctr_crypt(struct blkcipher_desc
*desc
,
287 struct blkcipher_walk
*walk
)
289 struct des3_ede_x86_ctx
*ctx
= crypto_blkcipher_ctx(desc
->tfm
);
290 unsigned int bsize
= DES3_EDE_BLOCK_SIZE
;
291 unsigned int nbytes
= walk
->nbytes
;
292 __be64
*src
= (__be64
*)walk
->src
.virt
.addr
;
293 __be64
*dst
= (__be64
*)walk
->dst
.virt
.addr
;
294 u64 ctrblk
= be64_to_cpu(*(__be64
*)walk
->iv
);
297 /* Process four block batch */
298 if (nbytes
>= bsize
* 3) {
300 /* create ctrblks for parallel encrypt */
301 ctrblocks
[0] = cpu_to_be64(ctrblk
++);
302 ctrblocks
[1] = cpu_to_be64(ctrblk
++);
303 ctrblocks
[2] = cpu_to_be64(ctrblk
++);
305 des3_ede_enc_blk_3way(ctx
, (u8
*)ctrblocks
,
308 dst
[0] = src
[0] ^ ctrblocks
[0];
309 dst
[1] = src
[1] ^ ctrblocks
[1];
310 dst
[2] = src
[2] ^ ctrblocks
[2];
314 } while ((nbytes
-= bsize
* 3) >= bsize
* 3);
320 /* Handle leftovers */
322 ctrblocks
[0] = cpu_to_be64(ctrblk
++);
324 des3_ede_enc_blk(ctx
, (u8
*)ctrblocks
, (u8
*)ctrblocks
);
326 dst
[0] = src
[0] ^ ctrblocks
[0];
330 } while ((nbytes
-= bsize
) >= bsize
);
333 *(__be64
*)walk
->iv
= cpu_to_be64(ctrblk
);
337 static int ctr_crypt(struct blkcipher_desc
*desc
, struct scatterlist
*dst
,
338 struct scatterlist
*src
, unsigned int nbytes
)
340 struct blkcipher_walk walk
;
343 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
344 err
= blkcipher_walk_virt_block(desc
, &walk
, DES3_EDE_BLOCK_SIZE
);
346 while ((nbytes
= walk
.nbytes
) >= DES3_EDE_BLOCK_SIZE
) {
347 nbytes
= __ctr_crypt(desc
, &walk
);
348 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
352 ctr_crypt_final(crypto_blkcipher_ctx(desc
->tfm
), &walk
);
353 err
= blkcipher_walk_done(desc
, &walk
, 0);
359 static int des3_ede_x86_setkey(struct crypto_tfm
*tfm
, const u8
*key
,
362 struct des3_ede_x86_ctx
*ctx
= crypto_tfm_ctx(tfm
);
366 /* Generate encryption context using generic implementation. */
367 err
= __des3_ede_setkey(ctx
->enc_expkey
, &tfm
->crt_flags
, key
, keylen
);
371 /* Fix encryption context for this implementation and form decryption
373 j
= DES3_EDE_EXPKEY_WORDS
- 2;
374 for (i
= 0; i
< DES3_EDE_EXPKEY_WORDS
; i
+= 2, j
-= 2) {
375 tmp
= ror32(ctx
->enc_expkey
[i
+ 1], 4);
376 ctx
->enc_expkey
[i
+ 1] = tmp
;
378 ctx
->dec_expkey
[j
+ 0] = ctx
->enc_expkey
[i
+ 0];
379 ctx
->dec_expkey
[j
+ 1] = tmp
;
385 static struct crypto_alg des3_ede_algs
[4] = { {
386 .cra_name
= "des3_ede",
387 .cra_driver_name
= "des3_ede-asm",
389 .cra_flags
= CRYPTO_ALG_TYPE_CIPHER
,
390 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
391 .cra_ctxsize
= sizeof(struct des3_ede_x86_ctx
),
393 .cra_module
= THIS_MODULE
,
396 .cia_min_keysize
= DES3_EDE_KEY_SIZE
,
397 .cia_max_keysize
= DES3_EDE_KEY_SIZE
,
398 .cia_setkey
= des3_ede_x86_setkey
,
399 .cia_encrypt
= des3_ede_x86_encrypt
,
400 .cia_decrypt
= des3_ede_x86_decrypt
,
404 .cra_name
= "ecb(des3_ede)",
405 .cra_driver_name
= "ecb-des3_ede-asm",
407 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
408 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
409 .cra_ctxsize
= sizeof(struct des3_ede_x86_ctx
),
411 .cra_type
= &crypto_blkcipher_type
,
412 .cra_module
= THIS_MODULE
,
415 .min_keysize
= DES3_EDE_KEY_SIZE
,
416 .max_keysize
= DES3_EDE_KEY_SIZE
,
417 .setkey
= des3_ede_x86_setkey
,
418 .encrypt
= ecb_encrypt
,
419 .decrypt
= ecb_decrypt
,
423 .cra_name
= "cbc(des3_ede)",
424 .cra_driver_name
= "cbc-des3_ede-asm",
426 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
427 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
428 .cra_ctxsize
= sizeof(struct des3_ede_x86_ctx
),
430 .cra_type
= &crypto_blkcipher_type
,
431 .cra_module
= THIS_MODULE
,
434 .min_keysize
= DES3_EDE_KEY_SIZE
,
435 .max_keysize
= DES3_EDE_KEY_SIZE
,
436 .ivsize
= DES3_EDE_BLOCK_SIZE
,
437 .setkey
= des3_ede_x86_setkey
,
438 .encrypt
= cbc_encrypt
,
439 .decrypt
= cbc_decrypt
,
443 .cra_name
= "ctr(des3_ede)",
444 .cra_driver_name
= "ctr-des3_ede-asm",
446 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
448 .cra_ctxsize
= sizeof(struct des3_ede_x86_ctx
),
450 .cra_type
= &crypto_blkcipher_type
,
451 .cra_module
= THIS_MODULE
,
454 .min_keysize
= DES3_EDE_KEY_SIZE
,
455 .max_keysize
= DES3_EDE_KEY_SIZE
,
456 .ivsize
= DES3_EDE_BLOCK_SIZE
,
457 .setkey
= des3_ede_x86_setkey
,
458 .encrypt
= ctr_crypt
,
459 .decrypt
= ctr_crypt
,
464 static bool is_blacklisted_cpu(void)
466 if (boot_cpu_data
.x86_vendor
!= X86_VENDOR_INTEL
)
469 if (boot_cpu_data
.x86
== 0x0f) {
471 * On Pentium 4, des3_ede-x86_64 is slower than generic C
472 * implementation because use of 64bit rotates (which are really
473 * slow on P4). Therefore blacklist P4s.
482 module_param(force
, int, 0);
483 MODULE_PARM_DESC(force
, "Force module load, ignore CPU blacklist");
485 static int __init
des3_ede_x86_init(void)
487 if (!force
&& is_blacklisted_cpu()) {
488 pr_info("des3_ede-x86_64: performance on this CPU would be suboptimal: disabling des3_ede-x86_64.\n");
492 return crypto_register_algs(des3_ede_algs
, ARRAY_SIZE(des3_ede_algs
));
495 static void __exit
des3_ede_x86_fini(void)
497 crypto_unregister_algs(des3_ede_algs
, ARRAY_SIZE(des3_ede_algs
));
500 module_init(des3_ede_x86_init
);
501 module_exit(des3_ede_x86_fini
);
503 MODULE_LICENSE("GPL");
504 MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized");
505 MODULE_ALIAS_CRYPTO("des3_ede");
506 MODULE_ALIAS_CRYPTO("des3_ede-asm");
507 MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>");