4 * Support for VIA PadLock hardware crypto engine.
6 * Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
10 #include <crypto/algapi.h>
11 #include <crypto/aes.h>
12 #include <crypto/padlock.h>
13 #include <linux/module.h>
14 #include <linux/init.h>
15 #include <linux/types.h>
16 #include <linux/errno.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel.h>
19 #include <linux/percpu.h>
20 #include <linux/smp.h>
21 #include <linux/slab.h>
22 #include <asm/cpu_device_id.h>
23 #include <asm/byteorder.h>
24 #include <asm/processor.h>
25 #include <asm/fpu/api.h>
28 * Number of data blocks actually fetched for each xcrypt insn.
29 * Processors with prefetch errata will fetch extra blocks.
31 static unsigned int ecb_fetch_blocks
= 2;
32 #define MAX_ECB_FETCH_BLOCKS (8)
33 #define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
35 static unsigned int cbc_fetch_blocks
= 1;
36 #define MAX_CBC_FETCH_BLOCKS (4)
37 #define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
41 unsigned int __attribute__ ((__packed__
))
48 } __attribute__ ((__aligned__(PADLOCK_ALIGNMENT
)));
50 /* Whenever making any changes to the following
51 * structure *make sure* you keep E, d_data
52 * and cword aligned on 16 Bytes boundaries and
53 * the Hardware can access 16 * 16 bytes of E and d_data
54 * (only the first 15 * 16 bytes matter but the HW reads
58 u32 E
[AES_MAX_KEYLENGTH_U32
]
59 __attribute__ ((__aligned__(PADLOCK_ALIGNMENT
)));
60 u32 d_data
[AES_MAX_KEYLENGTH_U32
]
61 __attribute__ ((__aligned__(PADLOCK_ALIGNMENT
)));
69 static DEFINE_PER_CPU(struct cword
*, paes_last_cword
);
71 /* Tells whether the ACE is capable to generate
72 the extended key for a given key_len. */
74 aes_hw_extkey_available(uint8_t key_len
)
76 /* TODO: We should check the actual CPU model/stepping
77 as it's possible that the capability will be
78 added in the next CPU revisions. */
84 static inline struct aes_ctx
*aes_ctx_common(void *ctx
)
86 unsigned long addr
= (unsigned long)ctx
;
87 unsigned long align
= PADLOCK_ALIGNMENT
;
89 if (align
<= crypto_tfm_ctx_alignment())
91 return (struct aes_ctx
*)ALIGN(addr
, align
);
94 static inline struct aes_ctx
*aes_ctx(struct crypto_tfm
*tfm
)
96 return aes_ctx_common(crypto_tfm_ctx(tfm
));
99 static inline struct aes_ctx
*blk_aes_ctx(struct crypto_blkcipher
*tfm
)
101 return aes_ctx_common(crypto_blkcipher_ctx(tfm
));
104 static int aes_set_key(struct crypto_tfm
*tfm
, const u8
*in_key
,
105 unsigned int key_len
)
107 struct aes_ctx
*ctx
= aes_ctx(tfm
);
108 const __le32
*key
= (const __le32
*)in_key
;
109 u32
*flags
= &tfm
->crt_flags
;
110 struct crypto_aes_ctx gen_aes
;
114 *flags
|= CRYPTO_TFM_RES_BAD_KEY_LEN
;
119 * If the hardware is capable of generating the extended key
120 * itself we must supply the plain key for both encryption
125 ctx
->E
[0] = le32_to_cpu(key
[0]);
126 ctx
->E
[1] = le32_to_cpu(key
[1]);
127 ctx
->E
[2] = le32_to_cpu(key
[2]);
128 ctx
->E
[3] = le32_to_cpu(key
[3]);
130 /* Prepare control words. */
131 memset(&ctx
->cword
, 0, sizeof(ctx
->cword
));
133 ctx
->cword
.decrypt
.encdec
= 1;
134 ctx
->cword
.encrypt
.rounds
= 10 + (key_len
- 16) / 4;
135 ctx
->cword
.decrypt
.rounds
= ctx
->cword
.encrypt
.rounds
;
136 ctx
->cword
.encrypt
.ksize
= (key_len
- 16) / 8;
137 ctx
->cword
.decrypt
.ksize
= ctx
->cword
.encrypt
.ksize
;
139 /* Don't generate extended keys if the hardware can do it. */
140 if (aes_hw_extkey_available(key_len
))
143 ctx
->D
= ctx
->d_data
;
144 ctx
->cword
.encrypt
.keygen
= 1;
145 ctx
->cword
.decrypt
.keygen
= 1;
147 if (crypto_aes_expand_key(&gen_aes
, in_key
, key_len
)) {
148 *flags
|= CRYPTO_TFM_RES_BAD_KEY_LEN
;
152 memcpy(ctx
->E
, gen_aes
.key_enc
, AES_MAX_KEYLENGTH
);
153 memcpy(ctx
->D
, gen_aes
.key_dec
, AES_MAX_KEYLENGTH
);
156 for_each_online_cpu(cpu
)
157 if (&ctx
->cword
.encrypt
== per_cpu(paes_last_cword
, cpu
) ||
158 &ctx
->cword
.decrypt
== per_cpu(paes_last_cword
, cpu
))
159 per_cpu(paes_last_cword
, cpu
) = NULL
;
164 /* ====== Encryption/decryption routines ====== */
166 /* These are the real call to PadLock. */
167 static inline void padlock_reset_key(struct cword
*cword
)
169 int cpu
= raw_smp_processor_id();
171 if (cword
!= per_cpu(paes_last_cword
, cpu
))
172 #ifndef CONFIG_X86_64
173 asm volatile ("pushfl; popfl");
175 asm volatile ("pushfq; popfq");
179 static inline void padlock_store_cword(struct cword
*cword
)
181 per_cpu(paes_last_cword
, raw_smp_processor_id()) = cword
;
185 * While the padlock instructions don't use FP/SSE registers, they
186 * generate a spurious DNA fault when cr0.ts is '1'. These instructions
187 * should be used only inside the irq_ts_save/restore() context
190 static inline void rep_xcrypt_ecb(const u8
*input
, u8
*output
, void *key
,
191 struct cword
*control_word
, int count
)
193 asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
194 : "+S"(input
), "+D"(output
)
195 : "d"(control_word
), "b"(key
), "c"(count
));
198 static inline u8
*rep_xcrypt_cbc(const u8
*input
, u8
*output
, void *key
,
199 u8
*iv
, struct cword
*control_word
, int count
)
201 asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
202 : "+S" (input
), "+D" (output
), "+a" (iv
)
203 : "d" (control_word
), "b" (key
), "c" (count
));
207 static void ecb_crypt_copy(const u8
*in
, u8
*out
, u32
*key
,
208 struct cword
*cword
, int count
)
211 * Padlock prefetches extra data so we must provide mapped input buffers.
212 * Assume there are at least 16 bytes of stack already in use.
214 u8 buf
[AES_BLOCK_SIZE
* (MAX_ECB_FETCH_BLOCKS
- 1) + PADLOCK_ALIGNMENT
- 1];
215 u8
*tmp
= PTR_ALIGN(&buf
[0], PADLOCK_ALIGNMENT
);
217 memcpy(tmp
, in
, count
* AES_BLOCK_SIZE
);
218 rep_xcrypt_ecb(tmp
, out
, key
, cword
, count
);
221 static u8
*cbc_crypt_copy(const u8
*in
, u8
*out
, u32
*key
,
222 u8
*iv
, struct cword
*cword
, int count
)
225 * Padlock prefetches extra data so we must provide mapped input buffers.
226 * Assume there are at least 16 bytes of stack already in use.
228 u8 buf
[AES_BLOCK_SIZE
* (MAX_CBC_FETCH_BLOCKS
- 1) + PADLOCK_ALIGNMENT
- 1];
229 u8
*tmp
= PTR_ALIGN(&buf
[0], PADLOCK_ALIGNMENT
);
231 memcpy(tmp
, in
, count
* AES_BLOCK_SIZE
);
232 return rep_xcrypt_cbc(tmp
, out
, key
, iv
, cword
, count
);
235 static inline void ecb_crypt(const u8
*in
, u8
*out
, u32
*key
,
236 struct cword
*cword
, int count
)
238 /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
239 * We could avoid some copying here but it's probably not worth it.
241 if (unlikely(((unsigned long)in
& ~PAGE_MASK
) + ecb_fetch_bytes
> PAGE_SIZE
)) {
242 ecb_crypt_copy(in
, out
, key
, cword
, count
);
246 rep_xcrypt_ecb(in
, out
, key
, cword
, count
);
249 static inline u8
*cbc_crypt(const u8
*in
, u8
*out
, u32
*key
,
250 u8
*iv
, struct cword
*cword
, int count
)
252 /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
253 if (unlikely(((unsigned long)in
& ~PAGE_MASK
) + cbc_fetch_bytes
> PAGE_SIZE
))
254 return cbc_crypt_copy(in
, out
, key
, iv
, cword
, count
);
256 return rep_xcrypt_cbc(in
, out
, key
, iv
, cword
, count
);
259 static inline void padlock_xcrypt_ecb(const u8
*input
, u8
*output
, void *key
,
260 void *control_word
, u32 count
)
262 u32 initial
= count
& (ecb_fetch_blocks
- 1);
264 if (count
< ecb_fetch_blocks
) {
265 ecb_crypt(input
, output
, key
, control_word
, count
);
272 asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
273 : "+S"(input
), "+D"(output
)
274 : "d"(control_word
), "b"(key
), "c"(initial
));
276 asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
277 : "+S"(input
), "+D"(output
)
278 : "d"(control_word
), "b"(key
), "c"(count
));
281 static inline u8
*padlock_xcrypt_cbc(const u8
*input
, u8
*output
, void *key
,
282 u8
*iv
, void *control_word
, u32 count
)
284 u32 initial
= count
& (cbc_fetch_blocks
- 1);
286 if (count
< cbc_fetch_blocks
)
287 return cbc_crypt(input
, output
, key
, iv
, control_word
, count
);
292 asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
293 : "+S" (input
), "+D" (output
), "+a" (iv
)
294 : "d" (control_word
), "b" (key
), "c" (initial
));
296 asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
297 : "+S" (input
), "+D" (output
), "+a" (iv
)
298 : "d" (control_word
), "b" (key
), "c" (count
));
302 static void aes_encrypt(struct crypto_tfm
*tfm
, u8
*out
, const u8
*in
)
304 struct aes_ctx
*ctx
= aes_ctx(tfm
);
307 padlock_reset_key(&ctx
->cword
.encrypt
);
308 ts_state
= irq_ts_save();
309 ecb_crypt(in
, out
, ctx
->E
, &ctx
->cword
.encrypt
, 1);
310 irq_ts_restore(ts_state
);
311 padlock_store_cword(&ctx
->cword
.encrypt
);
314 static void aes_decrypt(struct crypto_tfm
*tfm
, u8
*out
, const u8
*in
)
316 struct aes_ctx
*ctx
= aes_ctx(tfm
);
319 padlock_reset_key(&ctx
->cword
.encrypt
);
320 ts_state
= irq_ts_save();
321 ecb_crypt(in
, out
, ctx
->D
, &ctx
->cword
.decrypt
, 1);
322 irq_ts_restore(ts_state
);
323 padlock_store_cword(&ctx
->cword
.encrypt
);
326 static struct crypto_alg aes_alg
= {
328 .cra_driver_name
= "aes-padlock",
329 .cra_priority
= PADLOCK_CRA_PRIORITY
,
330 .cra_flags
= CRYPTO_ALG_TYPE_CIPHER
,
331 .cra_blocksize
= AES_BLOCK_SIZE
,
332 .cra_ctxsize
= sizeof(struct aes_ctx
),
333 .cra_alignmask
= PADLOCK_ALIGNMENT
- 1,
334 .cra_module
= THIS_MODULE
,
337 .cia_min_keysize
= AES_MIN_KEY_SIZE
,
338 .cia_max_keysize
= AES_MAX_KEY_SIZE
,
339 .cia_setkey
= aes_set_key
,
340 .cia_encrypt
= aes_encrypt
,
341 .cia_decrypt
= aes_decrypt
,
346 static int ecb_aes_encrypt(struct blkcipher_desc
*desc
,
347 struct scatterlist
*dst
, struct scatterlist
*src
,
350 struct aes_ctx
*ctx
= blk_aes_ctx(desc
->tfm
);
351 struct blkcipher_walk walk
;
355 padlock_reset_key(&ctx
->cword
.encrypt
);
357 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
358 err
= blkcipher_walk_virt(desc
, &walk
);
360 ts_state
= irq_ts_save();
361 while ((nbytes
= walk
.nbytes
)) {
362 padlock_xcrypt_ecb(walk
.src
.virt
.addr
, walk
.dst
.virt
.addr
,
363 ctx
->E
, &ctx
->cword
.encrypt
,
364 nbytes
/ AES_BLOCK_SIZE
);
365 nbytes
&= AES_BLOCK_SIZE
- 1;
366 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
368 irq_ts_restore(ts_state
);
370 padlock_store_cword(&ctx
->cword
.encrypt
);
375 static int ecb_aes_decrypt(struct blkcipher_desc
*desc
,
376 struct scatterlist
*dst
, struct scatterlist
*src
,
379 struct aes_ctx
*ctx
= blk_aes_ctx(desc
->tfm
);
380 struct blkcipher_walk walk
;
384 padlock_reset_key(&ctx
->cword
.decrypt
);
386 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
387 err
= blkcipher_walk_virt(desc
, &walk
);
389 ts_state
= irq_ts_save();
390 while ((nbytes
= walk
.nbytes
)) {
391 padlock_xcrypt_ecb(walk
.src
.virt
.addr
, walk
.dst
.virt
.addr
,
392 ctx
->D
, &ctx
->cword
.decrypt
,
393 nbytes
/ AES_BLOCK_SIZE
);
394 nbytes
&= AES_BLOCK_SIZE
- 1;
395 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
397 irq_ts_restore(ts_state
);
399 padlock_store_cword(&ctx
->cword
.encrypt
);
404 static struct crypto_alg ecb_aes_alg
= {
405 .cra_name
= "ecb(aes)",
406 .cra_driver_name
= "ecb-aes-padlock",
407 .cra_priority
= PADLOCK_COMPOSITE_PRIORITY
,
408 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
409 .cra_blocksize
= AES_BLOCK_SIZE
,
410 .cra_ctxsize
= sizeof(struct aes_ctx
),
411 .cra_alignmask
= PADLOCK_ALIGNMENT
- 1,
412 .cra_type
= &crypto_blkcipher_type
,
413 .cra_module
= THIS_MODULE
,
416 .min_keysize
= AES_MIN_KEY_SIZE
,
417 .max_keysize
= AES_MAX_KEY_SIZE
,
418 .setkey
= aes_set_key
,
419 .encrypt
= ecb_aes_encrypt
,
420 .decrypt
= ecb_aes_decrypt
,
425 static int cbc_aes_encrypt(struct blkcipher_desc
*desc
,
426 struct scatterlist
*dst
, struct scatterlist
*src
,
429 struct aes_ctx
*ctx
= blk_aes_ctx(desc
->tfm
);
430 struct blkcipher_walk walk
;
434 padlock_reset_key(&ctx
->cword
.encrypt
);
436 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
437 err
= blkcipher_walk_virt(desc
, &walk
);
439 ts_state
= irq_ts_save();
440 while ((nbytes
= walk
.nbytes
)) {
441 u8
*iv
= padlock_xcrypt_cbc(walk
.src
.virt
.addr
,
442 walk
.dst
.virt
.addr
, ctx
->E
,
443 walk
.iv
, &ctx
->cword
.encrypt
,
444 nbytes
/ AES_BLOCK_SIZE
);
445 memcpy(walk
.iv
, iv
, AES_BLOCK_SIZE
);
446 nbytes
&= AES_BLOCK_SIZE
- 1;
447 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
449 irq_ts_restore(ts_state
);
451 padlock_store_cword(&ctx
->cword
.decrypt
);
456 static int cbc_aes_decrypt(struct blkcipher_desc
*desc
,
457 struct scatterlist
*dst
, struct scatterlist
*src
,
460 struct aes_ctx
*ctx
= blk_aes_ctx(desc
->tfm
);
461 struct blkcipher_walk walk
;
465 padlock_reset_key(&ctx
->cword
.encrypt
);
467 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
468 err
= blkcipher_walk_virt(desc
, &walk
);
470 ts_state
= irq_ts_save();
471 while ((nbytes
= walk
.nbytes
)) {
472 padlock_xcrypt_cbc(walk
.src
.virt
.addr
, walk
.dst
.virt
.addr
,
473 ctx
->D
, walk
.iv
, &ctx
->cword
.decrypt
,
474 nbytes
/ AES_BLOCK_SIZE
);
475 nbytes
&= AES_BLOCK_SIZE
- 1;
476 err
= blkcipher_walk_done(desc
, &walk
, nbytes
);
479 irq_ts_restore(ts_state
);
481 padlock_store_cword(&ctx
->cword
.encrypt
);
486 static struct crypto_alg cbc_aes_alg
= {
487 .cra_name
= "cbc(aes)",
488 .cra_driver_name
= "cbc-aes-padlock",
489 .cra_priority
= PADLOCK_COMPOSITE_PRIORITY
,
490 .cra_flags
= CRYPTO_ALG_TYPE_BLKCIPHER
,
491 .cra_blocksize
= AES_BLOCK_SIZE
,
492 .cra_ctxsize
= sizeof(struct aes_ctx
),
493 .cra_alignmask
= PADLOCK_ALIGNMENT
- 1,
494 .cra_type
= &crypto_blkcipher_type
,
495 .cra_module
= THIS_MODULE
,
498 .min_keysize
= AES_MIN_KEY_SIZE
,
499 .max_keysize
= AES_MAX_KEY_SIZE
,
500 .ivsize
= AES_BLOCK_SIZE
,
501 .setkey
= aes_set_key
,
502 .encrypt
= cbc_aes_encrypt
,
503 .decrypt
= cbc_aes_decrypt
,
508 static struct x86_cpu_id padlock_cpu_id
[] = {
509 X86_FEATURE_MATCH(X86_FEATURE_XCRYPT
),
512 MODULE_DEVICE_TABLE(x86cpu
, padlock_cpu_id
);
514 static int __init
padlock_init(void)
517 struct cpuinfo_x86
*c
= &cpu_data(0);
519 if (!x86_match_cpu(padlock_cpu_id
))
522 if (!boot_cpu_has(X86_FEATURE_XCRYPT_EN
)) {
523 printk(KERN_NOTICE PFX
"VIA PadLock detected, but not enabled. Hmm, strange...\n");
527 if ((ret
= crypto_register_alg(&aes_alg
)))
530 if ((ret
= crypto_register_alg(&ecb_aes_alg
)))
533 if ((ret
= crypto_register_alg(&cbc_aes_alg
)))
536 printk(KERN_NOTICE PFX
"Using VIA PadLock ACE for AES algorithm.\n");
538 if (c
->x86
== 6 && c
->x86_model
== 15 && c
->x86_mask
== 2) {
539 ecb_fetch_blocks
= MAX_ECB_FETCH_BLOCKS
;
540 cbc_fetch_blocks
= MAX_CBC_FETCH_BLOCKS
;
541 printk(KERN_NOTICE PFX
"VIA Nano stepping 2 detected: enabling workaround.\n");
548 crypto_unregister_alg(&ecb_aes_alg
);
550 crypto_unregister_alg(&aes_alg
);
552 printk(KERN_ERR PFX
"VIA PadLock AES initialization failed.\n");
556 static void __exit
padlock_fini(void)
558 crypto_unregister_alg(&cbc_aes_alg
);
559 crypto_unregister_alg(&ecb_aes_alg
);
560 crypto_unregister_alg(&aes_alg
);
563 module_init(padlock_init
);
564 module_exit(padlock_fini
);
566 MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
567 MODULE_LICENSE("GPL");
568 MODULE_AUTHOR("Michal Ludvig");
570 MODULE_ALIAS_CRYPTO("aes");