2 * Copyright (C) 2003 Jana Saout <jana@saout.de>
3 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
4 * Copyright (C) 2006-2017 Red Hat, Inc. All rights reserved.
5 * Copyright (C) 2013-2017 Milan Broz <gmazyland@gmail.com>
7 * This file is released under the GPL.
10 #include <linux/completion.h>
11 #include <linux/err.h>
12 #include <linux/module.h>
13 #include <linux/init.h>
14 #include <linux/kernel.h>
15 #include <linux/key.h>
16 #include <linux/bio.h>
17 #include <linux/blkdev.h>
18 #include <linux/mempool.h>
19 #include <linux/slab.h>
20 #include <linux/crypto.h>
21 #include <linux/workqueue.h>
22 #include <linux/kthread.h>
23 #include <linux/backing-dev.h>
24 #include <linux/atomic.h>
25 #include <linux/scatterlist.h>
26 #include <linux/rbtree.h>
27 #include <linux/ctype.h>
29 #include <asm/unaligned.h>
30 #include <crypto/hash.h>
31 #include <crypto/md5.h>
32 #include <crypto/algapi.h>
33 #include <crypto/skcipher.h>
34 #include <crypto/aead.h>
35 #include <crypto/authenc.h>
36 #include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
37 #include <keys/user-type.h>
39 #include <linux/device-mapper.h>
41 #define DM_MSG_PREFIX "crypt"
44 * context holding the current state of a multi-part conversion
46 struct convert_context
{
47 struct completion restart
;
50 struct bvec_iter iter_in
;
51 struct bvec_iter iter_out
;
55 struct skcipher_request
*req
;
56 struct aead_request
*req_aead
;
62 * per bio private data
65 struct crypt_config
*cc
;
67 u8
*integrity_metadata
;
68 bool integrity_metadata_from_pool
;
69 struct work_struct work
;
71 struct convert_context ctx
;
77 struct rb_node rb_node
;
78 } CRYPTO_MINALIGN_ATTR
;
80 struct dm_crypt_request
{
81 struct convert_context
*ctx
;
82 struct scatterlist sg_in
[4];
83 struct scatterlist sg_out
[4];
89 struct crypt_iv_operations
{
90 int (*ctr
)(struct crypt_config
*cc
, struct dm_target
*ti
,
92 void (*dtr
)(struct crypt_config
*cc
);
93 int (*init
)(struct crypt_config
*cc
);
94 int (*wipe
)(struct crypt_config
*cc
);
95 int (*generator
)(struct crypt_config
*cc
, u8
*iv
,
96 struct dm_crypt_request
*dmreq
);
97 int (*post
)(struct crypt_config
*cc
, u8
*iv
,
98 struct dm_crypt_request
*dmreq
);
101 struct iv_benbi_private
{
105 #define LMK_SEED_SIZE 64 /* hash + 0 */
106 struct iv_lmk_private
{
107 struct crypto_shash
*hash_tfm
;
111 #define TCW_WHITENING_SIZE 16
112 struct iv_tcw_private
{
113 struct crypto_shash
*crc32_tfm
;
119 * Crypt: maps a linear range of a block device
120 * and encrypts / decrypts at the same time.
122 enum flags
{ DM_CRYPT_SUSPENDED
, DM_CRYPT_KEY_VALID
,
123 DM_CRYPT_SAME_CPU
, DM_CRYPT_NO_OFFLOAD
};
126 CRYPT_MODE_INTEGRITY_AEAD
, /* Use authenticated mode for cihper */
127 CRYPT_IV_LARGE_SECTORS
, /* Calculate IV from sector_size, not 512B sectors */
131 * The fields in here must be read only after initialization.
133 struct crypt_config
{
137 struct percpu_counter n_allocated_pages
;
139 struct workqueue_struct
*io_queue
;
140 struct workqueue_struct
*crypt_queue
;
142 spinlock_t write_thread_lock
;
143 struct task_struct
*write_thread
;
144 struct rb_root write_tree
;
150 const struct crypt_iv_operations
*iv_gen_ops
;
152 struct iv_benbi_private benbi
;
153 struct iv_lmk_private lmk
;
154 struct iv_tcw_private tcw
;
157 unsigned int iv_size
;
158 unsigned short int sector_size
;
159 unsigned char sector_shift
;
162 struct crypto_skcipher
**tfms
;
163 struct crypto_aead
**tfms_aead
;
166 unsigned long cipher_flags
;
169 * Layout of each crypto request:
171 * struct skcipher_request
174 * struct dm_crypt_request
178 * The padding is added so that dm_crypt_request and the IV are
181 unsigned int dmreq_start
;
183 unsigned int per_bio_data_size
;
186 unsigned int key_size
;
187 unsigned int key_parts
; /* independent parts in key buffer */
188 unsigned int key_extra_size
; /* additional keys length */
189 unsigned int key_mac_size
; /* MAC key size for authenc(...) */
191 unsigned int integrity_tag_size
;
192 unsigned int integrity_iv_size
;
193 unsigned int on_disk_tag_size
;
196 * pool for per bio private data, crypto requests,
197 * encryption requeusts/buffer pages and integrity tags
199 unsigned tag_pool_max_sectors
;
205 struct mutex bio_alloc_lock
;
207 u8
*authenc_key
; /* space for keys in authenc() format (if used) */
212 #define MAX_TAG_SIZE 480
213 #define POOL_ENTRY_SIZE 512
215 static DEFINE_SPINLOCK(dm_crypt_clients_lock
);
216 static unsigned dm_crypt_clients_n
= 0;
217 static volatile unsigned long dm_crypt_pages_per_client
;
218 #define DM_CRYPT_MEMORY_PERCENT 2
219 #define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_PAGES * 16)
221 static void clone_init(struct dm_crypt_io
*, struct bio
*);
222 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
);
223 static struct scatterlist
*crypt_get_sg_data(struct crypt_config
*cc
,
224 struct scatterlist
*sg
);
227 * Use this to access cipher attributes that are independent of the key.
229 static struct crypto_skcipher
*any_tfm(struct crypt_config
*cc
)
231 return cc
->cipher_tfm
.tfms
[0];
234 static struct crypto_aead
*any_tfm_aead(struct crypt_config
*cc
)
236 return cc
->cipher_tfm
.tfms_aead
[0];
240 * Different IV generation algorithms:
242 * plain: the initial vector is the 32-bit little-endian version of the sector
243 * number, padded with zeros if necessary.
245 * plain64: the initial vector is the 64-bit little-endian version of the sector
246 * number, padded with zeros if necessary.
248 * plain64be: the initial vector is the 64-bit big-endian version of the sector
249 * number, padded with zeros if necessary.
251 * essiv: "encrypted sector|salt initial vector", the sector number is
252 * encrypted with the bulk cipher using a salt as key. The salt
253 * should be derived from the bulk cipher's key via hashing.
255 * benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
256 * (needed for LRW-32-AES and possible other narrow block modes)
258 * null: the initial vector is always zero. Provides compatibility with
259 * obsolete loop_fish2 devices. Do not use for new devices.
261 * lmk: Compatible implementation of the block chaining mode used
262 * by the Loop-AES block device encryption system
263 * designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
264 * It operates on full 512 byte sectors and uses CBC
265 * with an IV derived from the sector number, the data and
266 * optionally extra IV seed.
267 * This means that after decryption the first block
268 * of sector must be tweaked according to decrypted data.
269 * Loop-AES can use three encryption schemes:
270 * version 1: is plain aes-cbc mode
271 * version 2: uses 64 multikey scheme with lmk IV generator
272 * version 3: the same as version 2 with additional IV seed
273 * (it uses 65 keys, last key is used as IV seed)
275 * tcw: Compatible implementation of the block chaining mode used
276 * by the TrueCrypt device encryption system (prior to version 4.1).
277 * For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
278 * It operates on full 512 byte sectors and uses CBC
279 * with an IV derived from initial key and the sector number.
280 * In addition, whitening value is applied on every sector, whitening
281 * is calculated from initial key, sector number and mixed using CRC32.
282 * Note that this encryption scheme is vulnerable to watermarking attacks
283 * and should be used for old compatible containers access only.
285 * eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)
286 * The IV is encrypted little-endian byte-offset (with the same key
287 * and cipher as the volume).
290 static int crypt_iv_plain_gen(struct crypt_config
*cc
, u8
*iv
,
291 struct dm_crypt_request
*dmreq
)
293 memset(iv
, 0, cc
->iv_size
);
294 *(__le32
*)iv
= cpu_to_le32(dmreq
->iv_sector
& 0xffffffff);
299 static int crypt_iv_plain64_gen(struct crypt_config
*cc
, u8
*iv
,
300 struct dm_crypt_request
*dmreq
)
302 memset(iv
, 0, cc
->iv_size
);
303 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
308 static int crypt_iv_plain64be_gen(struct crypt_config
*cc
, u8
*iv
,
309 struct dm_crypt_request
*dmreq
)
311 memset(iv
, 0, cc
->iv_size
);
312 /* iv_size is at least of size u64; usually it is 16 bytes */
313 *(__be64
*)&iv
[cc
->iv_size
- sizeof(u64
)] = cpu_to_be64(dmreq
->iv_sector
);
318 static int crypt_iv_essiv_gen(struct crypt_config
*cc
, u8
*iv
,
319 struct dm_crypt_request
*dmreq
)
322 * ESSIV encryption of the IV is now handled by the crypto API,
323 * so just pass the plain sector number here.
325 memset(iv
, 0, cc
->iv_size
);
326 *(__le64
*)iv
= cpu_to_le64(dmreq
->iv_sector
);
331 static int crypt_iv_benbi_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
334 unsigned bs
= crypto_skcipher_blocksize(any_tfm(cc
));
337 /* we need to calculate how far we must shift the sector count
338 * to get the cipher block count, we use this shift in _gen */
340 if (1 << log
!= bs
) {
341 ti
->error
= "cypher blocksize is not a power of 2";
346 ti
->error
= "cypher blocksize is > 512";
350 cc
->iv_gen_private
.benbi
.shift
= 9 - log
;
355 static void crypt_iv_benbi_dtr(struct crypt_config
*cc
)
359 static int crypt_iv_benbi_gen(struct crypt_config
*cc
, u8
*iv
,
360 struct dm_crypt_request
*dmreq
)
364 memset(iv
, 0, cc
->iv_size
- sizeof(u64
)); /* rest is cleared below */
366 val
= cpu_to_be64(((u64
)dmreq
->iv_sector
<< cc
->iv_gen_private
.benbi
.shift
) + 1);
367 put_unaligned(val
, (__be64
*)(iv
+ cc
->iv_size
- sizeof(u64
)));
372 static int crypt_iv_null_gen(struct crypt_config
*cc
, u8
*iv
,
373 struct dm_crypt_request
*dmreq
)
375 memset(iv
, 0, cc
->iv_size
);
380 static void crypt_iv_lmk_dtr(struct crypt_config
*cc
)
382 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
384 if (lmk
->hash_tfm
&& !IS_ERR(lmk
->hash_tfm
))
385 crypto_free_shash(lmk
->hash_tfm
);
386 lmk
->hash_tfm
= NULL
;
392 static int crypt_iv_lmk_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
395 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
397 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
)) {
398 ti
->error
= "Unsupported sector size for LMK";
402 lmk
->hash_tfm
= crypto_alloc_shash("md5", 0, 0);
403 if (IS_ERR(lmk
->hash_tfm
)) {
404 ti
->error
= "Error initializing LMK hash";
405 return PTR_ERR(lmk
->hash_tfm
);
408 /* No seed in LMK version 2 */
409 if (cc
->key_parts
== cc
->tfms_count
) {
414 lmk
->seed
= kzalloc(LMK_SEED_SIZE
, GFP_KERNEL
);
416 crypt_iv_lmk_dtr(cc
);
417 ti
->error
= "Error kmallocing seed storage in LMK";
424 static int crypt_iv_lmk_init(struct crypt_config
*cc
)
426 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
427 int subkey_size
= cc
->key_size
/ cc
->key_parts
;
429 /* LMK seed is on the position of LMK_KEYS + 1 key */
431 memcpy(lmk
->seed
, cc
->key
+ (cc
->tfms_count
* subkey_size
),
432 crypto_shash_digestsize(lmk
->hash_tfm
));
437 static int crypt_iv_lmk_wipe(struct crypt_config
*cc
)
439 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
442 memset(lmk
->seed
, 0, LMK_SEED_SIZE
);
447 static int crypt_iv_lmk_one(struct crypt_config
*cc
, u8
*iv
,
448 struct dm_crypt_request
*dmreq
,
451 struct iv_lmk_private
*lmk
= &cc
->iv_gen_private
.lmk
;
452 SHASH_DESC_ON_STACK(desc
, lmk
->hash_tfm
);
453 struct md5_state md5state
;
457 desc
->tfm
= lmk
->hash_tfm
;
459 r
= crypto_shash_init(desc
);
464 r
= crypto_shash_update(desc
, lmk
->seed
, LMK_SEED_SIZE
);
469 /* Sector is always 512B, block size 16, add data of blocks 1-31 */
470 r
= crypto_shash_update(desc
, data
+ 16, 16 * 31);
474 /* Sector is cropped to 56 bits here */
475 buf
[0] = cpu_to_le32(dmreq
->iv_sector
& 0xFFFFFFFF);
476 buf
[1] = cpu_to_le32((((u64
)dmreq
->iv_sector
>> 32) & 0x00FFFFFF) | 0x80000000);
477 buf
[2] = cpu_to_le32(4024);
479 r
= crypto_shash_update(desc
, (u8
*)buf
, sizeof(buf
));
483 /* No MD5 padding here */
484 r
= crypto_shash_export(desc
, &md5state
);
488 for (i
= 0; i
< MD5_HASH_WORDS
; i
++)
489 __cpu_to_le32s(&md5state
.hash
[i
]);
490 memcpy(iv
, &md5state
.hash
, cc
->iv_size
);
495 static int crypt_iv_lmk_gen(struct crypt_config
*cc
, u8
*iv
,
496 struct dm_crypt_request
*dmreq
)
498 struct scatterlist
*sg
;
502 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
) {
503 sg
= crypt_get_sg_data(cc
, dmreq
->sg_in
);
504 src
= kmap_atomic(sg_page(sg
));
505 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, src
+ sg
->offset
);
508 memset(iv
, 0, cc
->iv_size
);
513 static int crypt_iv_lmk_post(struct crypt_config
*cc
, u8
*iv
,
514 struct dm_crypt_request
*dmreq
)
516 struct scatterlist
*sg
;
520 if (bio_data_dir(dmreq
->ctx
->bio_in
) == WRITE
)
523 sg
= crypt_get_sg_data(cc
, dmreq
->sg_out
);
524 dst
= kmap_atomic(sg_page(sg
));
525 r
= crypt_iv_lmk_one(cc
, iv
, dmreq
, dst
+ sg
->offset
);
527 /* Tweak the first block of plaintext sector */
529 crypto_xor(dst
+ sg
->offset
, iv
, cc
->iv_size
);
535 static void crypt_iv_tcw_dtr(struct crypt_config
*cc
)
537 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
539 kzfree(tcw
->iv_seed
);
541 kzfree(tcw
->whitening
);
542 tcw
->whitening
= NULL
;
544 if (tcw
->crc32_tfm
&& !IS_ERR(tcw
->crc32_tfm
))
545 crypto_free_shash(tcw
->crc32_tfm
);
546 tcw
->crc32_tfm
= NULL
;
549 static int crypt_iv_tcw_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
552 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
554 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
)) {
555 ti
->error
= "Unsupported sector size for TCW";
559 if (cc
->key_size
<= (cc
->iv_size
+ TCW_WHITENING_SIZE
)) {
560 ti
->error
= "Wrong key size for TCW";
564 tcw
->crc32_tfm
= crypto_alloc_shash("crc32", 0, 0);
565 if (IS_ERR(tcw
->crc32_tfm
)) {
566 ti
->error
= "Error initializing CRC32 in TCW";
567 return PTR_ERR(tcw
->crc32_tfm
);
570 tcw
->iv_seed
= kzalloc(cc
->iv_size
, GFP_KERNEL
);
571 tcw
->whitening
= kzalloc(TCW_WHITENING_SIZE
, GFP_KERNEL
);
572 if (!tcw
->iv_seed
|| !tcw
->whitening
) {
573 crypt_iv_tcw_dtr(cc
);
574 ti
->error
= "Error allocating seed storage in TCW";
581 static int crypt_iv_tcw_init(struct crypt_config
*cc
)
583 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
584 int key_offset
= cc
->key_size
- cc
->iv_size
- TCW_WHITENING_SIZE
;
586 memcpy(tcw
->iv_seed
, &cc
->key
[key_offset
], cc
->iv_size
);
587 memcpy(tcw
->whitening
, &cc
->key
[key_offset
+ cc
->iv_size
],
593 static int crypt_iv_tcw_wipe(struct crypt_config
*cc
)
595 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
597 memset(tcw
->iv_seed
, 0, cc
->iv_size
);
598 memset(tcw
->whitening
, 0, TCW_WHITENING_SIZE
);
603 static int crypt_iv_tcw_whitening(struct crypt_config
*cc
,
604 struct dm_crypt_request
*dmreq
,
607 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
608 __le64 sector
= cpu_to_le64(dmreq
->iv_sector
);
609 u8 buf
[TCW_WHITENING_SIZE
];
610 SHASH_DESC_ON_STACK(desc
, tcw
->crc32_tfm
);
613 /* xor whitening with sector number */
614 crypto_xor_cpy(buf
, tcw
->whitening
, (u8
*)§or
, 8);
615 crypto_xor_cpy(&buf
[8], tcw
->whitening
+ 8, (u8
*)§or
, 8);
617 /* calculate crc32 for every 32bit part and xor it */
618 desc
->tfm
= tcw
->crc32_tfm
;
619 for (i
= 0; i
< 4; i
++) {
620 r
= crypto_shash_init(desc
);
623 r
= crypto_shash_update(desc
, &buf
[i
* 4], 4);
626 r
= crypto_shash_final(desc
, &buf
[i
* 4]);
630 crypto_xor(&buf
[0], &buf
[12], 4);
631 crypto_xor(&buf
[4], &buf
[8], 4);
633 /* apply whitening (8 bytes) to whole sector */
634 for (i
= 0; i
< ((1 << SECTOR_SHIFT
) / 8); i
++)
635 crypto_xor(data
+ i
* 8, buf
, 8);
637 memzero_explicit(buf
, sizeof(buf
));
641 static int crypt_iv_tcw_gen(struct crypt_config
*cc
, u8
*iv
,
642 struct dm_crypt_request
*dmreq
)
644 struct scatterlist
*sg
;
645 struct iv_tcw_private
*tcw
= &cc
->iv_gen_private
.tcw
;
646 __le64 sector
= cpu_to_le64(dmreq
->iv_sector
);
650 /* Remove whitening from ciphertext */
651 if (bio_data_dir(dmreq
->ctx
->bio_in
) != WRITE
) {
652 sg
= crypt_get_sg_data(cc
, dmreq
->sg_in
);
653 src
= kmap_atomic(sg_page(sg
));
654 r
= crypt_iv_tcw_whitening(cc
, dmreq
, src
+ sg
->offset
);
659 crypto_xor_cpy(iv
, tcw
->iv_seed
, (u8
*)§or
, 8);
661 crypto_xor_cpy(&iv
[8], tcw
->iv_seed
+ 8, (u8
*)§or
,
667 static int crypt_iv_tcw_post(struct crypt_config
*cc
, u8
*iv
,
668 struct dm_crypt_request
*dmreq
)
670 struct scatterlist
*sg
;
674 if (bio_data_dir(dmreq
->ctx
->bio_in
) != WRITE
)
677 /* Apply whitening on ciphertext */
678 sg
= crypt_get_sg_data(cc
, dmreq
->sg_out
);
679 dst
= kmap_atomic(sg_page(sg
));
680 r
= crypt_iv_tcw_whitening(cc
, dmreq
, dst
+ sg
->offset
);
686 static int crypt_iv_random_gen(struct crypt_config
*cc
, u8
*iv
,
687 struct dm_crypt_request
*dmreq
)
689 /* Used only for writes, there must be an additional space to store IV */
690 get_random_bytes(iv
, cc
->iv_size
);
694 static int crypt_iv_eboiv_ctr(struct crypt_config
*cc
, struct dm_target
*ti
,
697 if (test_bit(CRYPT_MODE_INTEGRITY_AEAD
, &cc
->cipher_flags
)) {
698 ti
->error
= "AEAD transforms not supported for EBOIV";
702 if (crypto_skcipher_blocksize(any_tfm(cc
)) != cc
->iv_size
) {
703 ti
->error
= "Block size of EBOIV cipher does "
704 "not match IV size of block cipher";
711 static int crypt_iv_eboiv_gen(struct crypt_config
*cc
, u8
*iv
,
712 struct dm_crypt_request
*dmreq
)
714 u8 buf
[MAX_CIPHER_BLOCKSIZE
] __aligned(__alignof__(__le64
));
715 struct skcipher_request
*req
;
716 struct scatterlist src
, dst
;
717 struct crypto_wait wait
;
720 req
= skcipher_request_alloc(any_tfm(cc
), GFP_KERNEL
| GFP_NOFS
);
724 memset(buf
, 0, cc
->iv_size
);
725 *(__le64
*)buf
= cpu_to_le64(dmreq
->iv_sector
* cc
->sector_size
);
727 sg_init_one(&src
, page_address(ZERO_PAGE(0)), cc
->iv_size
);
728 sg_init_one(&dst
, iv
, cc
->iv_size
);
729 skcipher_request_set_crypt(req
, &src
, &dst
, cc
->iv_size
, buf
);
730 skcipher_request_set_callback(req
, 0, crypto_req_done
, &wait
);
731 err
= crypto_wait_req(crypto_skcipher_encrypt(req
), &wait
);
732 skcipher_request_free(req
);
737 static const struct crypt_iv_operations crypt_iv_plain_ops
= {
738 .generator
= crypt_iv_plain_gen
741 static const struct crypt_iv_operations crypt_iv_plain64_ops
= {
742 .generator
= crypt_iv_plain64_gen
745 static const struct crypt_iv_operations crypt_iv_plain64be_ops
= {
746 .generator
= crypt_iv_plain64be_gen
749 static const struct crypt_iv_operations crypt_iv_essiv_ops
= {
750 .generator
= crypt_iv_essiv_gen
753 static const struct crypt_iv_operations crypt_iv_benbi_ops
= {
754 .ctr
= crypt_iv_benbi_ctr
,
755 .dtr
= crypt_iv_benbi_dtr
,
756 .generator
= crypt_iv_benbi_gen
759 static const struct crypt_iv_operations crypt_iv_null_ops
= {
760 .generator
= crypt_iv_null_gen
763 static const struct crypt_iv_operations crypt_iv_lmk_ops
= {
764 .ctr
= crypt_iv_lmk_ctr
,
765 .dtr
= crypt_iv_lmk_dtr
,
766 .init
= crypt_iv_lmk_init
,
767 .wipe
= crypt_iv_lmk_wipe
,
768 .generator
= crypt_iv_lmk_gen
,
769 .post
= crypt_iv_lmk_post
772 static const struct crypt_iv_operations crypt_iv_tcw_ops
= {
773 .ctr
= crypt_iv_tcw_ctr
,
774 .dtr
= crypt_iv_tcw_dtr
,
775 .init
= crypt_iv_tcw_init
,
776 .wipe
= crypt_iv_tcw_wipe
,
777 .generator
= crypt_iv_tcw_gen
,
778 .post
= crypt_iv_tcw_post
781 static struct crypt_iv_operations crypt_iv_random_ops
= {
782 .generator
= crypt_iv_random_gen
785 static struct crypt_iv_operations crypt_iv_eboiv_ops
= {
786 .ctr
= crypt_iv_eboiv_ctr
,
787 .generator
= crypt_iv_eboiv_gen
791 * Integrity extensions
793 static bool crypt_integrity_aead(struct crypt_config
*cc
)
795 return test_bit(CRYPT_MODE_INTEGRITY_AEAD
, &cc
->cipher_flags
);
798 static bool crypt_integrity_hmac(struct crypt_config
*cc
)
800 return crypt_integrity_aead(cc
) && cc
->key_mac_size
;
803 /* Get sg containing data */
804 static struct scatterlist
*crypt_get_sg_data(struct crypt_config
*cc
,
805 struct scatterlist
*sg
)
807 if (unlikely(crypt_integrity_aead(cc
)))
813 static int dm_crypt_integrity_io_alloc(struct dm_crypt_io
*io
, struct bio
*bio
)
815 struct bio_integrity_payload
*bip
;
816 unsigned int tag_len
;
819 if (!bio_sectors(bio
) || !io
->cc
->on_disk_tag_size
)
822 bip
= bio_integrity_alloc(bio
, GFP_NOIO
, 1);
826 tag_len
= io
->cc
->on_disk_tag_size
* (bio_sectors(bio
) >> io
->cc
->sector_shift
);
828 bip
->bip_iter
.bi_size
= tag_len
;
829 bip
->bip_iter
.bi_sector
= io
->cc
->start
+ io
->sector
;
831 ret
= bio_integrity_add_page(bio
, virt_to_page(io
->integrity_metadata
),
832 tag_len
, offset_in_page(io
->integrity_metadata
));
833 if (unlikely(ret
!= tag_len
))
839 static int crypt_integrity_ctr(struct crypt_config
*cc
, struct dm_target
*ti
)
841 #ifdef CONFIG_BLK_DEV_INTEGRITY
842 struct blk_integrity
*bi
= blk_get_integrity(cc
->dev
->bdev
->bd_disk
);
843 struct mapped_device
*md
= dm_table_get_md(ti
->table
);
845 /* From now we require underlying device with our integrity profile */
846 if (!bi
|| strcasecmp(bi
->profile
->name
, "DM-DIF-EXT-TAG")) {
847 ti
->error
= "Integrity profile not supported.";
851 if (bi
->tag_size
!= cc
->on_disk_tag_size
||
852 bi
->tuple_size
!= cc
->on_disk_tag_size
) {
853 ti
->error
= "Integrity profile tag size mismatch.";
856 if (1 << bi
->interval_exp
!= cc
->sector_size
) {
857 ti
->error
= "Integrity profile sector size mismatch.";
861 if (crypt_integrity_aead(cc
)) {
862 cc
->integrity_tag_size
= cc
->on_disk_tag_size
- cc
->integrity_iv_size
;
863 DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md
),
864 cc
->integrity_tag_size
, cc
->integrity_iv_size
);
866 if (crypto_aead_setauthsize(any_tfm_aead(cc
), cc
->integrity_tag_size
)) {
867 ti
->error
= "Integrity AEAD auth tag size is not supported.";
870 } else if (cc
->integrity_iv_size
)
871 DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md
),
872 cc
->integrity_iv_size
);
874 if ((cc
->integrity_tag_size
+ cc
->integrity_iv_size
) != bi
->tag_size
) {
875 ti
->error
= "Not enough space for integrity tag in the profile.";
881 ti
->error
= "Integrity profile not supported.";
886 static void crypt_convert_init(struct crypt_config
*cc
,
887 struct convert_context
*ctx
,
888 struct bio
*bio_out
, struct bio
*bio_in
,
891 ctx
->bio_in
= bio_in
;
892 ctx
->bio_out
= bio_out
;
894 ctx
->iter_in
= bio_in
->bi_iter
;
896 ctx
->iter_out
= bio_out
->bi_iter
;
897 ctx
->cc_sector
= sector
+ cc
->iv_offset
;
898 init_completion(&ctx
->restart
);
901 static struct dm_crypt_request
*dmreq_of_req(struct crypt_config
*cc
,
904 return (struct dm_crypt_request
*)((char *)req
+ cc
->dmreq_start
);
907 static void *req_of_dmreq(struct crypt_config
*cc
, struct dm_crypt_request
*dmreq
)
909 return (void *)((char *)dmreq
- cc
->dmreq_start
);
912 static u8
*iv_of_dmreq(struct crypt_config
*cc
,
913 struct dm_crypt_request
*dmreq
)
915 if (crypt_integrity_aead(cc
))
916 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
917 crypto_aead_alignmask(any_tfm_aead(cc
)) + 1);
919 return (u8
*)ALIGN((unsigned long)(dmreq
+ 1),
920 crypto_skcipher_alignmask(any_tfm(cc
)) + 1);
923 static u8
*org_iv_of_dmreq(struct crypt_config
*cc
,
924 struct dm_crypt_request
*dmreq
)
926 return iv_of_dmreq(cc
, dmreq
) + cc
->iv_size
;
929 static __le64
*org_sector_of_dmreq(struct crypt_config
*cc
,
930 struct dm_crypt_request
*dmreq
)
932 u8
*ptr
= iv_of_dmreq(cc
, dmreq
) + cc
->iv_size
+ cc
->iv_size
;
933 return (__le64
*) ptr
;
936 static unsigned int *org_tag_of_dmreq(struct crypt_config
*cc
,
937 struct dm_crypt_request
*dmreq
)
939 u8
*ptr
= iv_of_dmreq(cc
, dmreq
) + cc
->iv_size
+
940 cc
->iv_size
+ sizeof(uint64_t);
941 return (unsigned int*)ptr
;
944 static void *tag_from_dmreq(struct crypt_config
*cc
,
945 struct dm_crypt_request
*dmreq
)
947 struct convert_context
*ctx
= dmreq
->ctx
;
948 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
950 return &io
->integrity_metadata
[*org_tag_of_dmreq(cc
, dmreq
) *
951 cc
->on_disk_tag_size
];
954 static void *iv_tag_from_dmreq(struct crypt_config
*cc
,
955 struct dm_crypt_request
*dmreq
)
957 return tag_from_dmreq(cc
, dmreq
) + cc
->integrity_tag_size
;
960 static int crypt_convert_block_aead(struct crypt_config
*cc
,
961 struct convert_context
*ctx
,
962 struct aead_request
*req
,
963 unsigned int tag_offset
)
965 struct bio_vec bv_in
= bio_iter_iovec(ctx
->bio_in
, ctx
->iter_in
);
966 struct bio_vec bv_out
= bio_iter_iovec(ctx
->bio_out
, ctx
->iter_out
);
967 struct dm_crypt_request
*dmreq
;
968 u8
*iv
, *org_iv
, *tag_iv
, *tag
;
972 BUG_ON(cc
->integrity_iv_size
&& cc
->integrity_iv_size
!= cc
->iv_size
);
974 /* Reject unexpected unaligned bio. */
975 if (unlikely(bv_in
.bv_len
& (cc
->sector_size
- 1)))
978 dmreq
= dmreq_of_req(cc
, req
);
979 dmreq
->iv_sector
= ctx
->cc_sector
;
980 if (test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
))
981 dmreq
->iv_sector
>>= cc
->sector_shift
;
984 *org_tag_of_dmreq(cc
, dmreq
) = tag_offset
;
986 sector
= org_sector_of_dmreq(cc
, dmreq
);
987 *sector
= cpu_to_le64(ctx
->cc_sector
- cc
->iv_offset
);
989 iv
= iv_of_dmreq(cc
, dmreq
);
990 org_iv
= org_iv_of_dmreq(cc
, dmreq
);
991 tag
= tag_from_dmreq(cc
, dmreq
);
992 tag_iv
= iv_tag_from_dmreq(cc
, dmreq
);
995 * |----- AAD -------|------ DATA -------|-- AUTH TAG --|
996 * | (authenticated) | (auth+encryption) | |
997 * | sector_LE | IV | sector in/out | tag in/out |
999 sg_init_table(dmreq
->sg_in
, 4);
1000 sg_set_buf(&dmreq
->sg_in
[0], sector
, sizeof(uint64_t));
1001 sg_set_buf(&dmreq
->sg_in
[1], org_iv
, cc
->iv_size
);
1002 sg_set_page(&dmreq
->sg_in
[2], bv_in
.bv_page
, cc
->sector_size
, bv_in
.bv_offset
);
1003 sg_set_buf(&dmreq
->sg_in
[3], tag
, cc
->integrity_tag_size
);
1005 sg_init_table(dmreq
->sg_out
, 4);
1006 sg_set_buf(&dmreq
->sg_out
[0], sector
, sizeof(uint64_t));
1007 sg_set_buf(&dmreq
->sg_out
[1], org_iv
, cc
->iv_size
);
1008 sg_set_page(&dmreq
->sg_out
[2], bv_out
.bv_page
, cc
->sector_size
, bv_out
.bv_offset
);
1009 sg_set_buf(&dmreq
->sg_out
[3], tag
, cc
->integrity_tag_size
);
1011 if (cc
->iv_gen_ops
) {
1012 /* For READs use IV stored in integrity metadata */
1013 if (cc
->integrity_iv_size
&& bio_data_dir(ctx
->bio_in
) != WRITE
) {
1014 memcpy(org_iv
, tag_iv
, cc
->iv_size
);
1016 r
= cc
->iv_gen_ops
->generator(cc
, org_iv
, dmreq
);
1019 /* Store generated IV in integrity metadata */
1020 if (cc
->integrity_iv_size
)
1021 memcpy(tag_iv
, org_iv
, cc
->iv_size
);
1023 /* Working copy of IV, to be modified in crypto API */
1024 memcpy(iv
, org_iv
, cc
->iv_size
);
1027 aead_request_set_ad(req
, sizeof(uint64_t) + cc
->iv_size
);
1028 if (bio_data_dir(ctx
->bio_in
) == WRITE
) {
1029 aead_request_set_crypt(req
, dmreq
->sg_in
, dmreq
->sg_out
,
1030 cc
->sector_size
, iv
);
1031 r
= crypto_aead_encrypt(req
);
1032 if (cc
->integrity_tag_size
+ cc
->integrity_iv_size
!= cc
->on_disk_tag_size
)
1033 memset(tag
+ cc
->integrity_tag_size
+ cc
->integrity_iv_size
, 0,
1034 cc
->on_disk_tag_size
- (cc
->integrity_tag_size
+ cc
->integrity_iv_size
));
1036 aead_request_set_crypt(req
, dmreq
->sg_in
, dmreq
->sg_out
,
1037 cc
->sector_size
+ cc
->integrity_tag_size
, iv
);
1038 r
= crypto_aead_decrypt(req
);
1041 if (r
== -EBADMSG
) {
1042 char b
[BDEVNAME_SIZE
];
1043 DMERR_LIMIT("%s: INTEGRITY AEAD ERROR, sector %llu", bio_devname(ctx
->bio_in
, b
),
1044 (unsigned long long)le64_to_cpu(*sector
));
1047 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1048 r
= cc
->iv_gen_ops
->post(cc
, org_iv
, dmreq
);
1050 bio_advance_iter(ctx
->bio_in
, &ctx
->iter_in
, cc
->sector_size
);
1051 bio_advance_iter(ctx
->bio_out
, &ctx
->iter_out
, cc
->sector_size
);
1056 static int crypt_convert_block_skcipher(struct crypt_config
*cc
,
1057 struct convert_context
*ctx
,
1058 struct skcipher_request
*req
,
1059 unsigned int tag_offset
)
1061 struct bio_vec bv_in
= bio_iter_iovec(ctx
->bio_in
, ctx
->iter_in
);
1062 struct bio_vec bv_out
= bio_iter_iovec(ctx
->bio_out
, ctx
->iter_out
);
1063 struct scatterlist
*sg_in
, *sg_out
;
1064 struct dm_crypt_request
*dmreq
;
1065 u8
*iv
, *org_iv
, *tag_iv
;
1069 /* Reject unexpected unaligned bio. */
1070 if (unlikely(bv_in
.bv_len
& (cc
->sector_size
- 1)))
1073 dmreq
= dmreq_of_req(cc
, req
);
1074 dmreq
->iv_sector
= ctx
->cc_sector
;
1075 if (test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
))
1076 dmreq
->iv_sector
>>= cc
->sector_shift
;
1079 *org_tag_of_dmreq(cc
, dmreq
) = tag_offset
;
1081 iv
= iv_of_dmreq(cc
, dmreq
);
1082 org_iv
= org_iv_of_dmreq(cc
, dmreq
);
1083 tag_iv
= iv_tag_from_dmreq(cc
, dmreq
);
1085 sector
= org_sector_of_dmreq(cc
, dmreq
);
1086 *sector
= cpu_to_le64(ctx
->cc_sector
- cc
->iv_offset
);
1088 /* For skcipher we use only the first sg item */
1089 sg_in
= &dmreq
->sg_in
[0];
1090 sg_out
= &dmreq
->sg_out
[0];
1092 sg_init_table(sg_in
, 1);
1093 sg_set_page(sg_in
, bv_in
.bv_page
, cc
->sector_size
, bv_in
.bv_offset
);
1095 sg_init_table(sg_out
, 1);
1096 sg_set_page(sg_out
, bv_out
.bv_page
, cc
->sector_size
, bv_out
.bv_offset
);
1098 if (cc
->iv_gen_ops
) {
1099 /* For READs use IV stored in integrity metadata */
1100 if (cc
->integrity_iv_size
&& bio_data_dir(ctx
->bio_in
) != WRITE
) {
1101 memcpy(org_iv
, tag_iv
, cc
->integrity_iv_size
);
1103 r
= cc
->iv_gen_ops
->generator(cc
, org_iv
, dmreq
);
1106 /* Store generated IV in integrity metadata */
1107 if (cc
->integrity_iv_size
)
1108 memcpy(tag_iv
, org_iv
, cc
->integrity_iv_size
);
1110 /* Working copy of IV, to be modified in crypto API */
1111 memcpy(iv
, org_iv
, cc
->iv_size
);
1114 skcipher_request_set_crypt(req
, sg_in
, sg_out
, cc
->sector_size
, iv
);
1116 if (bio_data_dir(ctx
->bio_in
) == WRITE
)
1117 r
= crypto_skcipher_encrypt(req
);
1119 r
= crypto_skcipher_decrypt(req
);
1121 if (!r
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1122 r
= cc
->iv_gen_ops
->post(cc
, org_iv
, dmreq
);
1124 bio_advance_iter(ctx
->bio_in
, &ctx
->iter_in
, cc
->sector_size
);
1125 bio_advance_iter(ctx
->bio_out
, &ctx
->iter_out
, cc
->sector_size
);
1130 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1133 static void crypt_alloc_req_skcipher(struct crypt_config
*cc
,
1134 struct convert_context
*ctx
)
1136 unsigned key_index
= ctx
->cc_sector
& (cc
->tfms_count
- 1);
1139 ctx
->r
.req
= mempool_alloc(&cc
->req_pool
, GFP_NOIO
);
1141 skcipher_request_set_tfm(ctx
->r
.req
, cc
->cipher_tfm
.tfms
[key_index
]);
1144 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1145 * requests if driver request queue is full.
1147 skcipher_request_set_callback(ctx
->r
.req
,
1148 CRYPTO_TFM_REQ_MAY_BACKLOG
,
1149 kcryptd_async_done
, dmreq_of_req(cc
, ctx
->r
.req
));
1152 static void crypt_alloc_req_aead(struct crypt_config
*cc
,
1153 struct convert_context
*ctx
)
1155 if (!ctx
->r
.req_aead
)
1156 ctx
->r
.req_aead
= mempool_alloc(&cc
->req_pool
, GFP_NOIO
);
1158 aead_request_set_tfm(ctx
->r
.req_aead
, cc
->cipher_tfm
.tfms_aead
[0]);
1161 * Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1162 * requests if driver request queue is full.
1164 aead_request_set_callback(ctx
->r
.req_aead
,
1165 CRYPTO_TFM_REQ_MAY_BACKLOG
,
1166 kcryptd_async_done
, dmreq_of_req(cc
, ctx
->r
.req_aead
));
1169 static void crypt_alloc_req(struct crypt_config
*cc
,
1170 struct convert_context
*ctx
)
1172 if (crypt_integrity_aead(cc
))
1173 crypt_alloc_req_aead(cc
, ctx
);
1175 crypt_alloc_req_skcipher(cc
, ctx
);
1178 static void crypt_free_req_skcipher(struct crypt_config
*cc
,
1179 struct skcipher_request
*req
, struct bio
*base_bio
)
1181 struct dm_crypt_io
*io
= dm_per_bio_data(base_bio
, cc
->per_bio_data_size
);
1183 if ((struct skcipher_request
*)(io
+ 1) != req
)
1184 mempool_free(req
, &cc
->req_pool
);
1187 static void crypt_free_req_aead(struct crypt_config
*cc
,
1188 struct aead_request
*req
, struct bio
*base_bio
)
1190 struct dm_crypt_io
*io
= dm_per_bio_data(base_bio
, cc
->per_bio_data_size
);
1192 if ((struct aead_request
*)(io
+ 1) != req
)
1193 mempool_free(req
, &cc
->req_pool
);
1196 static void crypt_free_req(struct crypt_config
*cc
, void *req
, struct bio
*base_bio
)
1198 if (crypt_integrity_aead(cc
))
1199 crypt_free_req_aead(cc
, req
, base_bio
);
1201 crypt_free_req_skcipher(cc
, req
, base_bio
);
1205 * Encrypt / decrypt data from one bio to another one (can be the same one)
1207 static blk_status_t
crypt_convert(struct crypt_config
*cc
,
1208 struct convert_context
*ctx
)
1210 unsigned int tag_offset
= 0;
1211 unsigned int sector_step
= cc
->sector_size
>> SECTOR_SHIFT
;
1214 atomic_set(&ctx
->cc_pending
, 1);
1216 while (ctx
->iter_in
.bi_size
&& ctx
->iter_out
.bi_size
) {
1218 crypt_alloc_req(cc
, ctx
);
1219 atomic_inc(&ctx
->cc_pending
);
1221 if (crypt_integrity_aead(cc
))
1222 r
= crypt_convert_block_aead(cc
, ctx
, ctx
->r
.req_aead
, tag_offset
);
1224 r
= crypt_convert_block_skcipher(cc
, ctx
, ctx
->r
.req
, tag_offset
);
1228 * The request was queued by a crypto driver
1229 * but the driver request queue is full, let's wait.
1232 wait_for_completion(&ctx
->restart
);
1233 reinit_completion(&ctx
->restart
);
1236 * The request is queued and processed asynchronously,
1237 * completion function kcryptd_async_done() will be called.
1241 ctx
->cc_sector
+= sector_step
;
1245 * The request was already processed (synchronously).
1248 atomic_dec(&ctx
->cc_pending
);
1249 ctx
->cc_sector
+= sector_step
;
1254 * There was a data integrity error.
1257 atomic_dec(&ctx
->cc_pending
);
1258 return BLK_STS_PROTECTION
;
1260 * There was an error while processing the request.
1263 atomic_dec(&ctx
->cc_pending
);
1264 return BLK_STS_IOERR
;
1271 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
);
1274 * Generate a new unfragmented bio with the given size
1275 * This should never violate the device limitations (but only because
1276 * max_segment_size is being constrained to PAGE_SIZE).
1278 * This function may be called concurrently. If we allocate from the mempool
1279 * concurrently, there is a possibility of deadlock. For example, if we have
1280 * mempool of 256 pages, two processes, each wanting 256, pages allocate from
1281 * the mempool concurrently, it may deadlock in a situation where both processes
1282 * have allocated 128 pages and the mempool is exhausted.
1284 * In order to avoid this scenario we allocate the pages under a mutex.
1286 * In order to not degrade performance with excessive locking, we try
1287 * non-blocking allocations without a mutex first but on failure we fallback
1288 * to blocking allocations with a mutex.
1290 static struct bio
*crypt_alloc_buffer(struct dm_crypt_io
*io
, unsigned size
)
1292 struct crypt_config
*cc
= io
->cc
;
1294 unsigned int nr_iovecs
= (size
+ PAGE_SIZE
- 1) >> PAGE_SHIFT
;
1295 gfp_t gfp_mask
= GFP_NOWAIT
| __GFP_HIGHMEM
;
1296 unsigned i
, len
, remaining_size
;
1300 if (unlikely(gfp_mask
& __GFP_DIRECT_RECLAIM
))
1301 mutex_lock(&cc
->bio_alloc_lock
);
1303 clone
= bio_alloc_bioset(GFP_NOIO
, nr_iovecs
, &cc
->bs
);
1307 clone_init(io
, clone
);
1309 remaining_size
= size
;
1311 for (i
= 0; i
< nr_iovecs
; i
++) {
1312 page
= mempool_alloc(&cc
->page_pool
, gfp_mask
);
1314 crypt_free_buffer_pages(cc
, clone
);
1316 gfp_mask
|= __GFP_DIRECT_RECLAIM
;
1320 len
= (remaining_size
> PAGE_SIZE
) ? PAGE_SIZE
: remaining_size
;
1322 bio_add_page(clone
, page
, len
, 0);
1324 remaining_size
-= len
;
1327 /* Allocate space for integrity tags */
1328 if (dm_crypt_integrity_io_alloc(io
, clone
)) {
1329 crypt_free_buffer_pages(cc
, clone
);
1334 if (unlikely(gfp_mask
& __GFP_DIRECT_RECLAIM
))
1335 mutex_unlock(&cc
->bio_alloc_lock
);
1340 static void crypt_free_buffer_pages(struct crypt_config
*cc
, struct bio
*clone
)
1343 struct bvec_iter_all iter_all
;
1345 bio_for_each_segment_all(bv
, clone
, iter_all
) {
1346 BUG_ON(!bv
->bv_page
);
1347 mempool_free(bv
->bv_page
, &cc
->page_pool
);
1351 static void crypt_io_init(struct dm_crypt_io
*io
, struct crypt_config
*cc
,
1352 struct bio
*bio
, sector_t sector
)
1356 io
->sector
= sector
;
1358 io
->ctx
.r
.req
= NULL
;
1359 io
->integrity_metadata
= NULL
;
1360 io
->integrity_metadata_from_pool
= false;
1361 atomic_set(&io
->io_pending
, 0);
1364 static void crypt_inc_pending(struct dm_crypt_io
*io
)
1366 atomic_inc(&io
->io_pending
);
1370 * One of the bios was finished. Check for completion of
1371 * the whole request and correctly clean up the buffer.
1373 static void crypt_dec_pending(struct dm_crypt_io
*io
)
1375 struct crypt_config
*cc
= io
->cc
;
1376 struct bio
*base_bio
= io
->base_bio
;
1377 blk_status_t error
= io
->error
;
1379 if (!atomic_dec_and_test(&io
->io_pending
))
1383 crypt_free_req(cc
, io
->ctx
.r
.req
, base_bio
);
1385 if (unlikely(io
->integrity_metadata_from_pool
))
1386 mempool_free(io
->integrity_metadata
, &io
->cc
->tag_pool
);
1388 kfree(io
->integrity_metadata
);
1390 base_bio
->bi_status
= error
;
1391 bio_endio(base_bio
);
1395 * kcryptd/kcryptd_io:
1397 * Needed because it would be very unwise to do decryption in an
1398 * interrupt context.
1400 * kcryptd performs the actual encryption or decryption.
1402 * kcryptd_io performs the IO submission.
1404 * They must be separated as otherwise the final stages could be
1405 * starved by new requests which can block in the first stages due
1406 * to memory allocation.
1408 * The work is done per CPU global for all dm-crypt instances.
1409 * They should not depend on each other and do not block.
1411 static void crypt_endio(struct bio
*clone
)
1413 struct dm_crypt_io
*io
= clone
->bi_private
;
1414 struct crypt_config
*cc
= io
->cc
;
1415 unsigned rw
= bio_data_dir(clone
);
1419 * free the processed pages
1422 crypt_free_buffer_pages(cc
, clone
);
1424 error
= clone
->bi_status
;
1427 if (rw
== READ
&& !error
) {
1428 kcryptd_queue_crypt(io
);
1432 if (unlikely(error
))
1435 crypt_dec_pending(io
);
1438 static void clone_init(struct dm_crypt_io
*io
, struct bio
*clone
)
1440 struct crypt_config
*cc
= io
->cc
;
1442 clone
->bi_private
= io
;
1443 clone
->bi_end_io
= crypt_endio
;
1444 bio_set_dev(clone
, cc
->dev
->bdev
);
1445 clone
->bi_opf
= io
->base_bio
->bi_opf
;
1448 static int kcryptd_io_read(struct dm_crypt_io
*io
, gfp_t gfp
)
1450 struct crypt_config
*cc
= io
->cc
;
1454 * We need the original biovec array in order to decrypt
1455 * the whole bio data *afterwards* -- thanks to immutable
1456 * biovecs we don't need to worry about the block layer
1457 * modifying the biovec array; so leverage bio_clone_fast().
1459 clone
= bio_clone_fast(io
->base_bio
, gfp
, &cc
->bs
);
1463 crypt_inc_pending(io
);
1465 clone_init(io
, clone
);
1466 clone
->bi_iter
.bi_sector
= cc
->start
+ io
->sector
;
1468 if (dm_crypt_integrity_io_alloc(io
, clone
)) {
1469 crypt_dec_pending(io
);
1474 generic_make_request(clone
);
1478 static void kcryptd_io_read_work(struct work_struct
*work
)
1480 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1482 crypt_inc_pending(io
);
1483 if (kcryptd_io_read(io
, GFP_NOIO
))
1484 io
->error
= BLK_STS_RESOURCE
;
1485 crypt_dec_pending(io
);
1488 static void kcryptd_queue_read(struct dm_crypt_io
*io
)
1490 struct crypt_config
*cc
= io
->cc
;
1492 INIT_WORK(&io
->work
, kcryptd_io_read_work
);
1493 queue_work(cc
->io_queue
, &io
->work
);
1496 static void kcryptd_io_write(struct dm_crypt_io
*io
)
1498 struct bio
*clone
= io
->ctx
.bio_out
;
1500 generic_make_request(clone
);
1503 #define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1505 static int dmcrypt_write(void *data
)
1507 struct crypt_config
*cc
= data
;
1508 struct dm_crypt_io
*io
;
1511 struct rb_root write_tree
;
1512 struct blk_plug plug
;
1514 spin_lock_irq(&cc
->write_thread_lock
);
1517 if (!RB_EMPTY_ROOT(&cc
->write_tree
))
1520 set_current_state(TASK_INTERRUPTIBLE
);
1522 spin_unlock_irq(&cc
->write_thread_lock
);
1524 if (unlikely(kthread_should_stop())) {
1525 set_current_state(TASK_RUNNING
);
1531 set_current_state(TASK_RUNNING
);
1532 spin_lock_irq(&cc
->write_thread_lock
);
1533 goto continue_locked
;
1536 write_tree
= cc
->write_tree
;
1537 cc
->write_tree
= RB_ROOT
;
1538 spin_unlock_irq(&cc
->write_thread_lock
);
1540 BUG_ON(rb_parent(write_tree
.rb_node
));
1543 * Note: we cannot walk the tree here with rb_next because
1544 * the structures may be freed when kcryptd_io_write is called.
1546 blk_start_plug(&plug
);
1548 io
= crypt_io_from_node(rb_first(&write_tree
));
1549 rb_erase(&io
->rb_node
, &write_tree
);
1550 kcryptd_io_write(io
);
1551 } while (!RB_EMPTY_ROOT(&write_tree
));
1552 blk_finish_plug(&plug
);
1557 static void kcryptd_crypt_write_io_submit(struct dm_crypt_io
*io
, int async
)
1559 struct bio
*clone
= io
->ctx
.bio_out
;
1560 struct crypt_config
*cc
= io
->cc
;
1561 unsigned long flags
;
1563 struct rb_node
**rbp
, *parent
;
1565 if (unlikely(io
->error
)) {
1566 crypt_free_buffer_pages(cc
, clone
);
1568 crypt_dec_pending(io
);
1572 /* crypt_convert should have filled the clone bio */
1573 BUG_ON(io
->ctx
.iter_out
.bi_size
);
1575 clone
->bi_iter
.bi_sector
= cc
->start
+ io
->sector
;
1577 if (likely(!async
) && test_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
)) {
1578 generic_make_request(clone
);
1582 spin_lock_irqsave(&cc
->write_thread_lock
, flags
);
1583 if (RB_EMPTY_ROOT(&cc
->write_tree
))
1584 wake_up_process(cc
->write_thread
);
1585 rbp
= &cc
->write_tree
.rb_node
;
1587 sector
= io
->sector
;
1590 if (sector
< crypt_io_from_node(parent
)->sector
)
1591 rbp
= &(*rbp
)->rb_left
;
1593 rbp
= &(*rbp
)->rb_right
;
1595 rb_link_node(&io
->rb_node
, parent
, rbp
);
1596 rb_insert_color(&io
->rb_node
, &cc
->write_tree
);
1597 spin_unlock_irqrestore(&cc
->write_thread_lock
, flags
);
1600 static void kcryptd_crypt_write_convert(struct dm_crypt_io
*io
)
1602 struct crypt_config
*cc
= io
->cc
;
1605 sector_t sector
= io
->sector
;
1609 * Prevent io from disappearing until this function completes.
1611 crypt_inc_pending(io
);
1612 crypt_convert_init(cc
, &io
->ctx
, NULL
, io
->base_bio
, sector
);
1614 clone
= crypt_alloc_buffer(io
, io
->base_bio
->bi_iter
.bi_size
);
1615 if (unlikely(!clone
)) {
1616 io
->error
= BLK_STS_IOERR
;
1620 io
->ctx
.bio_out
= clone
;
1621 io
->ctx
.iter_out
= clone
->bi_iter
;
1623 sector
+= bio_sectors(clone
);
1625 crypt_inc_pending(io
);
1626 r
= crypt_convert(cc
, &io
->ctx
);
1629 crypt_finished
= atomic_dec_and_test(&io
->ctx
.cc_pending
);
1631 /* Encryption was already finished, submit io now */
1632 if (crypt_finished
) {
1633 kcryptd_crypt_write_io_submit(io
, 0);
1634 io
->sector
= sector
;
1638 crypt_dec_pending(io
);
1641 static void kcryptd_crypt_read_done(struct dm_crypt_io
*io
)
1643 crypt_dec_pending(io
);
1646 static void kcryptd_crypt_read_convert(struct dm_crypt_io
*io
)
1648 struct crypt_config
*cc
= io
->cc
;
1651 crypt_inc_pending(io
);
1653 crypt_convert_init(cc
, &io
->ctx
, io
->base_bio
, io
->base_bio
,
1656 r
= crypt_convert(cc
, &io
->ctx
);
1660 if (atomic_dec_and_test(&io
->ctx
.cc_pending
))
1661 kcryptd_crypt_read_done(io
);
1663 crypt_dec_pending(io
);
1666 static void kcryptd_async_done(struct crypto_async_request
*async_req
,
1669 struct dm_crypt_request
*dmreq
= async_req
->data
;
1670 struct convert_context
*ctx
= dmreq
->ctx
;
1671 struct dm_crypt_io
*io
= container_of(ctx
, struct dm_crypt_io
, ctx
);
1672 struct crypt_config
*cc
= io
->cc
;
1675 * A request from crypto driver backlog is going to be processed now,
1676 * finish the completion and continue in crypt_convert().
1677 * (Callback will be called for the second time for this request.)
1679 if (error
== -EINPROGRESS
) {
1680 complete(&ctx
->restart
);
1684 if (!error
&& cc
->iv_gen_ops
&& cc
->iv_gen_ops
->post
)
1685 error
= cc
->iv_gen_ops
->post(cc
, org_iv_of_dmreq(cc
, dmreq
), dmreq
);
1687 if (error
== -EBADMSG
) {
1688 char b
[BDEVNAME_SIZE
];
1689 DMERR_LIMIT("%s: INTEGRITY AEAD ERROR, sector %llu", bio_devname(ctx
->bio_in
, b
),
1690 (unsigned long long)le64_to_cpu(*org_sector_of_dmreq(cc
, dmreq
)));
1691 io
->error
= BLK_STS_PROTECTION
;
1692 } else if (error
< 0)
1693 io
->error
= BLK_STS_IOERR
;
1695 crypt_free_req(cc
, req_of_dmreq(cc
, dmreq
), io
->base_bio
);
1697 if (!atomic_dec_and_test(&ctx
->cc_pending
))
1700 if (bio_data_dir(io
->base_bio
) == READ
)
1701 kcryptd_crypt_read_done(io
);
1703 kcryptd_crypt_write_io_submit(io
, 1);
1706 static void kcryptd_crypt(struct work_struct
*work
)
1708 struct dm_crypt_io
*io
= container_of(work
, struct dm_crypt_io
, work
);
1710 if (bio_data_dir(io
->base_bio
) == READ
)
1711 kcryptd_crypt_read_convert(io
);
1713 kcryptd_crypt_write_convert(io
);
1716 static void kcryptd_queue_crypt(struct dm_crypt_io
*io
)
1718 struct crypt_config
*cc
= io
->cc
;
1720 INIT_WORK(&io
->work
, kcryptd_crypt
);
1721 queue_work(cc
->crypt_queue
, &io
->work
);
1724 static void crypt_free_tfms_aead(struct crypt_config
*cc
)
1726 if (!cc
->cipher_tfm
.tfms_aead
)
1729 if (cc
->cipher_tfm
.tfms_aead
[0] && !IS_ERR(cc
->cipher_tfm
.tfms_aead
[0])) {
1730 crypto_free_aead(cc
->cipher_tfm
.tfms_aead
[0]);
1731 cc
->cipher_tfm
.tfms_aead
[0] = NULL
;
1734 kfree(cc
->cipher_tfm
.tfms_aead
);
1735 cc
->cipher_tfm
.tfms_aead
= NULL
;
1738 static void crypt_free_tfms_skcipher(struct crypt_config
*cc
)
1742 if (!cc
->cipher_tfm
.tfms
)
1745 for (i
= 0; i
< cc
->tfms_count
; i
++)
1746 if (cc
->cipher_tfm
.tfms
[i
] && !IS_ERR(cc
->cipher_tfm
.tfms
[i
])) {
1747 crypto_free_skcipher(cc
->cipher_tfm
.tfms
[i
]);
1748 cc
->cipher_tfm
.tfms
[i
] = NULL
;
1751 kfree(cc
->cipher_tfm
.tfms
);
1752 cc
->cipher_tfm
.tfms
= NULL
;
1755 static void crypt_free_tfms(struct crypt_config
*cc
)
1757 if (crypt_integrity_aead(cc
))
1758 crypt_free_tfms_aead(cc
);
1760 crypt_free_tfms_skcipher(cc
);
1763 static int crypt_alloc_tfms_skcipher(struct crypt_config
*cc
, char *ciphermode
)
1768 cc
->cipher_tfm
.tfms
= kcalloc(cc
->tfms_count
,
1769 sizeof(struct crypto_skcipher
*),
1771 if (!cc
->cipher_tfm
.tfms
)
1774 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1775 cc
->cipher_tfm
.tfms
[i
] = crypto_alloc_skcipher(ciphermode
, 0, 0);
1776 if (IS_ERR(cc
->cipher_tfm
.tfms
[i
])) {
1777 err
= PTR_ERR(cc
->cipher_tfm
.tfms
[i
]);
1778 crypt_free_tfms(cc
);
1784 * dm-crypt performance can vary greatly depending on which crypto
1785 * algorithm implementation is used. Help people debug performance
1786 * problems by logging the ->cra_driver_name.
1788 DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode
,
1789 crypto_skcipher_alg(any_tfm(cc
))->base
.cra_driver_name
);
1793 static int crypt_alloc_tfms_aead(struct crypt_config
*cc
, char *ciphermode
)
1797 cc
->cipher_tfm
.tfms
= kmalloc(sizeof(struct crypto_aead
*), GFP_KERNEL
);
1798 if (!cc
->cipher_tfm
.tfms
)
1801 cc
->cipher_tfm
.tfms_aead
[0] = crypto_alloc_aead(ciphermode
, 0, 0);
1802 if (IS_ERR(cc
->cipher_tfm
.tfms_aead
[0])) {
1803 err
= PTR_ERR(cc
->cipher_tfm
.tfms_aead
[0]);
1804 crypt_free_tfms(cc
);
1808 DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode
,
1809 crypto_aead_alg(any_tfm_aead(cc
))->base
.cra_driver_name
);
1813 static int crypt_alloc_tfms(struct crypt_config
*cc
, char *ciphermode
)
1815 if (crypt_integrity_aead(cc
))
1816 return crypt_alloc_tfms_aead(cc
, ciphermode
);
1818 return crypt_alloc_tfms_skcipher(cc
, ciphermode
);
1821 static unsigned crypt_subkey_size(struct crypt_config
*cc
)
1823 return (cc
->key_size
- cc
->key_extra_size
) >> ilog2(cc
->tfms_count
);
1826 static unsigned crypt_authenckey_size(struct crypt_config
*cc
)
1828 return crypt_subkey_size(cc
) + RTA_SPACE(sizeof(struct crypto_authenc_key_param
));
1832 * If AEAD is composed like authenc(hmac(sha256),xts(aes)),
1833 * the key must be for some reason in special format.
1834 * This funcion converts cc->key to this special format.
1836 static void crypt_copy_authenckey(char *p
, const void *key
,
1837 unsigned enckeylen
, unsigned authkeylen
)
1839 struct crypto_authenc_key_param
*param
;
1842 rta
= (struct rtattr
*)p
;
1843 param
= RTA_DATA(rta
);
1844 param
->enckeylen
= cpu_to_be32(enckeylen
);
1845 rta
->rta_len
= RTA_LENGTH(sizeof(*param
));
1846 rta
->rta_type
= CRYPTO_AUTHENC_KEYA_PARAM
;
1847 p
+= RTA_SPACE(sizeof(*param
));
1848 memcpy(p
, key
+ enckeylen
, authkeylen
);
1850 memcpy(p
, key
, enckeylen
);
1853 static int crypt_setkey(struct crypt_config
*cc
)
1855 unsigned subkey_size
;
1858 /* Ignore extra keys (which are used for IV etc) */
1859 subkey_size
= crypt_subkey_size(cc
);
1861 if (crypt_integrity_hmac(cc
)) {
1862 if (subkey_size
< cc
->key_mac_size
)
1865 crypt_copy_authenckey(cc
->authenc_key
, cc
->key
,
1866 subkey_size
- cc
->key_mac_size
,
1870 for (i
= 0; i
< cc
->tfms_count
; i
++) {
1871 if (crypt_integrity_hmac(cc
))
1872 r
= crypto_aead_setkey(cc
->cipher_tfm
.tfms_aead
[i
],
1873 cc
->authenc_key
, crypt_authenckey_size(cc
));
1874 else if (crypt_integrity_aead(cc
))
1875 r
= crypto_aead_setkey(cc
->cipher_tfm
.tfms_aead
[i
],
1876 cc
->key
+ (i
* subkey_size
),
1879 r
= crypto_skcipher_setkey(cc
->cipher_tfm
.tfms
[i
],
1880 cc
->key
+ (i
* subkey_size
),
1886 if (crypt_integrity_hmac(cc
))
1887 memzero_explicit(cc
->authenc_key
, crypt_authenckey_size(cc
));
1894 static bool contains_whitespace(const char *str
)
1897 if (isspace(*str
++))
1902 static int crypt_set_keyring_key(struct crypt_config
*cc
, const char *key_string
)
1904 char *new_key_string
, *key_desc
;
1907 const struct user_key_payload
*ukp
;
1910 * Reject key_string with whitespace. dm core currently lacks code for
1911 * proper whitespace escaping in arguments on DM_TABLE_STATUS path.
1913 if (contains_whitespace(key_string
)) {
1914 DMERR("whitespace chars not allowed in key string");
1918 /* look for next ':' separating key_type from key_description */
1919 key_desc
= strpbrk(key_string
, ":");
1920 if (!key_desc
|| key_desc
== key_string
|| !strlen(key_desc
+ 1))
1923 if (strncmp(key_string
, "logon:", key_desc
- key_string
+ 1) &&
1924 strncmp(key_string
, "user:", key_desc
- key_string
+ 1))
1927 new_key_string
= kstrdup(key_string
, GFP_KERNEL
);
1928 if (!new_key_string
)
1931 key
= request_key(key_string
[0] == 'l' ? &key_type_logon
: &key_type_user
,
1932 key_desc
+ 1, NULL
);
1934 kzfree(new_key_string
);
1935 return PTR_ERR(key
);
1938 down_read(&key
->sem
);
1940 ukp
= user_key_payload_locked(key
);
1944 kzfree(new_key_string
);
1945 return -EKEYREVOKED
;
1948 if (cc
->key_size
!= ukp
->datalen
) {
1951 kzfree(new_key_string
);
1955 memcpy(cc
->key
, ukp
->data
, cc
->key_size
);
1960 /* clear the flag since following operations may invalidate previously valid key */
1961 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1963 ret
= crypt_setkey(cc
);
1966 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
1967 kzfree(cc
->key_string
);
1968 cc
->key_string
= new_key_string
;
1970 kzfree(new_key_string
);
1975 static int get_key_size(char **key_string
)
1980 if (*key_string
[0] != ':')
1981 return strlen(*key_string
) >> 1;
1983 /* look for next ':' in key string */
1984 colon
= strpbrk(*key_string
+ 1, ":");
1988 if (sscanf(*key_string
+ 1, "%u%c", &ret
, &dummy
) != 2 || dummy
!= ':')
1991 *key_string
= colon
;
1993 /* remaining key string should be :<logon|user>:<key_desc> */
2000 static int crypt_set_keyring_key(struct crypt_config
*cc
, const char *key_string
)
2005 static int get_key_size(char **key_string
)
2007 return (*key_string
[0] == ':') ? -EINVAL
: strlen(*key_string
) >> 1;
2012 static int crypt_set_key(struct crypt_config
*cc
, char *key
)
2015 int key_string_len
= strlen(key
);
2017 /* Hyphen (which gives a key_size of zero) means there is no key. */
2018 if (!cc
->key_size
&& strcmp(key
, "-"))
2021 /* ':' means the key is in kernel keyring, short-circuit normal key processing */
2022 if (key
[0] == ':') {
2023 r
= crypt_set_keyring_key(cc
, key
+ 1);
2027 /* clear the flag since following operations may invalidate previously valid key */
2028 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2030 /* wipe references to any kernel keyring key */
2031 kzfree(cc
->key_string
);
2032 cc
->key_string
= NULL
;
2034 /* Decode key from its hex representation. */
2035 if (cc
->key_size
&& hex2bin(cc
->key
, key
, cc
->key_size
) < 0)
2038 r
= crypt_setkey(cc
);
2040 set_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2043 /* Hex key string not needed after here, so wipe it. */
2044 memset(key
, '0', key_string_len
);
2049 static int crypt_wipe_key(struct crypt_config
*cc
)
2053 clear_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
);
2054 get_random_bytes(&cc
->key
, cc
->key_size
);
2056 /* Wipe IV private keys */
2057 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->wipe
) {
2058 r
= cc
->iv_gen_ops
->wipe(cc
);
2063 kzfree(cc
->key_string
);
2064 cc
->key_string
= NULL
;
2065 r
= crypt_setkey(cc
);
2066 memset(&cc
->key
, 0, cc
->key_size
* sizeof(u8
));
2071 static void crypt_calculate_pages_per_client(void)
2073 unsigned long pages
= (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT
/ 100;
2075 if (!dm_crypt_clients_n
)
2078 pages
/= dm_crypt_clients_n
;
2079 if (pages
< DM_CRYPT_MIN_PAGES_PER_CLIENT
)
2080 pages
= DM_CRYPT_MIN_PAGES_PER_CLIENT
;
2081 dm_crypt_pages_per_client
= pages
;
2084 static void *crypt_page_alloc(gfp_t gfp_mask
, void *pool_data
)
2086 struct crypt_config
*cc
= pool_data
;
2089 if (unlikely(percpu_counter_compare(&cc
->n_allocated_pages
, dm_crypt_pages_per_client
) >= 0) &&
2090 likely(gfp_mask
& __GFP_NORETRY
))
2093 page
= alloc_page(gfp_mask
);
2094 if (likely(page
!= NULL
))
2095 percpu_counter_add(&cc
->n_allocated_pages
, 1);
2100 static void crypt_page_free(void *page
, void *pool_data
)
2102 struct crypt_config
*cc
= pool_data
;
2105 percpu_counter_sub(&cc
->n_allocated_pages
, 1);
2108 static void crypt_dtr(struct dm_target
*ti
)
2110 struct crypt_config
*cc
= ti
->private;
2117 if (cc
->write_thread
)
2118 kthread_stop(cc
->write_thread
);
2121 destroy_workqueue(cc
->io_queue
);
2122 if (cc
->crypt_queue
)
2123 destroy_workqueue(cc
->crypt_queue
);
2125 crypt_free_tfms(cc
);
2127 bioset_exit(&cc
->bs
);
2129 mempool_exit(&cc
->page_pool
);
2130 mempool_exit(&cc
->req_pool
);
2131 mempool_exit(&cc
->tag_pool
);
2133 WARN_ON(percpu_counter_sum(&cc
->n_allocated_pages
) != 0);
2134 percpu_counter_destroy(&cc
->n_allocated_pages
);
2136 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->dtr
)
2137 cc
->iv_gen_ops
->dtr(cc
);
2140 dm_put_device(ti
, cc
->dev
);
2142 kzfree(cc
->cipher_string
);
2143 kzfree(cc
->key_string
);
2144 kzfree(cc
->cipher_auth
);
2145 kzfree(cc
->authenc_key
);
2147 mutex_destroy(&cc
->bio_alloc_lock
);
2149 /* Must zero key material before freeing */
2152 spin_lock(&dm_crypt_clients_lock
);
2153 WARN_ON(!dm_crypt_clients_n
);
2154 dm_crypt_clients_n
--;
2155 crypt_calculate_pages_per_client();
2156 spin_unlock(&dm_crypt_clients_lock
);
2159 static int crypt_ctr_ivmode(struct dm_target
*ti
, const char *ivmode
)
2161 struct crypt_config
*cc
= ti
->private;
2163 if (crypt_integrity_aead(cc
))
2164 cc
->iv_size
= crypto_aead_ivsize(any_tfm_aead(cc
));
2166 cc
->iv_size
= crypto_skcipher_ivsize(any_tfm(cc
));
2169 /* at least a 64 bit sector number should fit in our buffer */
2170 cc
->iv_size
= max(cc
->iv_size
,
2171 (unsigned int)(sizeof(u64
) / sizeof(u8
)));
2173 DMWARN("Selected cipher does not support IVs");
2177 /* Choose ivmode, see comments at iv code. */
2179 cc
->iv_gen_ops
= NULL
;
2180 else if (strcmp(ivmode
, "plain") == 0)
2181 cc
->iv_gen_ops
= &crypt_iv_plain_ops
;
2182 else if (strcmp(ivmode
, "plain64") == 0)
2183 cc
->iv_gen_ops
= &crypt_iv_plain64_ops
;
2184 else if (strcmp(ivmode
, "plain64be") == 0)
2185 cc
->iv_gen_ops
= &crypt_iv_plain64be_ops
;
2186 else if (strcmp(ivmode
, "essiv") == 0)
2187 cc
->iv_gen_ops
= &crypt_iv_essiv_ops
;
2188 else if (strcmp(ivmode
, "benbi") == 0)
2189 cc
->iv_gen_ops
= &crypt_iv_benbi_ops
;
2190 else if (strcmp(ivmode
, "null") == 0)
2191 cc
->iv_gen_ops
= &crypt_iv_null_ops
;
2192 else if (strcmp(ivmode
, "eboiv") == 0)
2193 cc
->iv_gen_ops
= &crypt_iv_eboiv_ops
;
2194 else if (strcmp(ivmode
, "lmk") == 0) {
2195 cc
->iv_gen_ops
= &crypt_iv_lmk_ops
;
2197 * Version 2 and 3 is recognised according
2198 * to length of provided multi-key string.
2199 * If present (version 3), last key is used as IV seed.
2200 * All keys (including IV seed) are always the same size.
2202 if (cc
->key_size
% cc
->key_parts
) {
2204 cc
->key_extra_size
= cc
->key_size
/ cc
->key_parts
;
2206 } else if (strcmp(ivmode
, "tcw") == 0) {
2207 cc
->iv_gen_ops
= &crypt_iv_tcw_ops
;
2208 cc
->key_parts
+= 2; /* IV + whitening */
2209 cc
->key_extra_size
= cc
->iv_size
+ TCW_WHITENING_SIZE
;
2210 } else if (strcmp(ivmode
, "random") == 0) {
2211 cc
->iv_gen_ops
= &crypt_iv_random_ops
;
2212 /* Need storage space in integrity fields. */
2213 cc
->integrity_iv_size
= cc
->iv_size
;
2215 ti
->error
= "Invalid IV mode";
2223 * Workaround to parse HMAC algorithm from AEAD crypto API spec.
2224 * The HMAC is needed to calculate tag size (HMAC digest size).
2225 * This should be probably done by crypto-api calls (once available...)
2227 static int crypt_ctr_auth_cipher(struct crypt_config
*cc
, char *cipher_api
)
2229 char *start
, *end
, *mac_alg
= NULL
;
2230 struct crypto_ahash
*mac
;
2232 if (!strstarts(cipher_api
, "authenc("))
2235 start
= strchr(cipher_api
, '(');
2236 end
= strchr(cipher_api
, ',');
2237 if (!start
|| !end
|| ++start
> end
)
2240 mac_alg
= kzalloc(end
- start
+ 1, GFP_KERNEL
);
2243 strncpy(mac_alg
, start
, end
- start
);
2245 mac
= crypto_alloc_ahash(mac_alg
, 0, 0);
2249 return PTR_ERR(mac
);
2251 cc
->key_mac_size
= crypto_ahash_digestsize(mac
);
2252 crypto_free_ahash(mac
);
2254 cc
->authenc_key
= kmalloc(crypt_authenckey_size(cc
), GFP_KERNEL
);
2255 if (!cc
->authenc_key
)
2261 static int crypt_ctr_cipher_new(struct dm_target
*ti
, char *cipher_in
, char *key
,
2262 char **ivmode
, char **ivopts
)
2264 struct crypt_config
*cc
= ti
->private;
2265 char *tmp
, *cipher_api
, buf
[CRYPTO_MAX_ALG_NAME
];
2271 * New format (capi: prefix)
2272 * capi:cipher_api_spec-iv:ivopts
2274 tmp
= &cipher_in
[strlen("capi:")];
2276 /* Separate IV options if present, it can contain another '-' in hash name */
2277 *ivopts
= strrchr(tmp
, ':');
2283 *ivmode
= strrchr(tmp
, '-');
2288 /* The rest is crypto API spec */
2291 /* Alloc AEAD, can be used only in new format. */
2292 if (crypt_integrity_aead(cc
)) {
2293 ret
= crypt_ctr_auth_cipher(cc
, cipher_api
);
2295 ti
->error
= "Invalid AEAD cipher spec";
2300 if (*ivmode
&& !strcmp(*ivmode
, "lmk"))
2301 cc
->tfms_count
= 64;
2303 if (*ivmode
&& !strcmp(*ivmode
, "essiv")) {
2305 ti
->error
= "Digest algorithm missing for ESSIV mode";
2308 ret
= snprintf(buf
, CRYPTO_MAX_ALG_NAME
, "essiv(%s,%s)",
2309 cipher_api
, *ivopts
);
2310 if (ret
< 0 || ret
>= CRYPTO_MAX_ALG_NAME
) {
2311 ti
->error
= "Cannot allocate cipher string";
2317 cc
->key_parts
= cc
->tfms_count
;
2319 /* Allocate cipher */
2320 ret
= crypt_alloc_tfms(cc
, cipher_api
);
2322 ti
->error
= "Error allocating crypto tfm";
2326 if (crypt_integrity_aead(cc
))
2327 cc
->iv_size
= crypto_aead_ivsize(any_tfm_aead(cc
));
2329 cc
->iv_size
= crypto_skcipher_ivsize(any_tfm(cc
));
2334 static int crypt_ctr_cipher_old(struct dm_target
*ti
, char *cipher_in
, char *key
,
2335 char **ivmode
, char **ivopts
)
2337 struct crypt_config
*cc
= ti
->private;
2338 char *tmp
, *cipher
, *chainmode
, *keycount
;
2339 char *cipher_api
= NULL
;
2343 if (strchr(cipher_in
, '(') || crypt_integrity_aead(cc
)) {
2344 ti
->error
= "Bad cipher specification";
2349 * Legacy dm-crypt cipher specification
2350 * cipher[:keycount]-mode-iv:ivopts
2353 keycount
= strsep(&tmp
, "-");
2354 cipher
= strsep(&keycount
, ":");
2358 else if (sscanf(keycount
, "%u%c", &cc
->tfms_count
, &dummy
) != 1 ||
2359 !is_power_of_2(cc
->tfms_count
)) {
2360 ti
->error
= "Bad cipher key count specification";
2363 cc
->key_parts
= cc
->tfms_count
;
2365 chainmode
= strsep(&tmp
, "-");
2366 *ivmode
= strsep(&tmp
, ":");
2370 * For compatibility with the original dm-crypt mapping format, if
2371 * only the cipher name is supplied, use cbc-plain.
2373 if (!chainmode
|| (!strcmp(chainmode
, "plain") && !*ivmode
)) {
2378 if (strcmp(chainmode
, "ecb") && !*ivmode
) {
2379 ti
->error
= "IV mechanism required";
2383 cipher_api
= kmalloc(CRYPTO_MAX_ALG_NAME
, GFP_KERNEL
);
2387 if (*ivmode
&& !strcmp(*ivmode
, "essiv")) {
2389 ti
->error
= "Digest algorithm missing for ESSIV mode";
2393 ret
= snprintf(cipher_api
, CRYPTO_MAX_ALG_NAME
,
2394 "essiv(%s(%s),%s)", chainmode
, cipher
, *ivopts
);
2396 ret
= snprintf(cipher_api
, CRYPTO_MAX_ALG_NAME
,
2397 "%s(%s)", chainmode
, cipher
);
2399 if (ret
< 0 || ret
>= CRYPTO_MAX_ALG_NAME
) {
2404 /* Allocate cipher */
2405 ret
= crypt_alloc_tfms(cc
, cipher_api
);
2407 ti
->error
= "Error allocating crypto tfm";
2415 ti
->error
= "Cannot allocate cipher strings";
2419 static int crypt_ctr_cipher(struct dm_target
*ti
, char *cipher_in
, char *key
)
2421 struct crypt_config
*cc
= ti
->private;
2422 char *ivmode
= NULL
, *ivopts
= NULL
;
2425 cc
->cipher_string
= kstrdup(cipher_in
, GFP_KERNEL
);
2426 if (!cc
->cipher_string
) {
2427 ti
->error
= "Cannot allocate cipher strings";
2431 if (strstarts(cipher_in
, "capi:"))
2432 ret
= crypt_ctr_cipher_new(ti
, cipher_in
, key
, &ivmode
, &ivopts
);
2434 ret
= crypt_ctr_cipher_old(ti
, cipher_in
, key
, &ivmode
, &ivopts
);
2439 ret
= crypt_ctr_ivmode(ti
, ivmode
);
2443 /* Initialize and set key */
2444 ret
= crypt_set_key(cc
, key
);
2446 ti
->error
= "Error decoding and setting key";
2451 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->ctr
) {
2452 ret
= cc
->iv_gen_ops
->ctr(cc
, ti
, ivopts
);
2454 ti
->error
= "Error creating IV";
2459 /* Initialize IV (set keys for ESSIV etc) */
2460 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
) {
2461 ret
= cc
->iv_gen_ops
->init(cc
);
2463 ti
->error
= "Error initialising IV";
2468 /* wipe the kernel key payload copy */
2470 memset(cc
->key
, 0, cc
->key_size
* sizeof(u8
));
2475 static int crypt_ctr_optional(struct dm_target
*ti
, unsigned int argc
, char **argv
)
2477 struct crypt_config
*cc
= ti
->private;
2478 struct dm_arg_set as
;
2479 static const struct dm_arg _args
[] = {
2480 {0, 6, "Invalid number of feature args"},
2482 unsigned int opt_params
, val
;
2483 const char *opt_string
, *sval
;
2487 /* Optional parameters */
2491 ret
= dm_read_arg_group(_args
, &as
, &opt_params
, &ti
->error
);
2495 while (opt_params
--) {
2496 opt_string
= dm_shift_arg(&as
);
2498 ti
->error
= "Not enough feature arguments";
2502 if (!strcasecmp(opt_string
, "allow_discards"))
2503 ti
->num_discard_bios
= 1;
2505 else if (!strcasecmp(opt_string
, "same_cpu_crypt"))
2506 set_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
);
2508 else if (!strcasecmp(opt_string
, "submit_from_crypt_cpus"))
2509 set_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
);
2510 else if (sscanf(opt_string
, "integrity:%u:", &val
) == 1) {
2511 if (val
== 0 || val
> MAX_TAG_SIZE
) {
2512 ti
->error
= "Invalid integrity arguments";
2515 cc
->on_disk_tag_size
= val
;
2516 sval
= strchr(opt_string
+ strlen("integrity:"), ':') + 1;
2517 if (!strcasecmp(sval
, "aead")) {
2518 set_bit(CRYPT_MODE_INTEGRITY_AEAD
, &cc
->cipher_flags
);
2519 } else if (strcasecmp(sval
, "none")) {
2520 ti
->error
= "Unknown integrity profile";
2524 cc
->cipher_auth
= kstrdup(sval
, GFP_KERNEL
);
2525 if (!cc
->cipher_auth
)
2527 } else if (sscanf(opt_string
, "sector_size:%hu%c", &cc
->sector_size
, &dummy
) == 1) {
2528 if (cc
->sector_size
< (1 << SECTOR_SHIFT
) ||
2529 cc
->sector_size
> 4096 ||
2530 (cc
->sector_size
& (cc
->sector_size
- 1))) {
2531 ti
->error
= "Invalid feature value for sector_size";
2534 if (ti
->len
& ((cc
->sector_size
>> SECTOR_SHIFT
) - 1)) {
2535 ti
->error
= "Device size is not multiple of sector_size feature";
2538 cc
->sector_shift
= __ffs(cc
->sector_size
) - SECTOR_SHIFT
;
2539 } else if (!strcasecmp(opt_string
, "iv_large_sectors"))
2540 set_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
);
2542 ti
->error
= "Invalid feature arguments";
2551 * Construct an encryption mapping:
2552 * <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
2554 static int crypt_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
2556 struct crypt_config
*cc
;
2557 const char *devname
= dm_table_device_name(ti
->table
);
2559 unsigned int align_mask
;
2560 unsigned long long tmpll
;
2562 size_t iv_size_padding
, additional_req_size
;
2566 ti
->error
= "Not enough arguments";
2570 key_size
= get_key_size(&argv
[1]);
2572 ti
->error
= "Cannot parse key size";
2576 cc
= kzalloc(struct_size(cc
, key
, key_size
), GFP_KERNEL
);
2578 ti
->error
= "Cannot allocate encryption context";
2581 cc
->key_size
= key_size
;
2582 cc
->sector_size
= (1 << SECTOR_SHIFT
);
2583 cc
->sector_shift
= 0;
2587 spin_lock(&dm_crypt_clients_lock
);
2588 dm_crypt_clients_n
++;
2589 crypt_calculate_pages_per_client();
2590 spin_unlock(&dm_crypt_clients_lock
);
2592 ret
= percpu_counter_init(&cc
->n_allocated_pages
, 0, GFP_KERNEL
);
2596 /* Optional parameters need to be read before cipher constructor */
2598 ret
= crypt_ctr_optional(ti
, argc
- 5, &argv
[5]);
2603 ret
= crypt_ctr_cipher(ti
, argv
[0], argv
[1]);
2607 if (crypt_integrity_aead(cc
)) {
2608 cc
->dmreq_start
= sizeof(struct aead_request
);
2609 cc
->dmreq_start
+= crypto_aead_reqsize(any_tfm_aead(cc
));
2610 align_mask
= crypto_aead_alignmask(any_tfm_aead(cc
));
2612 cc
->dmreq_start
= sizeof(struct skcipher_request
);
2613 cc
->dmreq_start
+= crypto_skcipher_reqsize(any_tfm(cc
));
2614 align_mask
= crypto_skcipher_alignmask(any_tfm(cc
));
2616 cc
->dmreq_start
= ALIGN(cc
->dmreq_start
, __alignof__(struct dm_crypt_request
));
2618 if (align_mask
< CRYPTO_MINALIGN
) {
2619 /* Allocate the padding exactly */
2620 iv_size_padding
= -(cc
->dmreq_start
+ sizeof(struct dm_crypt_request
))
2624 * If the cipher requires greater alignment than kmalloc
2625 * alignment, we don't know the exact position of the
2626 * initialization vector. We must assume worst case.
2628 iv_size_padding
= align_mask
;
2631 /* ...| IV + padding | original IV | original sec. number | bio tag offset | */
2632 additional_req_size
= sizeof(struct dm_crypt_request
) +
2633 iv_size_padding
+ cc
->iv_size
+
2636 sizeof(unsigned int);
2638 ret
= mempool_init_kmalloc_pool(&cc
->req_pool
, MIN_IOS
, cc
->dmreq_start
+ additional_req_size
);
2640 ti
->error
= "Cannot allocate crypt request mempool";
2644 cc
->per_bio_data_size
= ti
->per_io_data_size
=
2645 ALIGN(sizeof(struct dm_crypt_io
) + cc
->dmreq_start
+ additional_req_size
,
2646 ARCH_KMALLOC_MINALIGN
);
2648 ret
= mempool_init(&cc
->page_pool
, BIO_MAX_PAGES
, crypt_page_alloc
, crypt_page_free
, cc
);
2650 ti
->error
= "Cannot allocate page mempool";
2654 ret
= bioset_init(&cc
->bs
, MIN_IOS
, 0, BIOSET_NEED_BVECS
);
2656 ti
->error
= "Cannot allocate crypt bioset";
2660 mutex_init(&cc
->bio_alloc_lock
);
2663 if ((sscanf(argv
[2], "%llu%c", &tmpll
, &dummy
) != 1) ||
2664 (tmpll
& ((cc
->sector_size
>> SECTOR_SHIFT
) - 1))) {
2665 ti
->error
= "Invalid iv_offset sector";
2668 cc
->iv_offset
= tmpll
;
2670 ret
= dm_get_device(ti
, argv
[3], dm_table_get_mode(ti
->table
), &cc
->dev
);
2672 ti
->error
= "Device lookup failed";
2677 if (sscanf(argv
[4], "%llu%c", &tmpll
, &dummy
) != 1 || tmpll
!= (sector_t
)tmpll
) {
2678 ti
->error
= "Invalid device sector";
2683 if (crypt_integrity_aead(cc
) || cc
->integrity_iv_size
) {
2684 ret
= crypt_integrity_ctr(cc
, ti
);
2688 cc
->tag_pool_max_sectors
= POOL_ENTRY_SIZE
/ cc
->on_disk_tag_size
;
2689 if (!cc
->tag_pool_max_sectors
)
2690 cc
->tag_pool_max_sectors
= 1;
2692 ret
= mempool_init_kmalloc_pool(&cc
->tag_pool
, MIN_IOS
,
2693 cc
->tag_pool_max_sectors
* cc
->on_disk_tag_size
);
2695 ti
->error
= "Cannot allocate integrity tags mempool";
2699 cc
->tag_pool_max_sectors
<<= cc
->sector_shift
;
2703 cc
->io_queue
= alloc_workqueue("kcryptd_io/%s", WQ_MEM_RECLAIM
, 1, devname
);
2704 if (!cc
->io_queue
) {
2705 ti
->error
= "Couldn't create kcryptd io queue";
2709 if (test_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
))
2710 cc
->crypt_queue
= alloc_workqueue("kcryptd/%s", WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
,
2713 cc
->crypt_queue
= alloc_workqueue("kcryptd/%s",
2714 WQ_CPU_INTENSIVE
| WQ_MEM_RECLAIM
| WQ_UNBOUND
,
2715 num_online_cpus(), devname
);
2716 if (!cc
->crypt_queue
) {
2717 ti
->error
= "Couldn't create kcryptd queue";
2721 spin_lock_init(&cc
->write_thread_lock
);
2722 cc
->write_tree
= RB_ROOT
;
2724 cc
->write_thread
= kthread_create(dmcrypt_write
, cc
, "dmcrypt_write/%s", devname
);
2725 if (IS_ERR(cc
->write_thread
)) {
2726 ret
= PTR_ERR(cc
->write_thread
);
2727 cc
->write_thread
= NULL
;
2728 ti
->error
= "Couldn't spawn write thread";
2731 wake_up_process(cc
->write_thread
);
2733 ti
->num_flush_bios
= 1;
2742 static int crypt_map(struct dm_target
*ti
, struct bio
*bio
)
2744 struct dm_crypt_io
*io
;
2745 struct crypt_config
*cc
= ti
->private;
2748 * If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
2749 * - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
2750 * - for REQ_OP_DISCARD caller must use flush if IO ordering matters
2752 if (unlikely(bio
->bi_opf
& REQ_PREFLUSH
||
2753 bio_op(bio
) == REQ_OP_DISCARD
)) {
2754 bio_set_dev(bio
, cc
->dev
->bdev
);
2755 if (bio_sectors(bio
))
2756 bio
->bi_iter
.bi_sector
= cc
->start
+
2757 dm_target_offset(ti
, bio
->bi_iter
.bi_sector
);
2758 return DM_MAPIO_REMAPPED
;
2762 * Check if bio is too large, split as needed.
2764 if (unlikely(bio
->bi_iter
.bi_size
> (BIO_MAX_PAGES
<< PAGE_SHIFT
)) &&
2765 (bio_data_dir(bio
) == WRITE
|| cc
->on_disk_tag_size
))
2766 dm_accept_partial_bio(bio
, ((BIO_MAX_PAGES
<< PAGE_SHIFT
) >> SECTOR_SHIFT
));
2769 * Ensure that bio is a multiple of internal sector encryption size
2770 * and is aligned to this size as defined in IO hints.
2772 if (unlikely((bio
->bi_iter
.bi_sector
& ((cc
->sector_size
>> SECTOR_SHIFT
) - 1)) != 0))
2773 return DM_MAPIO_KILL
;
2775 if (unlikely(bio
->bi_iter
.bi_size
& (cc
->sector_size
- 1)))
2776 return DM_MAPIO_KILL
;
2778 io
= dm_per_bio_data(bio
, cc
->per_bio_data_size
);
2779 crypt_io_init(io
, cc
, bio
, dm_target_offset(ti
, bio
->bi_iter
.bi_sector
));
2781 if (cc
->on_disk_tag_size
) {
2782 unsigned tag_len
= cc
->on_disk_tag_size
* (bio_sectors(bio
) >> cc
->sector_shift
);
2784 if (unlikely(tag_len
> KMALLOC_MAX_SIZE
) ||
2785 unlikely(!(io
->integrity_metadata
= kmalloc(tag_len
,
2786 GFP_NOIO
| __GFP_NORETRY
| __GFP_NOMEMALLOC
| __GFP_NOWARN
)))) {
2787 if (bio_sectors(bio
) > cc
->tag_pool_max_sectors
)
2788 dm_accept_partial_bio(bio
, cc
->tag_pool_max_sectors
);
2789 io
->integrity_metadata
= mempool_alloc(&cc
->tag_pool
, GFP_NOIO
);
2790 io
->integrity_metadata_from_pool
= true;
2794 if (crypt_integrity_aead(cc
))
2795 io
->ctx
.r
.req_aead
= (struct aead_request
*)(io
+ 1);
2797 io
->ctx
.r
.req
= (struct skcipher_request
*)(io
+ 1);
2799 if (bio_data_dir(io
->base_bio
) == READ
) {
2800 if (kcryptd_io_read(io
, GFP_NOWAIT
))
2801 kcryptd_queue_read(io
);
2803 kcryptd_queue_crypt(io
);
2805 return DM_MAPIO_SUBMITTED
;
2808 static void crypt_status(struct dm_target
*ti
, status_type_t type
,
2809 unsigned status_flags
, char *result
, unsigned maxlen
)
2811 struct crypt_config
*cc
= ti
->private;
2813 int num_feature_args
= 0;
2816 case STATUSTYPE_INFO
:
2820 case STATUSTYPE_TABLE
:
2821 DMEMIT("%s ", cc
->cipher_string
);
2823 if (cc
->key_size
> 0) {
2825 DMEMIT(":%u:%s", cc
->key_size
, cc
->key_string
);
2827 for (i
= 0; i
< cc
->key_size
; i
++)
2828 DMEMIT("%02x", cc
->key
[i
]);
2832 DMEMIT(" %llu %s %llu", (unsigned long long)cc
->iv_offset
,
2833 cc
->dev
->name
, (unsigned long long)cc
->start
);
2835 num_feature_args
+= !!ti
->num_discard_bios
;
2836 num_feature_args
+= test_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
);
2837 num_feature_args
+= test_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
);
2838 num_feature_args
+= cc
->sector_size
!= (1 << SECTOR_SHIFT
);
2839 num_feature_args
+= test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
);
2840 if (cc
->on_disk_tag_size
)
2842 if (num_feature_args
) {
2843 DMEMIT(" %d", num_feature_args
);
2844 if (ti
->num_discard_bios
)
2845 DMEMIT(" allow_discards");
2846 if (test_bit(DM_CRYPT_SAME_CPU
, &cc
->flags
))
2847 DMEMIT(" same_cpu_crypt");
2848 if (test_bit(DM_CRYPT_NO_OFFLOAD
, &cc
->flags
))
2849 DMEMIT(" submit_from_crypt_cpus");
2850 if (cc
->on_disk_tag_size
)
2851 DMEMIT(" integrity:%u:%s", cc
->on_disk_tag_size
, cc
->cipher_auth
);
2852 if (cc
->sector_size
!= (1 << SECTOR_SHIFT
))
2853 DMEMIT(" sector_size:%d", cc
->sector_size
);
2854 if (test_bit(CRYPT_IV_LARGE_SECTORS
, &cc
->cipher_flags
))
2855 DMEMIT(" iv_large_sectors");
2862 static void crypt_postsuspend(struct dm_target
*ti
)
2864 struct crypt_config
*cc
= ti
->private;
2866 set_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
2869 static int crypt_preresume(struct dm_target
*ti
)
2871 struct crypt_config
*cc
= ti
->private;
2873 if (!test_bit(DM_CRYPT_KEY_VALID
, &cc
->flags
)) {
2874 DMERR("aborting resume - crypt key is not set.");
2881 static void crypt_resume(struct dm_target
*ti
)
2883 struct crypt_config
*cc
= ti
->private;
2885 clear_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
);
2888 /* Message interface
2892 static int crypt_message(struct dm_target
*ti
, unsigned argc
, char **argv
,
2893 char *result
, unsigned maxlen
)
2895 struct crypt_config
*cc
= ti
->private;
2896 int key_size
, ret
= -EINVAL
;
2901 if (!strcasecmp(argv
[0], "key")) {
2902 if (!test_bit(DM_CRYPT_SUSPENDED
, &cc
->flags
)) {
2903 DMWARN("not suspended during key manipulation.");
2906 if (argc
== 3 && !strcasecmp(argv
[1], "set")) {
2907 /* The key size may not be changed. */
2908 key_size
= get_key_size(&argv
[2]);
2909 if (key_size
< 0 || cc
->key_size
!= key_size
) {
2910 memset(argv
[2], '0', strlen(argv
[2]));
2914 ret
= crypt_set_key(cc
, argv
[2]);
2917 if (cc
->iv_gen_ops
&& cc
->iv_gen_ops
->init
)
2918 ret
= cc
->iv_gen_ops
->init(cc
);
2919 /* wipe the kernel key payload copy */
2921 memset(cc
->key
, 0, cc
->key_size
* sizeof(u8
));
2924 if (argc
== 2 && !strcasecmp(argv
[1], "wipe"))
2925 return crypt_wipe_key(cc
);
2929 DMWARN("unrecognised message received.");
2933 static int crypt_iterate_devices(struct dm_target
*ti
,
2934 iterate_devices_callout_fn fn
, void *data
)
2936 struct crypt_config
*cc
= ti
->private;
2938 return fn(ti
, cc
->dev
, cc
->start
, ti
->len
, data
);
2941 static void crypt_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
2943 struct crypt_config
*cc
= ti
->private;
2946 * Unfortunate constraint that is required to avoid the potential
2947 * for exceeding underlying device's max_segments limits -- due to
2948 * crypt_alloc_buffer() possibly allocating pages for the encryption
2949 * bio that are not as physically contiguous as the original bio.
2951 limits
->max_segment_size
= PAGE_SIZE
;
2953 limits
->logical_block_size
=
2954 max_t(unsigned short, limits
->logical_block_size
, cc
->sector_size
);
2955 limits
->physical_block_size
=
2956 max_t(unsigned, limits
->physical_block_size
, cc
->sector_size
);
2957 limits
->io_min
= max_t(unsigned, limits
->io_min
, cc
->sector_size
);
2960 static struct target_type crypt_target
= {
2962 .version
= {1, 19, 0},
2963 .module
= THIS_MODULE
,
2967 .status
= crypt_status
,
2968 .postsuspend
= crypt_postsuspend
,
2969 .preresume
= crypt_preresume
,
2970 .resume
= crypt_resume
,
2971 .message
= crypt_message
,
2972 .iterate_devices
= crypt_iterate_devices
,
2973 .io_hints
= crypt_io_hints
,
2976 static int __init
dm_crypt_init(void)
2980 r
= dm_register_target(&crypt_target
);
2982 DMERR("register failed %d", r
);
2987 static void __exit
dm_crypt_exit(void)
2989 dm_unregister_target(&crypt_target
);
2992 module_init(dm_crypt_init
);
2993 module_exit(dm_crypt_exit
);
2995 MODULE_AUTHOR("Jana Saout <jana@saout.de>");
2996 MODULE_DESCRIPTION(DM_NAME
" target for transparent encryption / decryption");
2997 MODULE_LICENSE("GPL");