?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'trace-printf-v6.13' of git://git.kernel.org/pub/scm/linux/kernel/git/trace...
[drm/drm-misc.git] / drivers / md / dm-integrity.c
blobee9f7cecd78e0e796caa5c4bcfc186262ecb2068
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved.
4 * Copyright (C) 2016-2017 Milan Broz
5 * Copyright (C) 2016-2017 Mikulas Patocka
6 *
7 * This file is released under the GPL.
8 */
9
10 #include "dm-bio-record.h"
11
12 #include <linux/compiler.h>
13 #include <linux/module.h>
14 #include <linux/device-mapper.h>
15 #include <linux/dm-io.h>
16 #include <linux/vmalloc.h>
17 #include <linux/sort.h>
18 #include <linux/rbtree.h>
19 #include <linux/delay.h>
20 #include <linux/random.h>
21 #include <linux/reboot.h>
22 #include <crypto/hash.h>
23 #include <crypto/skcipher.h>
24 #include <linux/async_tx.h>
25 #include <linux/dm-bufio.h>
26
27 #include "dm-audit.h"
28
29 #define DM_MSG_PREFIX "integrity"
30
31 #define DEFAULT_INTERLEAVE_SECTORS 32768
32 #define DEFAULT_JOURNAL_SIZE_FACTOR 7
33 #define DEFAULT_SECTORS_PER_BITMAP_BIT 32768
34 #define DEFAULT_BUFFER_SECTORS 128
35 #define DEFAULT_JOURNAL_WATERMARK 50
36 #define DEFAULT_SYNC_MSEC 10000
37 #define DEFAULT_MAX_JOURNAL_SECTORS (IS_ENABLED(CONFIG_64BIT) ? 131072 : 8192)
38 #define MIN_LOG2_INTERLEAVE_SECTORS 3
39 #define MAX_LOG2_INTERLEAVE_SECTORS 31
40 #define METADATA_WORKQUEUE_MAX_ACTIVE 16
41 #define RECALC_SECTORS (IS_ENABLED(CONFIG_64BIT) ? 32768 : 2048)
42 #define RECALC_WRITE_SUPER 16
43 #define BITMAP_BLOCK_SIZE 4096 /* don't change it */
44 #define BITMAP_FLUSH_INTERVAL (10 * HZ)
45 #define DISCARD_FILLER 0xf6
46 #define SALT_SIZE 16
47 #define RECHECK_POOL_SIZE 256
48
49 /*
50 * Warning - DEBUG_PRINT prints security-sensitive data to the log,
51 * so it should not be enabled in the official kernel
52 */
53 //#define DEBUG_PRINT
54 //#define INTERNAL_VERIFY
55
56 /*
57 * On disk structures
58 */
59
60 #define SB_MAGIC "integrt"
61 #define SB_VERSION_1 1
62 #define SB_VERSION_2 2
63 #define SB_VERSION_3 3
64 #define SB_VERSION_4 4
65 #define SB_VERSION_5 5
66 #define SB_VERSION_6 6
67 #define SB_SECTORS 8
68 #define MAX_SECTORS_PER_BLOCK 8
69
70 struct superblock {
71 __u8 magic[8];
72 __u8 version;
73 __u8 log2_interleave_sectors;
74 __le16 integrity_tag_size;
75 __le32 journal_sections;
76 __le64 provided_data_sectors; /* userspace uses this value */
77 __le32 flags;
78 __u8 log2_sectors_per_block;
79 __u8 log2_blocks_per_bitmap_bit;
80 __u8 pad[2];
81 __le64 recalc_sector;
82 __u8 pad2[8];
83 __u8 salt[SALT_SIZE];
84 };
85
86 #define SB_FLAG_HAVE_JOURNAL_MAC 0x1
87 #define SB_FLAG_RECALCULATING 0x2
88 #define SB_FLAG_DIRTY_BITMAP 0x4
89 #define SB_FLAG_FIXED_PADDING 0x8
90 #define SB_FLAG_FIXED_HMAC 0x10
91 #define SB_FLAG_INLINE 0x20
92
93 #define JOURNAL_ENTRY_ROUNDUP 8
94
95 typedef __le64 commit_id_t;
96 #define JOURNAL_MAC_PER_SECTOR 8
97
98 struct journal_entry {
99 union {
100 struct {
101 __le32 sector_lo;
102 __le32 sector_hi;
103 } s;
104 __le64 sector;
105 } u;
106 commit_id_t last_bytes[];
107 /* __u8 tag[0]; */
108 };
109
110 #define journal_entry_tag(ic, je) ((__u8 *)&(je)->last_bytes[(ic)->sectors_per_block])
111
112 #if BITS_PER_LONG == 64
113 #define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0)
114 #else
115 #define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0)
116 #endif
117 #define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector)
118 #define journal_entry_is_unused(je) ((je)->u.s.sector_hi == cpu_to_le32(-1))
119 #define journal_entry_set_unused(je) ((je)->u.s.sector_hi = cpu_to_le32(-1))
120 #define journal_entry_is_inprogress(je) ((je)->u.s.sector_hi == cpu_to_le32(-2))
121 #define journal_entry_set_inprogress(je) ((je)->u.s.sector_hi = cpu_to_le32(-2))
122
123 #define JOURNAL_BLOCK_SECTORS 8
124 #define JOURNAL_SECTOR_DATA ((1 << SECTOR_SHIFT) - sizeof(commit_id_t))
125 #define JOURNAL_MAC_SIZE (JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS)
126
127 struct journal_sector {
128 struct_group(sectors,
129 __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR];
130 __u8 mac[JOURNAL_MAC_PER_SECTOR];
131 );
132 commit_id_t commit_id;
133 };
134
135 #define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK]))
136
137 #define METADATA_PADDING_SECTORS 8
138
139 #define N_COMMIT_IDS 4
140
141 static unsigned char prev_commit_seq(unsigned char seq)
142 {
143 return (seq + N_COMMIT_IDS - 1) % N_COMMIT_IDS;
144 }
145
146 static unsigned char next_commit_seq(unsigned char seq)
147 {
148 return (seq + 1) % N_COMMIT_IDS;
149 }
150
151 /*
152 * In-memory structures
153 */
154
155 struct journal_node {
156 struct rb_node node;
157 sector_t sector;
158 };
159
160 struct alg_spec {
161 char *alg_string;
162 char *key_string;
163 __u8 *key;
164 unsigned int key_size;
165 };
166
167 struct dm_integrity_c {
168 struct dm_dev *dev;
169 struct dm_dev *meta_dev;
170 unsigned int tag_size;
171 __s8 log2_tag_size;
172 unsigned int tuple_size;
173 sector_t start;
174 mempool_t journal_io_mempool;
175 struct dm_io_client *io;
176 struct dm_bufio_client *bufio;
177 struct workqueue_struct *metadata_wq;
178 struct superblock *sb;
179 unsigned int journal_pages;
180 unsigned int n_bitmap_blocks;
181
182 struct page_list *journal;
183 struct page_list *journal_io;
184 struct page_list *journal_xor;
185 struct page_list *recalc_bitmap;
186 struct page_list *may_write_bitmap;
187 struct bitmap_block_status *bbs;
188 unsigned int bitmap_flush_interval;
189 int synchronous_mode;
190 struct bio_list synchronous_bios;
191 struct delayed_work bitmap_flush_work;
192
193 struct crypto_skcipher *journal_crypt;
194 struct scatterlist **journal_scatterlist;
195 struct scatterlist **journal_io_scatterlist;
196 struct skcipher_request **sk_requests;
197
198 struct crypto_shash *journal_mac;
199
200 struct journal_node *journal_tree;
201 struct rb_root journal_tree_root;
202
203 sector_t provided_data_sectors;
204
205 unsigned short journal_entry_size;
206 unsigned char journal_entries_per_sector;
207 unsigned char journal_section_entries;
208 unsigned short journal_section_sectors;
209 unsigned int journal_sections;
210 unsigned int journal_entries;
211 sector_t data_device_sectors;
212 sector_t meta_device_sectors;
213 unsigned int initial_sectors;
214 unsigned int metadata_run;
215 __s8 log2_metadata_run;
216 __u8 log2_buffer_sectors;
217 __u8 sectors_per_block;
218 __u8 log2_blocks_per_bitmap_bit;
219
220 unsigned char mode;
221
222 int failed;
223
224 struct crypto_shash *internal_hash;
225
226 struct dm_target *ti;
227
228 /* these variables are locked with endio_wait.lock */
229 struct rb_root in_progress;
230 struct list_head wait_list;
231 wait_queue_head_t endio_wait;
232 struct workqueue_struct *wait_wq;
233 struct workqueue_struct *offload_wq;
234
235 unsigned char commit_seq;
236 commit_id_t commit_ids[N_COMMIT_IDS];
237
238 unsigned int committed_section;
239 unsigned int n_committed_sections;
240
241 unsigned int uncommitted_section;
242 unsigned int n_uncommitted_sections;
243
244 unsigned int free_section;
245 unsigned char free_section_entry;
246 unsigned int free_sectors;
247
248 unsigned int free_sectors_threshold;
249
250 struct workqueue_struct *commit_wq;
251 struct work_struct commit_work;
252
253 struct workqueue_struct *writer_wq;
254 struct work_struct writer_work;
255
256 struct workqueue_struct *recalc_wq;
257 struct work_struct recalc_work;
258
259 struct bio_list flush_bio_list;
260
261 unsigned long autocommit_jiffies;
262 struct timer_list autocommit_timer;
263 unsigned int autocommit_msec;
264
265 wait_queue_head_t copy_to_journal_wait;
266
267 struct completion crypto_backoff;
268
269 bool wrote_to_journal;
270 bool journal_uptodate;
271 bool just_formatted;
272 bool recalculate_flag;
273 bool reset_recalculate_flag;
274 bool discard;
275 bool fix_padding;
276 bool fix_hmac;
277 bool legacy_recalculate;
278
279 struct alg_spec internal_hash_alg;
280 struct alg_spec journal_crypt_alg;
281 struct alg_spec journal_mac_alg;
282
283 atomic64_t number_of_mismatches;
284
285 mempool_t recheck_pool;
286 struct bio_set recheck_bios;
287 struct bio_set recalc_bios;
288
289 struct notifier_block reboot_notifier;
290 };
291
292 struct dm_integrity_range {
293 sector_t logical_sector;
294 sector_t n_sectors;
295 bool waiting;
296 union {
297 struct rb_node node;
298 struct {
299 struct task_struct *task;
300 struct list_head wait_entry;
301 };
302 };
303 };
304
305 struct dm_integrity_io {
306 struct work_struct work;
307
308 struct dm_integrity_c *ic;
309 enum req_op op;
310 bool fua;
311
312 struct dm_integrity_range range;
313
314 sector_t metadata_block;
315 unsigned int metadata_offset;
316
317 atomic_t in_flight;
318 blk_status_t bi_status;
319
320 struct completion *completion;
321
322 struct dm_bio_details bio_details;
323
324 char *integrity_payload;
325 unsigned payload_len;
326 bool integrity_payload_from_mempool;
327 bool integrity_range_locked;
328 };
329
330 struct journal_completion {
331 struct dm_integrity_c *ic;
332 atomic_t in_flight;
333 struct completion comp;
334 };
335
336 struct journal_io {
337 struct dm_integrity_range range;
338 struct journal_completion *comp;
339 };
340
341 struct bitmap_block_status {
342 struct work_struct work;
343 struct dm_integrity_c *ic;
344 unsigned int idx;
345 unsigned long *bitmap;
346 struct bio_list bio_queue;
347 spinlock_t bio_queue_lock;
348
349 };
350
351 static struct kmem_cache *journal_io_cache;
352
353 #define JOURNAL_IO_MEMPOOL 32
354
355 #ifdef DEBUG_PRINT
356 #define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__)
357 #define DEBUG_bytes(bytes, len, msg, ...) printk(KERN_DEBUG msg "%s%*ph\n", ##__VA_ARGS__, \
358 len ? ": " : "", len, bytes)
359 #else
360 #define DEBUG_print(x, ...) do { } while (0)
361 #define DEBUG_bytes(bytes, len, msg, ...) do { } while (0)
362 #endif
363
364 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
365 static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map);
366 static void integrity_bio_wait(struct work_struct *w);
367 static void dm_integrity_dtr(struct dm_target *ti);
368
369 static void dm_integrity_io_error(struct dm_integrity_c *ic, const char *msg, int err)
370 {
371 if (err == -EILSEQ)
372 atomic64_inc(&ic->number_of_mismatches);
373 if (!cmpxchg(&ic->failed, 0, err))
374 DMERR("Error on %s: %d", msg, err);
375 }
376
377 static int dm_integrity_failed(struct dm_integrity_c *ic)
378 {
379 return READ_ONCE(ic->failed);
380 }
381
382 static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic)
383 {
384 if (ic->legacy_recalculate)
385 return false;
386 if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ?
387 ic->internal_hash_alg.key || ic->journal_mac_alg.key :
388 ic->internal_hash_alg.key && !ic->journal_mac_alg.key)
389 return true;
390 return false;
391 }
392
393 static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned int i,
394 unsigned int j, unsigned char seq)
395 {
396 /*
397 * Xor the number with section and sector, so that if a piece of
398 * journal is written at wrong place, it is detected.
399 */
400 return ic->commit_ids[seq] ^ cpu_to_le64(((__u64)i << 32) ^ j);
401 }
402
403 static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector,
404 sector_t *area, sector_t *offset)
405 {
406 if (!ic->meta_dev) {
407 __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors;
408 *area = data_sector >> log2_interleave_sectors;
409 *offset = (unsigned int)data_sector & ((1U << log2_interleave_sectors) - 1);
410 } else {
411 *area = 0;
412 *offset = data_sector;
413 }
414 }
415
416 #define sector_to_block(ic, n) \
417 do { \
418 BUG_ON((n) & (unsigned int)((ic)->sectors_per_block - 1)); \
419 (n) >>= (ic)->sb->log2_sectors_per_block; \
420 } while (0)
421
422 static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area,
423 sector_t offset, unsigned int *metadata_offset)
424 {
425 __u64 ms;
426 unsigned int mo;
427
428 ms = area << ic->sb->log2_interleave_sectors;
429 if (likely(ic->log2_metadata_run >= 0))
430 ms += area << ic->log2_metadata_run;
431 else
432 ms += area * ic->metadata_run;
433 ms >>= ic->log2_buffer_sectors;
434
435 sector_to_block(ic, offset);
436
437 if (likely(ic->log2_tag_size >= 0)) {
438 ms += offset >> (SECTOR_SHIFT + ic->log2_buffer_sectors - ic->log2_tag_size);
439 mo = (offset << ic->log2_tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
440 } else {
441 ms += (__u64)offset * ic->tag_size >> (SECTOR_SHIFT + ic->log2_buffer_sectors);
442 mo = (offset * ic->tag_size) & ((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - 1);
443 }
444 *metadata_offset = mo;
445 return ms;
446 }
447
448 static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector_t offset)
449 {
450 sector_t result;
451
452 if (ic->meta_dev)
453 return offset;
454
455 result = area << ic->sb->log2_interleave_sectors;
456 if (likely(ic->log2_metadata_run >= 0))
457 result += (area + 1) << ic->log2_metadata_run;
458 else
459 result += (area + 1) * ic->metadata_run;
460
461 result += (sector_t)ic->initial_sectors + offset;
462 result += ic->start;
463
464 return result;
465 }
466
467 static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr)
468 {
469 if (unlikely(*sec_ptr >= ic->journal_sections))
470 *sec_ptr -= ic->journal_sections;
471 }
472
473 static void sb_set_version(struct dm_integrity_c *ic)
474 {
475 if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE))
476 ic->sb->version = SB_VERSION_6;
477 else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC))
478 ic->sb->version = SB_VERSION_5;
479 else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING))
480 ic->sb->version = SB_VERSION_4;
481 else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP))
482 ic->sb->version = SB_VERSION_3;
483 else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
484 ic->sb->version = SB_VERSION_2;
485 else
486 ic->sb->version = SB_VERSION_1;
487 }
488
489 static int sb_mac(struct dm_integrity_c *ic, bool wr)
490 {
491 SHASH_DESC_ON_STACK(desc, ic->journal_mac);
492 int r;
493 unsigned int mac_size = crypto_shash_digestsize(ic->journal_mac);
494 __u8 *sb = (__u8 *)ic->sb;
495 __u8 *mac = sb + (1 << SECTOR_SHIFT) - mac_size;
496
497 if (sizeof(struct superblock) + mac_size > 1 << SECTOR_SHIFT ||
498 mac_size > HASH_MAX_DIGESTSIZE) {
499 dm_integrity_io_error(ic, "digest is too long", -EINVAL);
500 return -EINVAL;
501 }
502
503 desc->tfm = ic->journal_mac;
504
505 if (likely(wr)) {
506 r = crypto_shash_digest(desc, sb, mac - sb, mac);
507 if (unlikely(r < 0)) {
508 dm_integrity_io_error(ic, "crypto_shash_digest", r);
509 return r;
510 }
511 } else {
512 __u8 actual_mac[HASH_MAX_DIGESTSIZE];
513
514 r = crypto_shash_digest(desc, sb, mac - sb, actual_mac);
515 if (unlikely(r < 0)) {
516 dm_integrity_io_error(ic, "crypto_shash_digest", r);
517 return r;
518 }
519 if (memcmp(mac, actual_mac, mac_size)) {
520 dm_integrity_io_error(ic, "superblock mac", -EILSEQ);
521 dm_audit_log_target(DM_MSG_PREFIX, "mac-superblock", ic->ti, 0);
522 return -EILSEQ;
523 }
524 }
525
526 return 0;
527 }
528
529 static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf)
530 {
531 struct dm_io_request io_req;
532 struct dm_io_region io_loc;
533 const enum req_op op = opf & REQ_OP_MASK;
534 int r;
535
536 io_req.bi_opf = opf;
537 io_req.mem.type = DM_IO_KMEM;
538 io_req.mem.ptr.addr = ic->sb;
539 io_req.notify.fn = NULL;
540 io_req.client = ic->io;
541 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
542 io_loc.sector = ic->start;
543 io_loc.count = SB_SECTORS;
544
545 if (op == REQ_OP_WRITE) {
546 sb_set_version(ic);
547 if (ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
548 r = sb_mac(ic, true);
549 if (unlikely(r))
550 return r;
551 }
552 }
553
554 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
555 if (unlikely(r))
556 return r;
557
558 if (op == REQ_OP_READ) {
559 if (ic->mode != 'R' && ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
560 r = sb_mac(ic, false);
561 if (unlikely(r))
562 return r;
563 }
564 }
565
566 return 0;
567 }
568
569 #define BITMAP_OP_TEST_ALL_SET 0
570 #define BITMAP_OP_TEST_ALL_CLEAR 1
571 #define BITMAP_OP_SET 2
572 #define BITMAP_OP_CLEAR 3
573
574 static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap,
575 sector_t sector, sector_t n_sectors, int mode)
576 {
577 unsigned long bit, end_bit, this_end_bit, page, end_page;
578 unsigned long *data;
579
580 if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) {
581 DMCRIT("invalid bitmap access (%llx,%llx,%d,%d,%d)",
582 sector,
583 n_sectors,
584 ic->sb->log2_sectors_per_block,
585 ic->log2_blocks_per_bitmap_bit,
586 mode);
587 BUG();
588 }
589
590 if (unlikely(!n_sectors))
591 return true;
592
593 bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
594 end_bit = (sector + n_sectors - 1) >>
595 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
596
597 page = bit / (PAGE_SIZE * 8);
598 bit %= PAGE_SIZE * 8;
599
600 end_page = end_bit / (PAGE_SIZE * 8);
601 end_bit %= PAGE_SIZE * 8;
602
603 repeat:
604 if (page < end_page)
605 this_end_bit = PAGE_SIZE * 8 - 1;
606 else
607 this_end_bit = end_bit;
608
609 data = lowmem_page_address(bitmap[page].page);
610
611 if (mode == BITMAP_OP_TEST_ALL_SET) {
612 while (bit <= this_end_bit) {
613 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
614 do {
615 if (data[bit / BITS_PER_LONG] != -1)
616 return false;
617 bit += BITS_PER_LONG;
618 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
619 continue;
620 }
621 if (!test_bit(bit, data))
622 return false;
623 bit++;
624 }
625 } else if (mode == BITMAP_OP_TEST_ALL_CLEAR) {
626 while (bit <= this_end_bit) {
627 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
628 do {
629 if (data[bit / BITS_PER_LONG] != 0)
630 return false;
631 bit += BITS_PER_LONG;
632 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
633 continue;
634 }
635 if (test_bit(bit, data))
636 return false;
637 bit++;
638 }
639 } else if (mode == BITMAP_OP_SET) {
640 while (bit <= this_end_bit) {
641 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
642 do {
643 data[bit / BITS_PER_LONG] = -1;
644 bit += BITS_PER_LONG;
645 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
646 continue;
647 }
648 __set_bit(bit, data);
649 bit++;
650 }
651 } else if (mode == BITMAP_OP_CLEAR) {
652 if (!bit && this_end_bit == PAGE_SIZE * 8 - 1)
653 clear_page(data);
654 else {
655 while (bit <= this_end_bit) {
656 if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) {
657 do {
658 data[bit / BITS_PER_LONG] = 0;
659 bit += BITS_PER_LONG;
660 } while (this_end_bit >= bit + BITS_PER_LONG - 1);
661 continue;
662 }
663 __clear_bit(bit, data);
664 bit++;
665 }
666 }
667 } else {
668 BUG();
669 }
670
671 if (unlikely(page < end_page)) {
672 bit = 0;
673 page++;
674 goto repeat;
675 }
676
677 return true;
678 }
679
680 static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *src)
681 {
682 unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE);
683 unsigned int i;
684
685 for (i = 0; i < n_bitmap_pages; i++) {
686 unsigned long *dst_data = lowmem_page_address(dst[i].page);
687 unsigned long *src_data = lowmem_page_address(src[i].page);
688
689 copy_page(dst_data, src_data);
690 }
691 }
692
693 static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t sector)
694 {
695 unsigned int bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
696 unsigned int bitmap_block = bit / (BITMAP_BLOCK_SIZE * 8);
697
698 BUG_ON(bitmap_block >= ic->n_bitmap_blocks);
699 return &ic->bbs[bitmap_block];
700 }
701
702 static void access_journal_check(struct dm_integrity_c *ic, unsigned int section, unsigned int offset,
703 bool e, const char *function)
704 {
705 #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY)
706 unsigned int limit = e ? ic->journal_section_entries : ic->journal_section_sectors;
707
708 if (unlikely(section >= ic->journal_sections) ||
709 unlikely(offset >= limit)) {
710 DMCRIT("%s: invalid access at (%u,%u), limit (%u,%u)",
711 function, section, offset, ic->journal_sections, limit);
712 BUG();
713 }
714 #endif
715 }
716
717 static void page_list_location(struct dm_integrity_c *ic, unsigned int section, unsigned int offset,
718 unsigned int *pl_index, unsigned int *pl_offset)
719 {
720 unsigned int sector;
721
722 access_journal_check(ic, section, offset, false, "page_list_location");
723
724 sector = section * ic->journal_section_sectors + offset;
725
726 *pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
727 *pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
728 }
729
730 static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl,
731 unsigned int section, unsigned int offset, unsigned int *n_sectors)
732 {
733 unsigned int pl_index, pl_offset;
734 char *va;
735
736 page_list_location(ic, section, offset, &pl_index, &pl_offset);
737
738 if (n_sectors)
739 *n_sectors = (PAGE_SIZE - pl_offset) >> SECTOR_SHIFT;
740
741 va = lowmem_page_address(pl[pl_index].page);
742
743 return (struct journal_sector *)(va + pl_offset);
744 }
745
746 static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset)
747 {
748 return access_page_list(ic, ic->journal, section, offset, NULL);
749 }
750
751 static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned int section, unsigned int n)
752 {
753 unsigned int rel_sector, offset;
754 struct journal_sector *js;
755
756 access_journal_check(ic, section, n, true, "access_journal_entry");
757
758 rel_sector = n % JOURNAL_BLOCK_SECTORS;
759 offset = n / JOURNAL_BLOCK_SECTORS;
760
761 js = access_journal(ic, section, rel_sector);
762 return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size);
763 }
764
765 static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned int section, unsigned int n)
766 {
767 n <<= ic->sb->log2_sectors_per_block;
768
769 n += JOURNAL_BLOCK_SECTORS;
770
771 access_journal_check(ic, section, n, false, "access_journal_data");
772
773 return access_journal(ic, section, n);
774 }
775
776 static void section_mac(struct dm_integrity_c *ic, unsigned int section, __u8 result[JOURNAL_MAC_SIZE])
777 {
778 SHASH_DESC_ON_STACK(desc, ic->journal_mac);
779 int r;
780 unsigned int j, size;
781
782 desc->tfm = ic->journal_mac;
783
784 r = crypto_shash_init(desc);
785 if (unlikely(r < 0)) {
786 dm_integrity_io_error(ic, "crypto_shash_init", r);
787 goto err;
788 }
789
790 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
791 __le64 section_le;
792
793 r = crypto_shash_update(desc, (__u8 *)&ic->sb->salt, SALT_SIZE);
794 if (unlikely(r < 0)) {
795 dm_integrity_io_error(ic, "crypto_shash_update", r);
796 goto err;
797 }
798
799 section_le = cpu_to_le64(section);
800 r = crypto_shash_update(desc, (__u8 *)&section_le, sizeof(section_le));
801 if (unlikely(r < 0)) {
802 dm_integrity_io_error(ic, "crypto_shash_update", r);
803 goto err;
804 }
805 }
806
807 for (j = 0; j < ic->journal_section_entries; j++) {
808 struct journal_entry *je = access_journal_entry(ic, section, j);
809
810 r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof(je->u.sector));
811 if (unlikely(r < 0)) {
812 dm_integrity_io_error(ic, "crypto_shash_update", r);
813 goto err;
814 }
815 }
816
817 size = crypto_shash_digestsize(ic->journal_mac);
818
819 if (likely(size <= JOURNAL_MAC_SIZE)) {
820 r = crypto_shash_final(desc, result);
821 if (unlikely(r < 0)) {
822 dm_integrity_io_error(ic, "crypto_shash_final", r);
823 goto err;
824 }
825 memset(result + size, 0, JOURNAL_MAC_SIZE - size);
826 } else {
827 __u8 digest[HASH_MAX_DIGESTSIZE];
828
829 if (WARN_ON(size > sizeof(digest))) {
830 dm_integrity_io_error(ic, "digest_size", -EINVAL);
831 goto err;
832 }
833 r = crypto_shash_final(desc, digest);
834 if (unlikely(r < 0)) {
835 dm_integrity_io_error(ic, "crypto_shash_final", r);
836 goto err;
837 }
838 memcpy(result, digest, JOURNAL_MAC_SIZE);
839 }
840
841 return;
842 err:
843 memset(result, 0, JOURNAL_MAC_SIZE);
844 }
845
846 static void rw_section_mac(struct dm_integrity_c *ic, unsigned int section, bool wr)
847 {
848 __u8 result[JOURNAL_MAC_SIZE];
849 unsigned int j;
850
851 if (!ic->journal_mac)
852 return;
853
854 section_mac(ic, section, result);
855
856 for (j = 0; j < JOURNAL_BLOCK_SECTORS; j++) {
857 struct journal_sector *js = access_journal(ic, section, j);
858
859 if (likely(wr))
860 memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR);
861 else {
862 if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) {
863 dm_integrity_io_error(ic, "journal mac", -EILSEQ);
864 dm_audit_log_target(DM_MSG_PREFIX, "mac-journal", ic->ti, 0);
865 }
866 }
867 }
868 }
869
870 static void complete_journal_op(void *context)
871 {
872 struct journal_completion *comp = context;
873
874 BUG_ON(!atomic_read(&comp->in_flight));
875 if (likely(atomic_dec_and_test(&comp->in_flight)))
876 complete(&comp->comp);
877 }
878
879 static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section,
880 unsigned int n_sections, struct journal_completion *comp)
881 {
882 struct async_submit_ctl submit;
883 size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT;
884 unsigned int pl_index, pl_offset, section_index;
885 struct page_list *source_pl, *target_pl;
886
887 if (likely(encrypt)) {
888 source_pl = ic->journal;
889 target_pl = ic->journal_io;
890 } else {
891 source_pl = ic->journal_io;
892 target_pl = ic->journal;
893 }
894
895 page_list_location(ic, section, 0, &pl_index, &pl_offset);
896
897 atomic_add(roundup(pl_offset + n_bytes, PAGE_SIZE) >> PAGE_SHIFT, &comp->in_flight);
898
899 init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, complete_journal_op, comp, NULL);
900
901 section_index = pl_index;
902
903 do {
904 size_t this_step;
905 struct page *src_pages[2];
906 struct page *dst_page;
907
908 while (unlikely(pl_index == section_index)) {
909 unsigned int dummy;
910
911 if (likely(encrypt))
912 rw_section_mac(ic, section, true);
913 section++;
914 n_sections--;
915 if (!n_sections)
916 break;
917 page_list_location(ic, section, 0, &section_index, &dummy);
918 }
919
920 this_step = min(n_bytes, (size_t)PAGE_SIZE - pl_offset);
921 dst_page = target_pl[pl_index].page;
922 src_pages[0] = source_pl[pl_index].page;
923 src_pages[1] = ic->journal_xor[pl_index].page;
924
925 async_xor(dst_page, src_pages, pl_offset, 2, this_step, &submit);
926
927 pl_index++;
928 pl_offset = 0;
929 n_bytes -= this_step;
930 } while (n_bytes);
931
932 BUG_ON(n_sections);
933
934 async_tx_issue_pending_all();
935 }
936
937 static void complete_journal_encrypt(void *data, int err)
938 {
939 struct journal_completion *comp = data;
940
941 if (unlikely(err)) {
942 if (likely(err == -EINPROGRESS)) {
943 complete(&comp->ic->crypto_backoff);
944 return;
945 }
946 dm_integrity_io_error(comp->ic, "asynchronous encrypt", err);
947 }
948 complete_journal_op(comp);
949 }
950
951 static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp)
952 {
953 int r;
954
955 skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
956 complete_journal_encrypt, comp);
957 if (likely(encrypt))
958 r = crypto_skcipher_encrypt(req);
959 else
960 r = crypto_skcipher_decrypt(req);
961 if (likely(!r))
962 return false;
963 if (likely(r == -EINPROGRESS))
964 return true;
965 if (likely(r == -EBUSY)) {
966 wait_for_completion(&comp->ic->crypto_backoff);
967 reinit_completion(&comp->ic->crypto_backoff);
968 return true;
969 }
970 dm_integrity_io_error(comp->ic, "encrypt", r);
971 return false;
972 }
973
974 static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section,
975 unsigned int n_sections, struct journal_completion *comp)
976 {
977 struct scatterlist **source_sg;
978 struct scatterlist **target_sg;
979
980 atomic_add(2, &comp->in_flight);
981
982 if (likely(encrypt)) {
983 source_sg = ic->journal_scatterlist;
984 target_sg = ic->journal_io_scatterlist;
985 } else {
986 source_sg = ic->journal_io_scatterlist;
987 target_sg = ic->journal_scatterlist;
988 }
989
990 do {
991 struct skcipher_request *req;
992 unsigned int ivsize;
993 char *iv;
994
995 if (likely(encrypt))
996 rw_section_mac(ic, section, true);
997
998 req = ic->sk_requests[section];
999 ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
1000 iv = req->iv;
1001
1002 memcpy(iv, iv + ivsize, ivsize);
1003
1004 req->src = source_sg[section];
1005 req->dst = target_sg[section];
1006
1007 if (unlikely(do_crypt(encrypt, req, comp)))
1008 atomic_inc(&comp->in_flight);
1009
1010 section++;
1011 n_sections--;
1012 } while (n_sections);
1013
1014 atomic_dec(&comp->in_flight);
1015 complete_journal_op(comp);
1016 }
1017
1018 static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section,
1019 unsigned int n_sections, struct journal_completion *comp)
1020 {
1021 if (ic->journal_xor)
1022 return xor_journal(ic, encrypt, section, n_sections, comp);
1023 else
1024 return crypt_journal(ic, encrypt, section, n_sections, comp);
1025 }
1026
1027 static void complete_journal_io(unsigned long error, void *context)
1028 {
1029 struct journal_completion *comp = context;
1030
1031 if (unlikely(error != 0))
1032 dm_integrity_io_error(comp->ic, "writing journal", -EIO);
1033 complete_journal_op(comp);
1034 }
1035
1036 static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf,
1037 unsigned int sector, unsigned int n_sectors,
1038 struct journal_completion *comp)
1039 {
1040 struct dm_io_request io_req;
1041 struct dm_io_region io_loc;
1042 unsigned int pl_index, pl_offset;
1043 int r;
1044
1045 if (unlikely(dm_integrity_failed(ic))) {
1046 if (comp)
1047 complete_journal_io(-1UL, comp);
1048 return;
1049 }
1050
1051 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
1052 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
1053
1054 io_req.bi_opf = opf;
1055 io_req.mem.type = DM_IO_PAGE_LIST;
1056 if (ic->journal_io)
1057 io_req.mem.ptr.pl = &ic->journal_io[pl_index];
1058 else
1059 io_req.mem.ptr.pl = &ic->journal[pl_index];
1060 io_req.mem.offset = pl_offset;
1061 if (likely(comp != NULL)) {
1062 io_req.notify.fn = complete_journal_io;
1063 io_req.notify.context = comp;
1064 } else {
1065 io_req.notify.fn = NULL;
1066 }
1067 io_req.client = ic->io;
1068 io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev;
1069 io_loc.sector = ic->start + SB_SECTORS + sector;
1070 io_loc.count = n_sectors;
1071
1072 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1073 if (unlikely(r)) {
1074 dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ?
1075 "reading journal" : "writing journal", r);
1076 if (comp) {
1077 WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1078 complete_journal_io(-1UL, comp);
1079 }
1080 }
1081 }
1082
1083 static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf,
1084 unsigned int section, unsigned int n_sections,
1085 struct journal_completion *comp)
1086 {
1087 unsigned int sector, n_sectors;
1088
1089 sector = section * ic->journal_section_sectors;
1090 n_sectors = n_sections * ic->journal_section_sectors;
1091
1092 rw_journal_sectors(ic, opf, sector, n_sectors, comp);
1093 }
1094
1095 static void write_journal(struct dm_integrity_c *ic, unsigned int commit_start, unsigned int commit_sections)
1096 {
1097 struct journal_completion io_comp;
1098 struct journal_completion crypt_comp_1;
1099 struct journal_completion crypt_comp_2;
1100 unsigned int i;
1101
1102 io_comp.ic = ic;
1103 init_completion(&io_comp.comp);
1104
1105 if (commit_start + commit_sections <= ic->journal_sections) {
1106 io_comp.in_flight = (atomic_t)ATOMIC_INIT(1);
1107 if (ic->journal_io) {
1108 crypt_comp_1.ic = ic;
1109 init_completion(&crypt_comp_1.comp);
1110 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1111 encrypt_journal(ic, true, commit_start, commit_sections, &crypt_comp_1);
1112 wait_for_completion_io(&crypt_comp_1.comp);
1113 } else {
1114 for (i = 0; i < commit_sections; i++)
1115 rw_section_mac(ic, commit_start + i, true);
1116 }
1117 rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start,
1118 commit_sections, &io_comp);
1119 } else {
1120 unsigned int to_end;
1121
1122 io_comp.in_flight = (atomic_t)ATOMIC_INIT(2);
1123 to_end = ic->journal_sections - commit_start;
1124 if (ic->journal_io) {
1125 crypt_comp_1.ic = ic;
1126 init_completion(&crypt_comp_1.comp);
1127 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1128 encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1);
1129 if (try_wait_for_completion(&crypt_comp_1.comp)) {
1130 rw_journal(ic, REQ_OP_WRITE | REQ_FUA,
1131 commit_start, to_end, &io_comp);
1132 reinit_completion(&crypt_comp_1.comp);
1133 crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0);
1134 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1);
1135 wait_for_completion_io(&crypt_comp_1.comp);
1136 } else {
1137 crypt_comp_2.ic = ic;
1138 init_completion(&crypt_comp_2.comp);
1139 crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0);
1140 encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2);
1141 wait_for_completion_io(&crypt_comp_1.comp);
1142 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
1143 wait_for_completion_io(&crypt_comp_2.comp);
1144 }
1145 } else {
1146 for (i = 0; i < to_end; i++)
1147 rw_section_mac(ic, commit_start + i, true);
1148 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp);
1149 for (i = 0; i < commit_sections - to_end; i++)
1150 rw_section_mac(ic, i, true);
1151 }
1152 rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp);
1153 }
1154
1155 wait_for_completion_io(&io_comp.comp);
1156 }
1157
1158 static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset,
1159 unsigned int n_sectors, sector_t target, io_notify_fn fn, void *data)
1160 {
1161 struct dm_io_request io_req;
1162 struct dm_io_region io_loc;
1163 int r;
1164 unsigned int sector, pl_index, pl_offset;
1165
1166 BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1));
1167
1168 if (unlikely(dm_integrity_failed(ic))) {
1169 fn(-1UL, data);
1170 return;
1171 }
1172
1173 sector = section * ic->journal_section_sectors + JOURNAL_BLOCK_SECTORS + offset;
1174
1175 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
1176 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
1177
1178 io_req.bi_opf = REQ_OP_WRITE;
1179 io_req.mem.type = DM_IO_PAGE_LIST;
1180 io_req.mem.ptr.pl = &ic->journal[pl_index];
1181 io_req.mem.offset = pl_offset;
1182 io_req.notify.fn = fn;
1183 io_req.notify.context = data;
1184 io_req.client = ic->io;
1185 io_loc.bdev = ic->dev->bdev;
1186 io_loc.sector = target;
1187 io_loc.count = n_sectors;
1188
1189 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1190 if (unlikely(r)) {
1191 WARN_ONCE(1, "asynchronous dm_io failed: %d", r);
1192 fn(-1UL, data);
1193 }
1194 }
1195
1196 static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2)
1197 {
1198 return range1->logical_sector < range2->logical_sector + range2->n_sectors &&
1199 range1->logical_sector + range1->n_sectors > range2->logical_sector;
1200 }
1201
1202 static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting)
1203 {
1204 struct rb_node **n = &ic->in_progress.rb_node;
1205 struct rb_node *parent;
1206
1207 BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned int)(ic->sectors_per_block - 1));
1208
1209 if (likely(check_waiting)) {
1210 struct dm_integrity_range *range;
1211
1212 list_for_each_entry(range, &ic->wait_list, wait_entry) {
1213 if (unlikely(ranges_overlap(range, new_range)))
1214 return false;
1215 }
1216 }
1217
1218 parent = NULL;
1219
1220 while (*n) {
1221 struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node);
1222
1223 parent = *n;
1224 if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector)
1225 n = &range->node.rb_left;
1226 else if (new_range->logical_sector >= range->logical_sector + range->n_sectors)
1227 n = &range->node.rb_right;
1228 else
1229 return false;
1230 }
1231
1232 rb_link_node(&new_range->node, parent, n);
1233 rb_insert_color(&new_range->node, &ic->in_progress);
1234
1235 return true;
1236 }
1237
1238 static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range)
1239 {
1240 rb_erase(&range->node, &ic->in_progress);
1241 while (unlikely(!list_empty(&ic->wait_list))) {
1242 struct dm_integrity_range *last_range =
1243 list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry);
1244 struct task_struct *last_range_task;
1245
1246 last_range_task = last_range->task;
1247 list_del(&last_range->wait_entry);
1248 if (!add_new_range(ic, last_range, false)) {
1249 last_range->task = last_range_task;
1250 list_add(&last_range->wait_entry, &ic->wait_list);
1251 break;
1252 }
1253 last_range->waiting = false;
1254 wake_up_process(last_range_task);
1255 }
1256 }
1257
1258 static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range)
1259 {
1260 unsigned long flags;
1261
1262 spin_lock_irqsave(&ic->endio_wait.lock, flags);
1263 remove_range_unlocked(ic, range);
1264 spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1265 }
1266
1267 static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1268 {
1269 new_range->waiting = true;
1270 list_add_tail(&new_range->wait_entry, &ic->wait_list);
1271 new_range->task = current;
1272 do {
1273 __set_current_state(TASK_UNINTERRUPTIBLE);
1274 spin_unlock_irq(&ic->endio_wait.lock);
1275 io_schedule();
1276 spin_lock_irq(&ic->endio_wait.lock);
1277 } while (unlikely(new_range->waiting));
1278 }
1279
1280 static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range)
1281 {
1282 if (unlikely(!add_new_range(ic, new_range, true)))
1283 wait_and_add_new_range(ic, new_range);
1284 }
1285
1286 static void init_journal_node(struct journal_node *node)
1287 {
1288 RB_CLEAR_NODE(&node->node);
1289 node->sector = (sector_t)-1;
1290 }
1291
1292 static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *node, sector_t sector)
1293 {
1294 struct rb_node **link;
1295 struct rb_node *parent;
1296
1297 node->sector = sector;
1298 BUG_ON(!RB_EMPTY_NODE(&node->node));
1299
1300 link = &ic->journal_tree_root.rb_node;
1301 parent = NULL;
1302
1303 while (*link) {
1304 struct journal_node *j;
1305
1306 parent = *link;
1307 j = container_of(parent, struct journal_node, node);
1308 if (sector < j->sector)
1309 link = &j->node.rb_left;
1310 else
1311 link = &j->node.rb_right;
1312 }
1313
1314 rb_link_node(&node->node, parent, link);
1315 rb_insert_color(&node->node, &ic->journal_tree_root);
1316 }
1317
1318 static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node *node)
1319 {
1320 BUG_ON(RB_EMPTY_NODE(&node->node));
1321 rb_erase(&node->node, &ic->journal_tree_root);
1322 init_journal_node(node);
1323 }
1324
1325 #define NOT_FOUND (-1U)
1326
1327 static unsigned int find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector)
1328 {
1329 struct rb_node *n = ic->journal_tree_root.rb_node;
1330 unsigned int found = NOT_FOUND;
1331
1332 *next_sector = (sector_t)-1;
1333 while (n) {
1334 struct journal_node *j = container_of(n, struct journal_node, node);
1335
1336 if (sector == j->sector)
1337 found = j - ic->journal_tree;
1338
1339 if (sector < j->sector) {
1340 *next_sector = j->sector;
1341 n = j->node.rb_left;
1342 } else
1343 n = j->node.rb_right;
1344 }
1345
1346 return found;
1347 }
1348
1349 static bool test_journal_node(struct dm_integrity_c *ic, unsigned int pos, sector_t sector)
1350 {
1351 struct journal_node *node, *next_node;
1352 struct rb_node *next;
1353
1354 if (unlikely(pos >= ic->journal_entries))
1355 return false;
1356 node = &ic->journal_tree[pos];
1357 if (unlikely(RB_EMPTY_NODE(&node->node)))
1358 return false;
1359 if (unlikely(node->sector != sector))
1360 return false;
1361
1362 next = rb_next(&node->node);
1363 if (unlikely(!next))
1364 return true;
1365
1366 next_node = container_of(next, struct journal_node, node);
1367 return next_node->sector != sector;
1368 }
1369
1370 static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_node *node)
1371 {
1372 struct rb_node *next;
1373 struct journal_node *next_node;
1374 unsigned int next_section;
1375
1376 BUG_ON(RB_EMPTY_NODE(&node->node));
1377
1378 next = rb_next(&node->node);
1379 if (unlikely(!next))
1380 return false;
1381
1382 next_node = container_of(next, struct journal_node, node);
1383
1384 if (next_node->sector != node->sector)
1385 return false;
1386
1387 next_section = (unsigned int)(next_node - ic->journal_tree) / ic->journal_section_entries;
1388 if (next_section >= ic->committed_section &&
1389 next_section < ic->committed_section + ic->n_committed_sections)
1390 return true;
1391 if (next_section + ic->journal_sections < ic->committed_section + ic->n_committed_sections)
1392 return true;
1393
1394 return false;
1395 }
1396
1397 #define TAG_READ 0
1398 #define TAG_WRITE 1
1399 #define TAG_CMP 2
1400
1401 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block,
1402 unsigned int *metadata_offset, unsigned int total_size, int op)
1403 {
1404 #define MAY_BE_FILLER 1
1405 #define MAY_BE_HASH 2
1406 unsigned int hash_offset = 0;
1407 unsigned int may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0);
1408
1409 do {
1410 unsigned char *data, *dp;
1411 struct dm_buffer *b;
1412 unsigned int to_copy;
1413 int r;
1414
1415 r = dm_integrity_failed(ic);
1416 if (unlikely(r))
1417 return r;
1418
1419 data = dm_bufio_read(ic->bufio, *metadata_block, &b);
1420 if (IS_ERR(data))
1421 return PTR_ERR(data);
1422
1423 to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size);
1424 dp = data + *metadata_offset;
1425 if (op == TAG_READ) {
1426 memcpy(tag, dp, to_copy);
1427 } else if (op == TAG_WRITE) {
1428 if (memcmp(dp, tag, to_copy)) {
1429 memcpy(dp, tag, to_copy);
1430 dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy);
1431 }
1432 } else {
1433 /* e.g.: op == TAG_CMP */
1434
1435 if (likely(is_power_of_2(ic->tag_size))) {
1436 if (unlikely(memcmp(dp, tag, to_copy)))
1437 if (unlikely(!ic->discard) ||
1438 unlikely(memchr_inv(dp, DISCARD_FILLER, to_copy) != NULL)) {
1439 goto thorough_test;
1440 }
1441 } else {
1442 unsigned int i, ts;
1443 thorough_test:
1444 ts = total_size;
1445
1446 for (i = 0; i < to_copy; i++, ts--) {
1447 if (unlikely(dp[i] != tag[i]))
1448 may_be &= ~MAY_BE_HASH;
1449 if (likely(dp[i] != DISCARD_FILLER))
1450 may_be &= ~MAY_BE_FILLER;
1451 hash_offset++;
1452 if (unlikely(hash_offset == ic->tag_size)) {
1453 if (unlikely(!may_be)) {
1454 dm_bufio_release(b);
1455 return ts;
1456 }
1457 hash_offset = 0;
1458 may_be = MAY_BE_HASH | (ic->discard ? MAY_BE_FILLER : 0);
1459 }
1460 }
1461 }
1462 }
1463 dm_bufio_release(b);
1464
1465 tag += to_copy;
1466 *metadata_offset += to_copy;
1467 if (unlikely(*metadata_offset == 1U << SECTOR_SHIFT << ic->log2_buffer_sectors)) {
1468 (*metadata_block)++;
1469 *metadata_offset = 0;
1470 }
1471
1472 if (unlikely(!is_power_of_2(ic->tag_size)))
1473 hash_offset = (hash_offset + to_copy) % ic->tag_size;
1474
1475 total_size -= to_copy;
1476 } while (unlikely(total_size));
1477
1478 return 0;
1479 #undef MAY_BE_FILLER
1480 #undef MAY_BE_HASH
1481 }
1482
1483 struct flush_request {
1484 struct dm_io_request io_req;
1485 struct dm_io_region io_reg;
1486 struct dm_integrity_c *ic;
1487 struct completion comp;
1488 };
1489
1490 static void flush_notify(unsigned long error, void *fr_)
1491 {
1492 struct flush_request *fr = fr_;
1493
1494 if (unlikely(error != 0))
1495 dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO);
1496 complete(&fr->comp);
1497 }
1498
1499 static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data)
1500 {
1501 int r;
1502 struct flush_request fr;
1503
1504 if (!ic->meta_dev)
1505 flush_data = false;
1506 if (flush_data) {
1507 fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1508 fr.io_req.mem.type = DM_IO_KMEM;
1509 fr.io_req.mem.ptr.addr = NULL;
1510 fr.io_req.notify.fn = flush_notify;
1511 fr.io_req.notify.context = &fr;
1512 fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio);
1513 fr.io_reg.bdev = ic->dev->bdev;
1514 fr.io_reg.sector = 0;
1515 fr.io_reg.count = 0;
1516 fr.ic = ic;
1517 init_completion(&fr.comp);
1518 r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT);
1519 BUG_ON(r);
1520 }
1521
1522 r = dm_bufio_write_dirty_buffers(ic->bufio);
1523 if (unlikely(r))
1524 dm_integrity_io_error(ic, "writing tags", r);
1525
1526 if (flush_data)
1527 wait_for_completion(&fr.comp);
1528 }
1529
1530 static void sleep_on_endio_wait(struct dm_integrity_c *ic)
1531 {
1532 DECLARE_WAITQUEUE(wait, current);
1533
1534 __add_wait_queue(&ic->endio_wait, &wait);
1535 __set_current_state(TASK_UNINTERRUPTIBLE);
1536 spin_unlock_irq(&ic->endio_wait.lock);
1537 io_schedule();
1538 spin_lock_irq(&ic->endio_wait.lock);
1539 __remove_wait_queue(&ic->endio_wait, &wait);
1540 }
1541
1542 static void autocommit_fn(struct timer_list *t)
1543 {
1544 struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer);
1545
1546 if (likely(!dm_integrity_failed(ic)))
1547 queue_work(ic->commit_wq, &ic->commit_work);
1548 }
1549
1550 static void schedule_autocommit(struct dm_integrity_c *ic)
1551 {
1552 if (!timer_pending(&ic->autocommit_timer))
1553 mod_timer(&ic->autocommit_timer, jiffies + ic->autocommit_jiffies);
1554 }
1555
1556 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1557 {
1558 struct bio *bio;
1559 unsigned long flags;
1560
1561 spin_lock_irqsave(&ic->endio_wait.lock, flags);
1562 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1563 bio_list_add(&ic->flush_bio_list, bio);
1564 spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1565
1566 queue_work(ic->commit_wq, &ic->commit_work);
1567 }
1568
1569 static void do_endio(struct dm_integrity_c *ic, struct bio *bio)
1570 {
1571 int r;
1572
1573 r = dm_integrity_failed(ic);
1574 if (unlikely(r) && !bio->bi_status)
1575 bio->bi_status = errno_to_blk_status(r);
1576 if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) {
1577 unsigned long flags;
1578
1579 spin_lock_irqsave(&ic->endio_wait.lock, flags);
1580 bio_list_add(&ic->synchronous_bios, bio);
1581 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
1582 spin_unlock_irqrestore(&ic->endio_wait.lock, flags);
1583 return;
1584 }
1585 bio_endio(bio);
1586 }
1587
1588 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1589 {
1590 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1591
1592 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1593 submit_flush_bio(ic, dio);
1594 else
1595 do_endio(ic, bio);
1596 }
1597
1598 static void dec_in_flight(struct dm_integrity_io *dio)
1599 {
1600 if (atomic_dec_and_test(&dio->in_flight)) {
1601 struct dm_integrity_c *ic = dio->ic;
1602 struct bio *bio;
1603
1604 remove_range(ic, &dio->range);
1605
1606 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))
1607 schedule_autocommit(ic);
1608
1609 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1610 if (unlikely(dio->bi_status) && !bio->bi_status)
1611 bio->bi_status = dio->bi_status;
1612 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1613 dio->range.logical_sector += dio->range.n_sectors;
1614 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1615 INIT_WORK(&dio->work, integrity_bio_wait);
1616 queue_work(ic->offload_wq, &dio->work);
1617 return;
1618 }
1619 do_endio_flush(ic, dio);
1620 }
1621 }
1622
1623 static void integrity_end_io(struct bio *bio)
1624 {
1625 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1626
1627 dm_bio_restore(&dio->bio_details, bio);
1628 if (bio->bi_integrity)
1629 bio->bi_opf |= REQ_INTEGRITY;
1630
1631 if (dio->completion)
1632 complete(dio->completion);
1633
1634 dec_in_flight(dio);
1635 }
1636
1637 static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector,
1638 const char *data, char *result)
1639 {
1640 __le64 sector_le = cpu_to_le64(sector);
1641 SHASH_DESC_ON_STACK(req, ic->internal_hash);
1642 int r;
1643 unsigned int digest_size;
1644
1645 req->tfm = ic->internal_hash;
1646
1647 r = crypto_shash_init(req);
1648 if (unlikely(r < 0)) {
1649 dm_integrity_io_error(ic, "crypto_shash_init", r);
1650 goto failed;
1651 }
1652
1653 if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) {
1654 r = crypto_shash_update(req, (__u8 *)&ic->sb->salt, SALT_SIZE);
1655 if (unlikely(r < 0)) {
1656 dm_integrity_io_error(ic, "crypto_shash_update", r);
1657 goto failed;
1658 }
1659 }
1660
1661 r = crypto_shash_update(req, (const __u8 *)&sector_le, sizeof(sector_le));
1662 if (unlikely(r < 0)) {
1663 dm_integrity_io_error(ic, "crypto_shash_update", r);
1664 goto failed;
1665 }
1666
1667 r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT);
1668 if (unlikely(r < 0)) {
1669 dm_integrity_io_error(ic, "crypto_shash_update", r);
1670 goto failed;
1671 }
1672
1673 r = crypto_shash_final(req, result);
1674 if (unlikely(r < 0)) {
1675 dm_integrity_io_error(ic, "crypto_shash_final", r);
1676 goto failed;
1677 }
1678
1679 digest_size = crypto_shash_digestsize(ic->internal_hash);
1680 if (unlikely(digest_size < ic->tag_size))
1681 memset(result + digest_size, 0, ic->tag_size - digest_size);
1682
1683 return;
1684
1685 failed:
1686 /* this shouldn't happen anyway, the hash functions have no reason to fail */
1687 get_random_bytes(result, ic->tag_size);
1688 }
1689
1690 static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum)
1691 {
1692 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1693 struct dm_integrity_c *ic = dio->ic;
1694 struct bvec_iter iter;
1695 struct bio_vec bv;
1696 sector_t sector, logical_sector, area, offset;
1697 struct page *page;
1698
1699 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1700 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset,
1701 &dio->metadata_offset);
1702 sector = get_data_sector(ic, area, offset);
1703 logical_sector = dio->range.logical_sector;
1704
1705 page = mempool_alloc(&ic->recheck_pool, GFP_NOIO);
1706
1707 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1708 unsigned pos = 0;
1709
1710 do {
1711 sector_t alignment;
1712 char *mem;
1713 char *buffer = page_to_virt(page);
1714 int r;
1715 struct dm_io_request io_req;
1716 struct dm_io_region io_loc;
1717 io_req.bi_opf = REQ_OP_READ;
1718 io_req.mem.type = DM_IO_KMEM;
1719 io_req.mem.ptr.addr = buffer;
1720 io_req.notify.fn = NULL;
1721 io_req.client = ic->io;
1722 io_loc.bdev = ic->dev->bdev;
1723 io_loc.sector = sector;
1724 io_loc.count = ic->sectors_per_block;
1725
1726 /* Align the bio to logical block size */
1727 alignment = dio->range.logical_sector | bio_sectors(bio) | (PAGE_SIZE >> SECTOR_SHIFT);
1728 alignment &= -alignment;
1729 io_loc.sector = round_down(io_loc.sector, alignment);
1730 io_loc.count += sector - io_loc.sector;
1731 buffer += (sector - io_loc.sector) << SECTOR_SHIFT;
1732 io_loc.count = round_up(io_loc.count, alignment);
1733
1734 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
1735 if (unlikely(r)) {
1736 dio->bi_status = errno_to_blk_status(r);
1737 goto free_ret;
1738 }
1739
1740 integrity_sector_checksum(ic, logical_sector, buffer, checksum);
1741 r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block,
1742 &dio->metadata_offset, ic->tag_size, TAG_CMP);
1743 if (r) {
1744 if (r > 0) {
1745 DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
1746 bio->bi_bdev, logical_sector);
1747 atomic64_inc(&ic->number_of_mismatches);
1748 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
1749 bio, logical_sector, 0);
1750 r = -EILSEQ;
1751 }
1752 dio->bi_status = errno_to_blk_status(r);
1753 goto free_ret;
1754 }
1755
1756 mem = bvec_kmap_local(&bv);
1757 memcpy(mem + pos, buffer, ic->sectors_per_block << SECTOR_SHIFT);
1758 kunmap_local(mem);
1759
1760 pos += ic->sectors_per_block << SECTOR_SHIFT;
1761 sector += ic->sectors_per_block;
1762 logical_sector += ic->sectors_per_block;
1763 } while (pos < bv.bv_len);
1764 }
1765 free_ret:
1766 mempool_free(page, &ic->recheck_pool);
1767 }
1768
1769 static void integrity_metadata(struct work_struct *w)
1770 {
1771 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1772 struct dm_integrity_c *ic = dio->ic;
1773
1774 int r;
1775
1776 if (ic->internal_hash) {
1777 struct bvec_iter iter;
1778 struct bio_vec bv;
1779 unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash);
1780 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1781 char *checksums;
1782 unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
1783 char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
1784 sector_t sector;
1785 unsigned int sectors_to_process;
1786
1787 if (unlikely(ic->mode == 'R'))
1788 goto skip_io;
1789
1790 if (likely(dio->op != REQ_OP_DISCARD))
1791 checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space,
1792 GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1793 else
1794 checksums = kmalloc(PAGE_SIZE, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN);
1795 if (!checksums) {
1796 checksums = checksums_onstack;
1797 if (WARN_ON(extra_space &&
1798 digest_size > sizeof(checksums_onstack))) {
1799 r = -EINVAL;
1800 goto error;
1801 }
1802 }
1803
1804 if (unlikely(dio->op == REQ_OP_DISCARD)) {
1805 unsigned int bi_size = dio->bio_details.bi_iter.bi_size;
1806 unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE;
1807 unsigned int max_blocks = max_size / ic->tag_size;
1808
1809 memset(checksums, DISCARD_FILLER, max_size);
1810
1811 while (bi_size) {
1812 unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
1813
1814 this_step_blocks = min(this_step_blocks, max_blocks);
1815 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1816 this_step_blocks * ic->tag_size, TAG_WRITE);
1817 if (unlikely(r)) {
1818 if (likely(checksums != checksums_onstack))
1819 kfree(checksums);
1820 goto error;
1821 }
1822
1823 bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block);
1824 }
1825
1826 if (likely(checksums != checksums_onstack))
1827 kfree(checksums);
1828 goto skip_io;
1829 }
1830
1831 sector = dio->range.logical_sector;
1832 sectors_to_process = dio->range.n_sectors;
1833
1834 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1835 struct bio_vec bv_copy = bv;
1836 unsigned int pos;
1837 char *mem, *checksums_ptr;
1838
1839 again:
1840 mem = bvec_kmap_local(&bv_copy);
1841 pos = 0;
1842 checksums_ptr = checksums;
1843 do {
1844 integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr);
1845 checksums_ptr += ic->tag_size;
1846 sectors_to_process -= ic->sectors_per_block;
1847 pos += ic->sectors_per_block << SECTOR_SHIFT;
1848 sector += ic->sectors_per_block;
1849 } while (pos < bv_copy.bv_len && sectors_to_process && checksums != checksums_onstack);
1850 kunmap_local(mem);
1851
1852 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1853 checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);
1854 if (unlikely(r)) {
1855 if (likely(checksums != checksums_onstack))
1856 kfree(checksums);
1857 if (r > 0) {
1858 integrity_recheck(dio, checksums_onstack);
1859 goto skip_io;
1860 }
1861 goto error;
1862 }
1863
1864 if (!sectors_to_process)
1865 break;
1866
1867 if (unlikely(pos < bv_copy.bv_len)) {
1868 bv_copy.bv_offset += pos;
1869 bv_copy.bv_len -= pos;
1870 goto again;
1871 }
1872 }
1873
1874 if (likely(checksums != checksums_onstack))
1875 kfree(checksums);
1876 } else {
1877 struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;
1878
1879 if (bip) {
1880 struct bio_vec biv;
1881 struct bvec_iter iter;
1882 unsigned int data_to_process = dio->range.n_sectors;
1883
1884 sector_to_block(ic, data_to_process);
1885 data_to_process *= ic->tag_size;
1886
1887 bip_for_each_vec(biv, bip, iter) {
1888 unsigned char *tag;
1889 unsigned int this_len;
1890
1891 BUG_ON(PageHighMem(biv.bv_page));
1892 tag = bvec_virt(&biv);
1893 this_len = min(biv.bv_len, data_to_process);
1894 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1895 this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);
1896 if (unlikely(r))
1897 goto error;
1898 data_to_process -= this_len;
1899 if (!data_to_process)
1900 break;
1901 }
1902 }
1903 }
1904 skip_io:
1905 dec_in_flight(dio);
1906 return;
1907 error:
1908 dio->bi_status = errno_to_blk_status(r);
1909 dec_in_flight(dio);
1910 }
1911
1912 static inline bool dm_integrity_check_limits(struct dm_integrity_c *ic, sector_t logical_sector, struct bio *bio)
1913 {
1914 if (unlikely(logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1915 DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx",
1916 logical_sector, bio_sectors(bio),
1917 ic->provided_data_sectors);
1918 return false;
1919 }
1920 if (unlikely((logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block - 1))) {
1921 DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x",
1922 ic->sectors_per_block,
1923 logical_sector, bio_sectors(bio));
1924 return false;
1925 }
1926 if (ic->sectors_per_block > 1 && likely(bio_op(bio) != REQ_OP_DISCARD)) {
1927 struct bvec_iter iter;
1928 struct bio_vec bv;
1929
1930 bio_for_each_segment(bv, bio, iter) {
1931 if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) {
1932 DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary",
1933 bv.bv_offset, bv.bv_len, ic->sectors_per_block);
1934 return false;
1935 }
1936 }
1937 }
1938 return true;
1939 }
1940
1941 static int dm_integrity_map(struct dm_target *ti, struct bio *bio)
1942 {
1943 struct dm_integrity_c *ic = ti->private;
1944 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1945 struct bio_integrity_payload *bip;
1946
1947 sector_t area, offset;
1948
1949 dio->ic = ic;
1950 dio->bi_status = 0;
1951 dio->op = bio_op(bio);
1952
1953 if (ic->mode == 'I') {
1954 bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector);
1955 dio->integrity_payload = NULL;
1956 dio->integrity_payload_from_mempool = false;
1957 dio->integrity_range_locked = false;
1958 return dm_integrity_map_inline(dio, true);
1959 }
1960
1961 if (unlikely(dio->op == REQ_OP_DISCARD)) {
1962 if (ti->max_io_len) {
1963 sector_t sec = dm_target_offset(ti, bio->bi_iter.bi_sector);
1964 unsigned int log2_max_io_len = __fls(ti->max_io_len);
1965 sector_t start_boundary = sec >> log2_max_io_len;
1966 sector_t end_boundary = (sec + bio_sectors(bio) - 1) >> log2_max_io_len;
1967
1968 if (start_boundary < end_boundary) {
1969 sector_t len = ti->max_io_len - (sec & (ti->max_io_len - 1));
1970
1971 dm_accept_partial_bio(bio, len);
1972 }
1973 }
1974 }
1975
1976 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1977 submit_flush_bio(ic, dio);
1978 return DM_MAPIO_SUBMITTED;
1979 }
1980
1981 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1982 dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;
1983 if (unlikely(dio->fua)) {
1984 /*
1985 * Don't pass down the FUA flag because we have to flush
1986 * disk cache anyway.
1987 */
1988 bio->bi_opf &= ~REQ_FUA;
1989 }
1990 if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio)))
1991 return DM_MAPIO_KILL;
1992
1993 bip = bio_integrity(bio);
1994 if (!ic->internal_hash) {
1995 if (bip) {
1996 unsigned int wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block;
1997
1998 if (ic->log2_tag_size >= 0)
1999 wanted_tag_size <<= ic->log2_tag_size;
2000 else
2001 wanted_tag_size *= ic->tag_size;
2002 if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) {
2003 DMERR("Invalid integrity data size %u, expected %u",
2004 bip->bip_iter.bi_size, wanted_tag_size);
2005 return DM_MAPIO_KILL;
2006 }
2007 }
2008 } else {
2009 if (unlikely(bip != NULL)) {
2010 DMERR("Unexpected integrity data when using internal hash");
2011 return DM_MAPIO_KILL;
2012 }
2013 }
2014
2015 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))
2016 return DM_MAPIO_KILL;
2017
2018 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
2019 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
2020 bio->bi_iter.bi_sector = get_data_sector(ic, area, offset);
2021
2022 dm_integrity_map_continue(dio, true);
2023 return DM_MAPIO_SUBMITTED;
2024 }
2025
2026 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
2027 unsigned int journal_section, unsigned int journal_entry)
2028 {
2029 struct dm_integrity_c *ic = dio->ic;
2030 sector_t logical_sector;
2031 unsigned int n_sectors;
2032
2033 logical_sector = dio->range.logical_sector;
2034 n_sectors = dio->range.n_sectors;
2035 do {
2036 struct bio_vec bv = bio_iovec(bio);
2037 char *mem;
2038
2039 if (unlikely(bv.bv_len >> SECTOR_SHIFT > n_sectors))
2040 bv.bv_len = n_sectors << SECTOR_SHIFT;
2041 n_sectors -= bv.bv_len >> SECTOR_SHIFT;
2042 bio_advance_iter(bio, &bio->bi_iter, bv.bv_len);
2043 retry_kmap:
2044 mem = kmap_local_page(bv.bv_page);
2045 if (likely(dio->op == REQ_OP_WRITE))
2046 flush_dcache_page(bv.bv_page);
2047
2048 do {
2049 struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry);
2050
2051 if (unlikely(dio->op == REQ_OP_READ)) {
2052 struct journal_sector *js;
2053 char *mem_ptr;
2054 unsigned int s;
2055
2056 if (unlikely(journal_entry_is_inprogress(je))) {
2057 flush_dcache_page(bv.bv_page);
2058 kunmap_local(mem);
2059
2060 __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
2061 goto retry_kmap;
2062 }
2063 smp_rmb();
2064 BUG_ON(journal_entry_get_sector(je) != logical_sector);
2065 js = access_journal_data(ic, journal_section, journal_entry);
2066 mem_ptr = mem + bv.bv_offset;
2067 s = 0;
2068 do {
2069 memcpy(mem_ptr, js, JOURNAL_SECTOR_DATA);
2070 *(commit_id_t *)(mem_ptr + JOURNAL_SECTOR_DATA) = je->last_bytes[s];
2071 js++;
2072 mem_ptr += 1 << SECTOR_SHIFT;
2073 } while (++s < ic->sectors_per_block);
2074 #ifdef INTERNAL_VERIFY
2075 if (ic->internal_hash) {
2076 char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2077
2078 integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack);
2079 if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) {
2080 DMERR_LIMIT("Checksum failed when reading from journal, at sector 0x%llx",
2081 logical_sector);
2082 dm_audit_log_bio(DM_MSG_PREFIX, "journal-checksum",
2083 bio, logical_sector, 0);
2084 }
2085 }
2086 #endif
2087 }
2088
2089 if (!ic->internal_hash) {
2090 struct bio_integrity_payload *bip = bio_integrity(bio);
2091 unsigned int tag_todo = ic->tag_size;
2092 char *tag_ptr = journal_entry_tag(ic, je);
2093
2094 if (bip) {
2095 do {
2096 struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter);
2097 unsigned int tag_now = min(biv.bv_len, tag_todo);
2098 char *tag_addr;
2099
2100 BUG_ON(PageHighMem(biv.bv_page));
2101 tag_addr = bvec_virt(&biv);
2102 if (likely(dio->op == REQ_OP_WRITE))
2103 memcpy(tag_ptr, tag_addr, tag_now);
2104 else
2105 memcpy(tag_addr, tag_ptr, tag_now);
2106 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now);
2107 tag_ptr += tag_now;
2108 tag_todo -= tag_now;
2109 } while (unlikely(tag_todo));
2110 } else if (likely(dio->op == REQ_OP_WRITE))
2111 memset(tag_ptr, 0, tag_todo);
2112 }
2113
2114 if (likely(dio->op == REQ_OP_WRITE)) {
2115 struct journal_sector *js;
2116 unsigned int s;
2117
2118 js = access_journal_data(ic, journal_section, journal_entry);
2119 memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT);
2120
2121 s = 0;
2122 do {
2123 je->last_bytes[s] = js[s].commit_id;
2124 } while (++s < ic->sectors_per_block);
2125
2126 if (ic->internal_hash) {
2127 unsigned int digest_size = crypto_shash_digestsize(ic->internal_hash);
2128
2129 if (unlikely(digest_size > ic->tag_size)) {
2130 char checksums_onstack[HASH_MAX_DIGESTSIZE];
2131
2132 integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack);
2133 memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size);
2134 } else
2135 integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je));
2136 }
2137
2138 journal_entry_set_sector(je, logical_sector);
2139 }
2140 logical_sector += ic->sectors_per_block;
2141
2142 journal_entry++;
2143 if (unlikely(journal_entry == ic->journal_section_entries)) {
2144 journal_entry = 0;
2145 journal_section++;
2146 wraparound_section(ic, &journal_section);
2147 }
2148
2149 bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT;
2150 } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT);
2151
2152 if (unlikely(dio->op == REQ_OP_READ))
2153 flush_dcache_page(bv.bv_page);
2154 kunmap_local(mem);
2155 } while (n_sectors);
2156
2157 if (likely(dio->op == REQ_OP_WRITE)) {
2158 smp_mb();
2159 if (unlikely(waitqueue_active(&ic->copy_to_journal_wait)))
2160 wake_up(&ic->copy_to_journal_wait);
2161 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
2162 queue_work(ic->commit_wq, &ic->commit_work);
2163 else
2164 schedule_autocommit(ic);
2165 } else
2166 remove_range(ic, &dio->range);
2167
2168 if (unlikely(bio->bi_iter.bi_size)) {
2169 sector_t area, offset;
2170
2171 dio->range.logical_sector = logical_sector;
2172 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
2173 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
2174 return true;
2175 }
2176
2177 return false;
2178 }
2179
2180 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
2181 {
2182 struct dm_integrity_c *ic = dio->ic;
2183 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2184 unsigned int journal_section, journal_entry;
2185 unsigned int journal_read_pos;
2186 sector_t recalc_sector;
2187 struct completion read_comp;
2188 bool discard_retried = false;
2189 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;
2190
2191 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')
2192 need_sync_io = true;
2193
2194 if (need_sync_io && from_map) {
2195 INIT_WORK(&dio->work, integrity_bio_wait);
2196 queue_work(ic->offload_wq, &dio->work);
2197 return;
2198 }
2199
2200 lock_retry:
2201 spin_lock_irq(&ic->endio_wait.lock);
2202 retry:
2203 if (unlikely(dm_integrity_failed(ic))) {
2204 spin_unlock_irq(&ic->endio_wait.lock);
2205 do_endio(ic, bio);
2206 return;
2207 }
2208 dio->range.n_sectors = bio_sectors(bio);
2209 journal_read_pos = NOT_FOUND;
2210 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {
2211 if (dio->op == REQ_OP_WRITE) {
2212 unsigned int next_entry, i, pos;
2213 unsigned int ws, we, range_sectors;
2214
2215 dio->range.n_sectors = min(dio->range.n_sectors,
2216 (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block);
2217 if (unlikely(!dio->range.n_sectors)) {
2218 if (from_map)
2219 goto offload_to_thread;
2220 sleep_on_endio_wait(ic);
2221 goto retry;
2222 }
2223 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
2224 ic->free_sectors -= range_sectors;
2225 journal_section = ic->free_section;
2226 journal_entry = ic->free_section_entry;
2227
2228 next_entry = ic->free_section_entry + range_sectors;
2229 ic->free_section_entry = next_entry % ic->journal_section_entries;
2230 ic->free_section += next_entry / ic->journal_section_entries;
2231 ic->n_uncommitted_sections += next_entry / ic->journal_section_entries;
2232 wraparound_section(ic, &ic->free_section);
2233
2234 pos = journal_section * ic->journal_section_entries + journal_entry;
2235 ws = journal_section;
2236 we = journal_entry;
2237 i = 0;
2238 do {
2239 struct journal_entry *je;
2240
2241 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
2242 pos++;
2243 if (unlikely(pos >= ic->journal_entries))
2244 pos = 0;
2245
2246 je = access_journal_entry(ic, ws, we);
2247 BUG_ON(!journal_entry_is_unused(je));
2248 journal_entry_set_inprogress(je);
2249 we++;
2250 if (unlikely(we == ic->journal_section_entries)) {
2251 we = 0;
2252 ws++;
2253 wraparound_section(ic, &ws);
2254 }
2255 } while ((i += ic->sectors_per_block) < dio->range.n_sectors);
2256
2257 spin_unlock_irq(&ic->endio_wait.lock);
2258 goto journal_read_write;
2259 } else {
2260 sector_t next_sector;
2261
2262 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2263 if (likely(journal_read_pos == NOT_FOUND)) {
2264 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
2265 dio->range.n_sectors = next_sector - dio->range.logical_sector;
2266 } else {
2267 unsigned int i;
2268 unsigned int jp = journal_read_pos + 1;
2269
2270 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
2271 if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
2272 break;
2273 }
2274 dio->range.n_sectors = i;
2275 }
2276 }
2277 }
2278 if (unlikely(!add_new_range(ic, &dio->range, true))) {
2279 /*
2280 * We must not sleep in the request routine because it could
2281 * stall bios on current->bio_list.
2282 * So, we offload the bio to a workqueue if we have to sleep.
2283 */
2284 if (from_map) {
2285 offload_to_thread:
2286 spin_unlock_irq(&ic->endio_wait.lock);
2287 INIT_WORK(&dio->work, integrity_bio_wait);
2288 queue_work(ic->wait_wq, &dio->work);
2289 return;
2290 }
2291 if (journal_read_pos != NOT_FOUND)
2292 dio->range.n_sectors = ic->sectors_per_block;
2293 wait_and_add_new_range(ic, &dio->range);
2294 /*
2295 * wait_and_add_new_range drops the spinlock, so the journal
2296 * may have been changed arbitrarily. We need to recheck.
2297 * To simplify the code, we restrict I/O size to just one block.
2298 */
2299 if (journal_read_pos != NOT_FOUND) {
2300 sector_t next_sector;
2301 unsigned int new_pos;
2302
2303 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2304 if (unlikely(new_pos != journal_read_pos)) {
2305 remove_range_unlocked(ic, &dio->range);
2306 goto retry;
2307 }
2308 }
2309 }
2310 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {
2311 sector_t next_sector;
2312 unsigned int new_pos;
2313
2314 new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2315 if (unlikely(new_pos != NOT_FOUND) ||
2316 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {
2317 remove_range_unlocked(ic, &dio->range);
2318 spin_unlock_irq(&ic->endio_wait.lock);
2319 queue_work(ic->commit_wq, &ic->commit_work);
2320 flush_workqueue(ic->commit_wq);
2321 queue_work(ic->writer_wq, &ic->writer_work);
2322 flush_workqueue(ic->writer_wq);
2323 discard_retried = true;
2324 goto lock_retry;
2325 }
2326 }
2327 recalc_sector = le64_to_cpu(ic->sb->recalc_sector);
2328 spin_unlock_irq(&ic->endio_wait.lock);
2329
2330 if (unlikely(journal_read_pos != NOT_FOUND)) {
2331 journal_section = journal_read_pos / ic->journal_section_entries;
2332 journal_entry = journal_read_pos % ic->journal_section_entries;
2333 goto journal_read_write;
2334 }
2335
2336 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {
2337 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2338 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
2339 struct bitmap_block_status *bbs;
2340
2341 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);
2342 spin_lock(&bbs->bio_queue_lock);
2343 bio_list_add(&bbs->bio_queue, bio);
2344 spin_unlock(&bbs->bio_queue_lock);
2345 queue_work(ic->writer_wq, &bbs->work);
2346 return;
2347 }
2348 }
2349
2350 dio->in_flight = (atomic_t)ATOMIC_INIT(2);
2351
2352 if (need_sync_io) {
2353 init_completion(&read_comp);
2354 dio->completion = &read_comp;
2355 } else
2356 dio->completion = NULL;
2357
2358 dm_bio_record(&dio->bio_details, bio);
2359 bio_set_dev(bio, ic->dev->bdev);
2360 bio->bi_integrity = NULL;
2361 bio->bi_opf &= ~REQ_INTEGRITY;
2362 bio->bi_end_io = integrity_end_io;
2363 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
2364
2365 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {
2366 integrity_metadata(&dio->work);
2367 dm_integrity_flush_buffers(ic, false);
2368
2369 dio->in_flight = (atomic_t)ATOMIC_INIT(1);
2370 dio->completion = NULL;
2371
2372 submit_bio_noacct(bio);
2373
2374 return;
2375 }
2376
2377 submit_bio_noacct(bio);
2378
2379 if (need_sync_io) {
2380 wait_for_completion_io(&read_comp);
2381 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
2382 dio->range.logical_sector + dio->range.n_sectors > recalc_sector)
2383 goto skip_check;
2384 if (ic->mode == 'B') {
2385 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,
2386 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
2387 goto skip_check;
2388 }
2389
2390 if (likely(!bio->bi_status))
2391 integrity_metadata(&dio->work);
2392 else
2393 skip_check:
2394 dec_in_flight(dio);
2395 } else {
2396 INIT_WORK(&dio->work, integrity_metadata);
2397 queue_work(ic->metadata_wq, &dio->work);
2398 }
2399
2400 return;
2401
2402 journal_read_write:
2403 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
2404 goto lock_retry;
2405
2406 do_endio_flush(ic, dio);
2407 }
2408
2409 static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map)
2410 {
2411 struct dm_integrity_c *ic = dio->ic;
2412 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2413 struct bio_integrity_payload *bip;
2414 unsigned ret;
2415 sector_t recalc_sector;
2416
2417 if (unlikely(bio_integrity(bio))) {
2418 bio->bi_status = BLK_STS_NOTSUPP;
2419 bio_endio(bio);
2420 return DM_MAPIO_SUBMITTED;
2421 }
2422
2423 bio_set_dev(bio, ic->dev->bdev);
2424 if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0))
2425 return DM_MAPIO_REMAPPED;
2426
2427 retry:
2428 if (!dio->integrity_payload) {
2429 unsigned digest_size, extra_size;
2430 dio->payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block);
2431 digest_size = crypto_shash_digestsize(ic->internal_hash);
2432 extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0;
2433 dio->payload_len += extra_size;
2434 dio->integrity_payload = kmalloc(dio->payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
2435 if (unlikely(!dio->integrity_payload)) {
2436 const unsigned x_size = PAGE_SIZE << 1;
2437 if (dio->payload_len > x_size) {
2438 unsigned sectors = ((x_size - extra_size) / ic->tuple_size) << ic->sb->log2_sectors_per_block;
2439 if (WARN_ON(!sectors || sectors >= bio_sectors(bio))) {
2440 bio->bi_status = BLK_STS_NOTSUPP;
2441 bio_endio(bio);
2442 return DM_MAPIO_SUBMITTED;
2443 }
2444 dm_accept_partial_bio(bio, sectors);
2445 goto retry;
2446 }
2447 }
2448 }
2449
2450 dio->range.logical_sector = bio->bi_iter.bi_sector;
2451 dio->range.n_sectors = bio_sectors(bio);
2452
2453 if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)))
2454 goto skip_spinlock;
2455 #ifdef CONFIG_64BIT
2456 /*
2457 * On 64-bit CPUs we can optimize the lock away (so that it won't cause
2458 * cache line bouncing) and use acquire/release barriers instead.
2459 *
2460 * Paired with smp_store_release in integrity_recalc_inline.
2461 */
2462 recalc_sector = le64_to_cpu(smp_load_acquire(&ic->sb->recalc_sector));
2463 if (likely(dio->range.logical_sector + dio->range.n_sectors <= recalc_sector))
2464 goto skip_spinlock;
2465 #endif
2466 spin_lock_irq(&ic->endio_wait.lock);
2467 recalc_sector = le64_to_cpu(ic->sb->recalc_sector);
2468 if (dio->range.logical_sector + dio->range.n_sectors <= recalc_sector)
2469 goto skip_unlock;
2470 if (unlikely(!add_new_range(ic, &dio->range, true))) {
2471 if (from_map) {
2472 spin_unlock_irq(&ic->endio_wait.lock);
2473 INIT_WORK(&dio->work, integrity_bio_wait);
2474 queue_work(ic->wait_wq, &dio->work);
2475 return DM_MAPIO_SUBMITTED;
2476 }
2477 wait_and_add_new_range(ic, &dio->range);
2478 }
2479 dio->integrity_range_locked = true;
2480 skip_unlock:
2481 spin_unlock_irq(&ic->endio_wait.lock);
2482 skip_spinlock:
2483
2484 if (unlikely(!dio->integrity_payload)) {
2485 dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO));
2486 dio->integrity_payload_from_mempool = true;
2487 }
2488
2489 dio->bio_details.bi_iter = bio->bi_iter;
2490
2491 if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) {
2492 return DM_MAPIO_KILL;
2493 }
2494
2495 bio->bi_iter.bi_sector += ic->start + SB_SECTORS;
2496
2497 bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
2498 if (IS_ERR(bip)) {
2499 bio->bi_status = errno_to_blk_status(PTR_ERR(bip));
2500 bio_endio(bio);
2501 return DM_MAPIO_SUBMITTED;
2502 }
2503
2504 if (dio->op == REQ_OP_WRITE) {
2505 unsigned pos = 0;
2506 while (dio->bio_details.bi_iter.bi_size) {
2507 struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2508 const char *mem = bvec_kmap_local(&bv);
2509 if (ic->tag_size < ic->tuple_size)
2510 memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size);
2511 integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, dio->integrity_payload + pos);
2512 kunmap_local(mem);
2513 pos += ic->tuple_size;
2514 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2515 }
2516 }
2517
2518 ret = bio_integrity_add_page(bio, virt_to_page(dio->integrity_payload),
2519 dio->payload_len, offset_in_page(dio->integrity_payload));
2520 if (unlikely(ret != dio->payload_len)) {
2521 bio->bi_status = BLK_STS_RESOURCE;
2522 bio_endio(bio);
2523 return DM_MAPIO_SUBMITTED;
2524 }
2525
2526 return DM_MAPIO_REMAPPED;
2527 }
2528
2529 static inline void dm_integrity_free_payload(struct dm_integrity_io *dio)
2530 {
2531 struct dm_integrity_c *ic = dio->ic;
2532 if (unlikely(dio->integrity_payload_from_mempool))
2533 mempool_free(virt_to_page(dio->integrity_payload), &ic->recheck_pool);
2534 else
2535 kfree(dio->integrity_payload);
2536 dio->integrity_payload = NULL;
2537 dio->integrity_payload_from_mempool = false;
2538 }
2539
2540 static void dm_integrity_inline_recheck(struct work_struct *w)
2541 {
2542 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2543 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2544 struct dm_integrity_c *ic = dio->ic;
2545 struct bio *outgoing_bio;
2546 void *outgoing_data;
2547
2548 dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO));
2549 dio->integrity_payload_from_mempool = true;
2550
2551 outgoing_data = dio->integrity_payload + PAGE_SIZE;
2552
2553 while (dio->bio_details.bi_iter.bi_size) {
2554 char digest[HASH_MAX_DIGESTSIZE];
2555 int r;
2556 struct bio_integrity_payload *bip;
2557 struct bio_vec bv;
2558 char *mem;
2559
2560 outgoing_bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recheck_bios);
2561
2562 r = bio_add_page(outgoing_bio, virt_to_page(outgoing_data), ic->sectors_per_block << SECTOR_SHIFT, 0);
2563 if (unlikely(r != (ic->sectors_per_block << SECTOR_SHIFT))) {
2564 bio_put(outgoing_bio);
2565 bio->bi_status = BLK_STS_RESOURCE;
2566 bio_endio(bio);
2567 return;
2568 }
2569
2570 bip = bio_integrity_alloc(outgoing_bio, GFP_NOIO, 1);
2571 if (IS_ERR(bip)) {
2572 bio_put(outgoing_bio);
2573 bio->bi_status = errno_to_blk_status(PTR_ERR(bip));
2574 bio_endio(bio);
2575 return;
2576 }
2577
2578 r = bio_integrity_add_page(outgoing_bio, virt_to_page(dio->integrity_payload), ic->tuple_size, 0);
2579 if (unlikely(r != ic->tuple_size)) {
2580 bio_put(outgoing_bio);
2581 bio->bi_status = BLK_STS_RESOURCE;
2582 bio_endio(bio);
2583 return;
2584 }
2585
2586 outgoing_bio->bi_iter.bi_sector = dio->bio_details.bi_iter.bi_sector + ic->start + SB_SECTORS;
2587
2588 r = submit_bio_wait(outgoing_bio);
2589 if (unlikely(r != 0)) {
2590 bio_put(outgoing_bio);
2591 bio->bi_status = errno_to_blk_status(r);
2592 bio_endio(bio);
2593 return;
2594 }
2595 bio_put(outgoing_bio);
2596
2597 integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, outgoing_data, digest);
2598 if (unlikely(memcmp(digest, dio->integrity_payload, min(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)))) {
2599 DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx",
2600 ic->dev->bdev, dio->bio_details.bi_iter.bi_sector);
2601 atomic64_inc(&ic->number_of_mismatches);
2602 dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum",
2603 bio, dio->bio_details.bi_iter.bi_sector, 0);
2604
2605 bio->bi_status = BLK_STS_PROTECTION;
2606 bio_endio(bio);
2607 return;
2608 }
2609
2610 bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2611 mem = bvec_kmap_local(&bv);
2612 memcpy(mem, outgoing_data, ic->sectors_per_block << SECTOR_SHIFT);
2613 kunmap_local(mem);
2614
2615 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2616 }
2617
2618 bio_endio(bio);
2619 }
2620
2621 static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
2622 {
2623 struct dm_integrity_c *ic = ti->private;
2624 if (ic->mode == 'I') {
2625 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2626 if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK)) {
2627 unsigned pos = 0;
2628 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
2629 unlikely(dio->integrity_range_locked))
2630 goto skip_check;
2631 while (dio->bio_details.bi_iter.bi_size) {
2632 char digest[HASH_MAX_DIGESTSIZE];
2633 struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter);
2634 char *mem = bvec_kmap_local(&bv);
2635 //memset(mem, 0xff, ic->sectors_per_block << SECTOR_SHIFT);
2636 integrity_sector_checksum(ic, dio->bio_details.bi_iter.bi_sector, mem, digest);
2637 if (unlikely(memcmp(digest, dio->integrity_payload + pos,
2638 min(crypto_shash_digestsize(ic->internal_hash), ic->tag_size)))) {
2639 kunmap_local(mem);
2640 dm_integrity_free_payload(dio);
2641 INIT_WORK(&dio->work, dm_integrity_inline_recheck);
2642 queue_work(ic->offload_wq, &dio->work);
2643 return DM_ENDIO_INCOMPLETE;
2644 }
2645 kunmap_local(mem);
2646 pos += ic->tuple_size;
2647 bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT);
2648 }
2649 }
2650 skip_check:
2651 dm_integrity_free_payload(dio);
2652 if (unlikely(dio->integrity_range_locked))
2653 remove_range(ic, &dio->range);
2654 }
2655 return DM_ENDIO_DONE;
2656 }
2657
2658 static void integrity_bio_wait(struct work_struct *w)
2659 {
2660 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2661 struct dm_integrity_c *ic = dio->ic;
2662
2663 if (ic->mode == 'I') {
2664 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
2665 int r = dm_integrity_map_inline(dio, false);
2666 switch (r) {
2667 case DM_MAPIO_KILL:
2668 bio->bi_status = BLK_STS_IOERR;
2669 fallthrough;
2670 case DM_MAPIO_REMAPPED:
2671 submit_bio_noacct(bio);
2672 fallthrough;
2673 case DM_MAPIO_SUBMITTED:
2674 return;
2675 default:
2676 BUG();
2677 }
2678 } else {
2679 dm_integrity_map_continue(dio, false);
2680 }
2681 }
2682
2683 static void pad_uncommitted(struct dm_integrity_c *ic)
2684 {
2685 if (ic->free_section_entry) {
2686 ic->free_sectors -= ic->journal_section_entries - ic->free_section_entry;
2687 ic->free_section_entry = 0;
2688 ic->free_section++;
2689 wraparound_section(ic, &ic->free_section);
2690 ic->n_uncommitted_sections++;
2691 }
2692 if (WARN_ON(ic->journal_sections * ic->journal_section_entries !=
2693 (ic->n_uncommitted_sections + ic->n_committed_sections) *
2694 ic->journal_section_entries + ic->free_sectors)) {
2695 DMCRIT("journal_sections %u, journal_section_entries %u, "
2696 "n_uncommitted_sections %u, n_committed_sections %u, "
2697 "journal_section_entries %u, free_sectors %u",
2698 ic->journal_sections, ic->journal_section_entries,
2699 ic->n_uncommitted_sections, ic->n_committed_sections,
2700 ic->journal_section_entries, ic->free_sectors);
2701 }
2702 }
2703
2704 static void integrity_commit(struct work_struct *w)
2705 {
2706 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work);
2707 unsigned int commit_start, commit_sections;
2708 unsigned int i, j, n;
2709 struct bio *flushes;
2710
2711 del_timer(&ic->autocommit_timer);
2712
2713 if (ic->mode == 'I')
2714 return;
2715
2716 spin_lock_irq(&ic->endio_wait.lock);
2717 flushes = bio_list_get(&ic->flush_bio_list);
2718 if (unlikely(ic->mode != 'J')) {
2719 spin_unlock_irq(&ic->endio_wait.lock);
2720 dm_integrity_flush_buffers(ic, true);
2721 goto release_flush_bios;
2722 }
2723
2724 pad_uncommitted(ic);
2725 commit_start = ic->uncommitted_section;
2726 commit_sections = ic->n_uncommitted_sections;
2727 spin_unlock_irq(&ic->endio_wait.lock);
2728
2729 if (!commit_sections)
2730 goto release_flush_bios;
2731
2732 ic->wrote_to_journal = true;
2733
2734 i = commit_start;
2735 for (n = 0; n < commit_sections; n++) {
2736 for (j = 0; j < ic->journal_section_entries; j++) {
2737 struct journal_entry *je;
2738
2739 je = access_journal_entry(ic, i, j);
2740 io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je));
2741 }
2742 for (j = 0; j < ic->journal_section_sectors; j++) {
2743 struct journal_sector *js;
2744
2745 js = access_journal(ic, i, j);
2746 js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq);
2747 }
2748 i++;
2749 if (unlikely(i >= ic->journal_sections))
2750 ic->commit_seq = next_commit_seq(ic->commit_seq);
2751 wraparound_section(ic, &i);
2752 }
2753 smp_rmb();
2754
2755 write_journal(ic, commit_start, commit_sections);
2756
2757 spin_lock_irq(&ic->endio_wait.lock);
2758 ic->uncommitted_section += commit_sections;
2759 wraparound_section(ic, &ic->uncommitted_section);
2760 ic->n_uncommitted_sections -= commit_sections;
2761 ic->n_committed_sections += commit_sections;
2762 spin_unlock_irq(&ic->endio_wait.lock);
2763
2764 if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold)
2765 queue_work(ic->writer_wq, &ic->writer_work);
2766
2767 release_flush_bios:
2768 while (flushes) {
2769 struct bio *next = flushes->bi_next;
2770
2771 flushes->bi_next = NULL;
2772 do_endio(ic, flushes);
2773 flushes = next;
2774 }
2775 }
2776
2777 static void complete_copy_from_journal(unsigned long error, void *context)
2778 {
2779 struct journal_io *io = context;
2780 struct journal_completion *comp = io->comp;
2781 struct dm_integrity_c *ic = comp->ic;
2782
2783 remove_range(ic, &io->range);
2784 mempool_free(io, &ic->journal_io_mempool);
2785 if (unlikely(error != 0))
2786 dm_integrity_io_error(ic, "copying from journal", -EIO);
2787 complete_journal_op(comp);
2788 }
2789
2790 static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js,
2791 struct journal_entry *je)
2792 {
2793 unsigned int s = 0;
2794
2795 do {
2796 js->commit_id = je->last_bytes[s];
2797 js++;
2798 } while (++s < ic->sectors_per_block);
2799 }
2800
2801 static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start,
2802 unsigned int write_sections, bool from_replay)
2803 {
2804 unsigned int i, j, n;
2805 struct journal_completion comp;
2806 struct blk_plug plug;
2807
2808 blk_start_plug(&plug);
2809
2810 comp.ic = ic;
2811 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
2812 init_completion(&comp.comp);
2813
2814 i = write_start;
2815 for (n = 0; n < write_sections; n++, i++, wraparound_section(ic, &i)) {
2816 #ifndef INTERNAL_VERIFY
2817 if (unlikely(from_replay))
2818 #endif
2819 rw_section_mac(ic, i, false);
2820 for (j = 0; j < ic->journal_section_entries; j++) {
2821 struct journal_entry *je = access_journal_entry(ic, i, j);
2822 sector_t sec, area, offset;
2823 unsigned int k, l, next_loop;
2824 sector_t metadata_block;
2825 unsigned int metadata_offset;
2826 struct journal_io *io;
2827
2828 if (journal_entry_is_unused(je))
2829 continue;
2830 BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay);
2831 sec = journal_entry_get_sector(je);
2832 if (unlikely(from_replay)) {
2833 if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) {
2834 dm_integrity_io_error(ic, "invalid sector in journal", -EIO);
2835 sec &= ~(sector_t)(ic->sectors_per_block - 1);
2836 }
2837 if (unlikely(sec >= ic->provided_data_sectors)) {
2838 journal_entry_set_unused(je);
2839 continue;
2840 }
2841 }
2842 get_area_and_offset(ic, sec, &area, &offset);
2843 restore_last_bytes(ic, access_journal_data(ic, i, j), je);
2844 for (k = j + 1; k < ic->journal_section_entries; k++) {
2845 struct journal_entry *je2 = access_journal_entry(ic, i, k);
2846 sector_t sec2, area2, offset2;
2847
2848 if (journal_entry_is_unused(je2))
2849 break;
2850 BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay);
2851 sec2 = journal_entry_get_sector(je2);
2852 if (unlikely(sec2 >= ic->provided_data_sectors))
2853 break;
2854 get_area_and_offset(ic, sec2, &area2, &offset2);
2855 if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block))
2856 break;
2857 restore_last_bytes(ic, access_journal_data(ic, i, k), je2);
2858 }
2859 next_loop = k - 1;
2860
2861 io = mempool_alloc(&ic->journal_io_mempool, GFP_NOIO);
2862 io->comp = &comp;
2863 io->range.logical_sector = sec;
2864 io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block;
2865
2866 spin_lock_irq(&ic->endio_wait.lock);
2867 add_new_range_and_wait(ic, &io->range);
2868
2869 if (likely(!from_replay)) {
2870 struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries];
2871
2872 /* don't write if there is newer committed sector */
2873 while (j < k && find_newer_committed_node(ic, &section_node[j])) {
2874 struct journal_entry *je2 = access_journal_entry(ic, i, j);
2875
2876 journal_entry_set_unused(je2);
2877 remove_journal_node(ic, &section_node[j]);
2878 j++;
2879 sec += ic->sectors_per_block;
2880 offset += ic->sectors_per_block;
2881 }
2882 while (j < k && find_newer_committed_node(ic, &section_node[k - 1])) {
2883 struct journal_entry *je2 = access_journal_entry(ic, i, k - 1);
2884
2885 journal_entry_set_unused(je2);
2886 remove_journal_node(ic, &section_node[k - 1]);
2887 k--;
2888 }
2889 if (j == k) {
2890 remove_range_unlocked(ic, &io->range);
2891 spin_unlock_irq(&ic->endio_wait.lock);
2892 mempool_free(io, &ic->journal_io_mempool);
2893 goto skip_io;
2894 }
2895 for (l = j; l < k; l++)
2896 remove_journal_node(ic, &section_node[l]);
2897 }
2898 spin_unlock_irq(&ic->endio_wait.lock);
2899
2900 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
2901 for (l = j; l < k; l++) {
2902 int r;
2903 struct journal_entry *je2 = access_journal_entry(ic, i, l);
2904
2905 if (
2906 #ifndef INTERNAL_VERIFY
2907 unlikely(from_replay) &&
2908 #endif
2909 ic->internal_hash) {
2910 char test_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)];
2911
2912 integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block),
2913 (char *)access_journal_data(ic, i, l), test_tag);
2914 if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) {
2915 dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ);
2916 dm_audit_log_target(DM_MSG_PREFIX, "integrity-replay-journal", ic->ti, 0);
2917 }
2918 }
2919
2920 journal_entry_set_unused(je2);
2921 r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset,
2922 ic->tag_size, TAG_WRITE);
2923 if (unlikely(r))
2924 dm_integrity_io_error(ic, "reading tags", r);
2925 }
2926
2927 atomic_inc(&comp.in_flight);
2928 copy_from_journal(ic, i, j << ic->sb->log2_sectors_per_block,
2929 (k - j) << ic->sb->log2_sectors_per_block,
2930 get_data_sector(ic, area, offset),
2931 complete_copy_from_journal, io);
2932 skip_io:
2933 j = next_loop;
2934 }
2935 }
2936
2937 dm_bufio_write_dirty_buffers_async(ic->bufio);
2938
2939 blk_finish_plug(&plug);
2940
2941 complete_journal_op(&comp);
2942 wait_for_completion_io(&comp.comp);
2943
2944 dm_integrity_flush_buffers(ic, true);
2945 }
2946
2947 static void integrity_writer(struct work_struct *w)
2948 {
2949 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work);
2950 unsigned int write_start, write_sections;
2951 unsigned int prev_free_sectors;
2952
2953 spin_lock_irq(&ic->endio_wait.lock);
2954 write_start = ic->committed_section;
2955 write_sections = ic->n_committed_sections;
2956 spin_unlock_irq(&ic->endio_wait.lock);
2957
2958 if (!write_sections)
2959 return;
2960
2961 do_journal_write(ic, write_start, write_sections, false);
2962
2963 spin_lock_irq(&ic->endio_wait.lock);
2964
2965 ic->committed_section += write_sections;
2966 wraparound_section(ic, &ic->committed_section);
2967 ic->n_committed_sections -= write_sections;
2968
2969 prev_free_sectors = ic->free_sectors;
2970 ic->free_sectors += write_sections * ic->journal_section_entries;
2971 if (unlikely(!prev_free_sectors))
2972 wake_up_locked(&ic->endio_wait);
2973
2974 spin_unlock_irq(&ic->endio_wait.lock);
2975 }
2976
2977 static void recalc_write_super(struct dm_integrity_c *ic)
2978 {
2979 int r;
2980
2981 dm_integrity_flush_buffers(ic, false);
2982 if (dm_integrity_failed(ic))
2983 return;
2984
2985 r = sync_rw_sb(ic, REQ_OP_WRITE);
2986 if (unlikely(r))
2987 dm_integrity_io_error(ic, "writing superblock", r);
2988 }
2989
2990 static void integrity_recalc(struct work_struct *w)
2991 {
2992 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
2993 size_t recalc_tags_size;
2994 u8 *recalc_buffer = NULL;
2995 u8 *recalc_tags = NULL;
2996 struct dm_integrity_range range;
2997 struct dm_io_request io_req;
2998 struct dm_io_region io_loc;
2999 sector_t area, offset;
3000 sector_t metadata_block;
3001 unsigned int metadata_offset;
3002 sector_t logical_sector, n_sectors;
3003 __u8 *t;
3004 unsigned int i;
3005 int r;
3006 unsigned int super_counter = 0;
3007 unsigned recalc_sectors = RECALC_SECTORS;
3008
3009 retry:
3010 recalc_buffer = __vmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO);
3011 if (!recalc_buffer) {
3012 oom:
3013 recalc_sectors >>= 1;
3014 if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block)
3015 goto retry;
3016 DMCRIT("out of memory for recalculate buffer - recalculation disabled");
3017 goto free_ret;
3018 }
3019 recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size;
3020 if (crypto_shash_digestsize(ic->internal_hash) > ic->tag_size)
3021 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tag_size;
3022 recalc_tags = kvmalloc(recalc_tags_size, GFP_NOIO);
3023 if (!recalc_tags) {
3024 vfree(recalc_buffer);
3025 recalc_buffer = NULL;
3026 goto oom;
3027 }
3028
3029 DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector));
3030
3031 spin_lock_irq(&ic->endio_wait.lock);
3032
3033 next_chunk:
3034
3035 if (unlikely(dm_post_suspending(ic->ti)))
3036 goto unlock_ret;
3037
3038 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
3039 if (unlikely(range.logical_sector >= ic->provided_data_sectors)) {
3040 if (ic->mode == 'B') {
3041 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3042 DEBUG_print("queue_delayed_work: bitmap_flush_work\n");
3043 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
3044 }
3045 goto unlock_ret;
3046 }
3047
3048 get_area_and_offset(ic, range.logical_sector, &area, &offset);
3049 range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector);
3050 if (!ic->meta_dev)
3051 range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsigned int)offset);
3052
3053 add_new_range_and_wait(ic, &range);
3054 spin_unlock_irq(&ic->endio_wait.lock);
3055 logical_sector = range.logical_sector;
3056 n_sectors = range.n_sectors;
3057
3058 if (ic->mode == 'B') {
3059 if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
3060 goto advance_and_next;
3061
3062 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector,
3063 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) {
3064 logical_sector += ic->sectors_per_block;
3065 n_sectors -= ic->sectors_per_block;
3066 cond_resched();
3067 }
3068 while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block,
3069 ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) {
3070 n_sectors -= ic->sectors_per_block;
3071 cond_resched();
3072 }
3073 get_area_and_offset(ic, logical_sector, &area, &offset);
3074 }
3075
3076 DEBUG_print("recalculating: %llx, %llx\n", logical_sector, n_sectors);
3077
3078 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
3079 recalc_write_super(ic);
3080 if (ic->mode == 'B')
3081 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval);
3082
3083 super_counter = 0;
3084 }
3085
3086 if (unlikely(dm_integrity_failed(ic)))
3087 goto err;
3088
3089 io_req.bi_opf = REQ_OP_READ;
3090 io_req.mem.type = DM_IO_VMA;
3091 io_req.mem.ptr.addr = recalc_buffer;
3092 io_req.notify.fn = NULL;
3093 io_req.client = ic->io;
3094 io_loc.bdev = ic->dev->bdev;
3095 io_loc.sector = get_data_sector(ic, area, offset);
3096 io_loc.count = n_sectors;
3097
3098 r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT);
3099 if (unlikely(r)) {
3100 dm_integrity_io_error(ic, "reading data", r);
3101 goto err;
3102 }
3103
3104 t = recalc_tags;
3105 for (i = 0; i < n_sectors; i += ic->sectors_per_block) {
3106 integrity_sector_checksum(ic, logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t);
3107 t += ic->tag_size;
3108 }
3109
3110 metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset);
3111
3112 r = dm_integrity_rw_tag(ic, recalc_tags, &metadata_block, &metadata_offset, t - recalc_tags, TAG_WRITE);
3113 if (unlikely(r)) {
3114 dm_integrity_io_error(ic, "writing tags", r);
3115 goto err;
3116 }
3117
3118 if (ic->mode == 'B') {
3119 sector_t start, end;
3120
3121 start = (range.logical_sector >>
3122 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) <<
3123 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
3124 end = ((range.logical_sector + range.n_sectors) >>
3125 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) <<
3126 (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
3127 block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR);
3128 }
3129
3130 advance_and_next:
3131 cond_resched();
3132
3133 spin_lock_irq(&ic->endio_wait.lock);
3134 remove_range_unlocked(ic, &range);
3135 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
3136 goto next_chunk;
3137
3138 err:
3139 remove_range(ic, &range);
3140 goto free_ret;
3141
3142 unlock_ret:
3143 spin_unlock_irq(&ic->endio_wait.lock);
3144
3145 recalc_write_super(ic);
3146
3147 free_ret:
3148 vfree(recalc_buffer);
3149 kvfree(recalc_tags);
3150 }
3151
3152 static void integrity_recalc_inline(struct work_struct *w)
3153 {
3154 struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work);
3155 size_t recalc_tags_size;
3156 u8 *recalc_buffer = NULL;
3157 u8 *recalc_tags = NULL;
3158 struct dm_integrity_range range;
3159 struct bio *bio;
3160 struct bio_integrity_payload *bip;
3161 __u8 *t;
3162 unsigned int i;
3163 int r;
3164 unsigned ret;
3165 unsigned int super_counter = 0;
3166 unsigned recalc_sectors = RECALC_SECTORS;
3167
3168 retry:
3169 recalc_buffer = kmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO | __GFP_NOWARN);
3170 if (!recalc_buffer) {
3171 oom:
3172 recalc_sectors >>= 1;
3173 if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block)
3174 goto retry;
3175 DMCRIT("out of memory for recalculate buffer - recalculation disabled");
3176 goto free_ret;
3177 }
3178
3179 recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size;
3180 if (crypto_shash_digestsize(ic->internal_hash) > ic->tuple_size)
3181 recalc_tags_size += crypto_shash_digestsize(ic->internal_hash) - ic->tuple_size;
3182 recalc_tags = kmalloc(recalc_tags_size, GFP_NOIO | __GFP_NOWARN);
3183 if (!recalc_tags) {
3184 kfree(recalc_buffer);
3185 recalc_buffer = NULL;
3186 goto oom;
3187 }
3188
3189 spin_lock_irq(&ic->endio_wait.lock);
3190
3191 next_chunk:
3192 if (unlikely(dm_post_suspending(ic->ti)))
3193 goto unlock_ret;
3194
3195 range.logical_sector = le64_to_cpu(ic->sb->recalc_sector);
3196 if (unlikely(range.logical_sector >= ic->provided_data_sectors))
3197 goto unlock_ret;
3198 range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector);
3199
3200 add_new_range_and_wait(ic, &range);
3201 spin_unlock_irq(&ic->endio_wait.lock);
3202
3203 if (unlikely(++super_counter == RECALC_WRITE_SUPER)) {
3204 recalc_write_super(ic);
3205 super_counter = 0;
3206 }
3207
3208 if (unlikely(dm_integrity_failed(ic)))
3209 goto err;
3210
3211 DEBUG_print("recalculating: %llx - %llx\n", range.logical_sector, range.n_sectors);
3212
3213 bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recalc_bios);
3214 bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector;
3215 __bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer));
3216 r = submit_bio_wait(bio);
3217 bio_put(bio);
3218 if (unlikely(r)) {
3219 dm_integrity_io_error(ic, "reading data", r);
3220 goto err;
3221 }
3222
3223 t = recalc_tags;
3224 for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) {
3225 memset(t, 0, ic->tuple_size);
3226 integrity_sector_checksum(ic, range.logical_sector + i, recalc_buffer + (i << SECTOR_SHIFT), t);
3227 t += ic->tuple_size;
3228 }
3229
3230 bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_WRITE, GFP_NOIO, &ic->recalc_bios);
3231 bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector;
3232 __bio_add_page(bio, virt_to_page(recalc_buffer), range.n_sectors << SECTOR_SHIFT, offset_in_page(recalc_buffer));
3233
3234 bip = bio_integrity_alloc(bio, GFP_NOIO, 1);
3235 if (unlikely(IS_ERR(bip))) {
3236 bio_put(bio);
3237 DMCRIT("out of memory for bio integrity payload - recalculation disabled");
3238 goto err;
3239 }
3240 ret = bio_integrity_add_page(bio, virt_to_page(recalc_tags), t - recalc_tags, offset_in_page(recalc_tags));
3241 if (unlikely(ret != t - recalc_tags)) {
3242 bio_put(bio);
3243 dm_integrity_io_error(ic, "attaching integrity tags", -ENOMEM);
3244 goto err;
3245 }
3246
3247 r = submit_bio_wait(bio);
3248 bio_put(bio);
3249 if (unlikely(r)) {
3250 dm_integrity_io_error(ic, "writing data", r);
3251 goto err;
3252 }
3253
3254 cond_resched();
3255 spin_lock_irq(&ic->endio_wait.lock);
3256 remove_range_unlocked(ic, &range);
3257 #ifdef CONFIG_64BIT
3258 /* Paired with smp_load_acquire in dm_integrity_map_inline. */
3259 smp_store_release(&ic->sb->recalc_sector, cpu_to_le64(range.logical_sector + range.n_sectors));
3260 #else
3261 ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors);
3262 #endif
3263 goto next_chunk;
3264
3265 err:
3266 remove_range(ic, &range);
3267 goto free_ret;
3268
3269 unlock_ret:
3270 spin_unlock_irq(&ic->endio_wait.lock);
3271
3272 recalc_write_super(ic);
3273
3274 free_ret:
3275 kfree(recalc_buffer);
3276 kfree(recalc_tags);
3277 }
3278
3279 static void bitmap_block_work(struct work_struct *w)
3280 {
3281 struct bitmap_block_status *bbs = container_of(w, struct bitmap_block_status, work);
3282 struct dm_integrity_c *ic = bbs->ic;
3283 struct bio *bio;
3284 struct bio_list bio_queue;
3285 struct bio_list waiting;
3286
3287 bio_list_init(&waiting);
3288
3289 spin_lock(&bbs->bio_queue_lock);
3290 bio_queue = bbs->bio_queue;
3291 bio_list_init(&bbs->bio_queue);
3292 spin_unlock(&bbs->bio_queue_lock);
3293
3294 while ((bio = bio_list_pop(&bio_queue))) {
3295 struct dm_integrity_io *dio;
3296
3297 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
3298
3299 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
3300 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
3301 remove_range(ic, &dio->range);
3302 INIT_WORK(&dio->work, integrity_bio_wait);
3303 queue_work(ic->offload_wq, &dio->work);
3304 } else {
3305 block_bitmap_op(ic, ic->journal, dio->range.logical_sector,
3306 dio->range.n_sectors, BITMAP_OP_SET);
3307 bio_list_add(&waiting, bio);
3308 }
3309 }
3310
3311 if (bio_list_empty(&waiting))
3312 return;
3313
3314 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC,
3315 bbs->idx * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT),
3316 BITMAP_BLOCK_SIZE >> SECTOR_SHIFT, NULL);
3317
3318 while ((bio = bio_list_pop(&waiting))) {
3319 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
3320
3321 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
3322 dio->range.n_sectors, BITMAP_OP_SET);
3323
3324 remove_range(ic, &dio->range);
3325 INIT_WORK(&dio->work, integrity_bio_wait);
3326 queue_work(ic->offload_wq, &dio->work);
3327 }
3328
3329 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval);
3330 }
3331
3332 static void bitmap_flush_work(struct work_struct *work)
3333 {
3334 struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work);
3335 struct dm_integrity_range range;
3336 unsigned long limit;
3337 struct bio *bio;
3338
3339 dm_integrity_flush_buffers(ic, false);
3340
3341 range.logical_sector = 0;
3342 range.n_sectors = ic->provided_data_sectors;
3343
3344 spin_lock_irq(&ic->endio_wait.lock);
3345 add_new_range_and_wait(ic, &range);
3346 spin_unlock_irq(&ic->endio_wait.lock);
3347
3348 dm_integrity_flush_buffers(ic, true);
3349
3350 limit = ic->provided_data_sectors;
3351 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3352 limit = le64_to_cpu(ic->sb->recalc_sector)
3353 >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)
3354 << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit);
3355 }
3356 /*DEBUG_print("zeroing journal\n");*/
3357 block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR);
3358 block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR);
3359
3360 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3361 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3362
3363 spin_lock_irq(&ic->endio_wait.lock);
3364 remove_range_unlocked(ic, &range);
3365 while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) {
3366 bio_endio(bio);
3367 spin_unlock_irq(&ic->endio_wait.lock);
3368 spin_lock_irq(&ic->endio_wait.lock);
3369 }
3370 spin_unlock_irq(&ic->endio_wait.lock);
3371 }
3372
3373
3374 static void init_journal(struct dm_integrity_c *ic, unsigned int start_section,
3375 unsigned int n_sections, unsigned char commit_seq)
3376 {
3377 unsigned int i, j, n;
3378
3379 if (!n_sections)
3380 return;
3381
3382 for (n = 0; n < n_sections; n++) {
3383 i = start_section + n;
3384 wraparound_section(ic, &i);
3385 for (j = 0; j < ic->journal_section_sectors; j++) {
3386 struct journal_sector *js = access_journal(ic, i, j);
3387
3388 BUILD_BUG_ON(sizeof(js->sectors) != JOURNAL_SECTOR_DATA);
3389 memset(&js->sectors, 0, sizeof(js->sectors));
3390 js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq);
3391 }
3392 for (j = 0; j < ic->journal_section_entries; j++) {
3393 struct journal_entry *je = access_journal_entry(ic, i, j);
3394
3395 journal_entry_set_unused(je);
3396 }
3397 }
3398
3399 write_journal(ic, start_section, n_sections);
3400 }
3401
3402 static int find_commit_seq(struct dm_integrity_c *ic, unsigned int i, unsigned int j, commit_id_t id)
3403 {
3404 unsigned char k;
3405
3406 for (k = 0; k < N_COMMIT_IDS; k++) {
3407 if (dm_integrity_commit_id(ic, i, j, k) == id)
3408 return k;
3409 }
3410 dm_integrity_io_error(ic, "journal commit id", -EIO);
3411 return -EIO;
3412 }
3413
3414 static void replay_journal(struct dm_integrity_c *ic)
3415 {
3416 unsigned int i, j;
3417 bool used_commit_ids[N_COMMIT_IDS];
3418 unsigned int max_commit_id_sections[N_COMMIT_IDS];
3419 unsigned int write_start, write_sections;
3420 unsigned int continue_section;
3421 bool journal_empty;
3422 unsigned char unused, last_used, want_commit_seq;
3423
3424 if (ic->mode == 'R')
3425 return;
3426
3427 if (ic->journal_uptodate)
3428 return;
3429
3430 last_used = 0;
3431 write_start = 0;
3432
3433 if (!ic->just_formatted) {
3434 DEBUG_print("reading journal\n");
3435 rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL);
3436 if (ic->journal_io)
3437 DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal");
3438 if (ic->journal_io) {
3439 struct journal_completion crypt_comp;
3440
3441 crypt_comp.ic = ic;
3442 init_completion(&crypt_comp.comp);
3443 crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0);
3444 encrypt_journal(ic, false, 0, ic->journal_sections, &crypt_comp);
3445 wait_for_completion(&crypt_comp.comp);
3446 }
3447 DEBUG_bytes(lowmem_page_address(ic->journal[0].page), 64, "decrypted journal");
3448 }
3449
3450 if (dm_integrity_failed(ic))
3451 goto clear_journal;
3452
3453 journal_empty = true;
3454 memset(used_commit_ids, 0, sizeof(used_commit_ids));
3455 memset(max_commit_id_sections, 0, sizeof(max_commit_id_sections));
3456 for (i = 0; i < ic->journal_sections; i++) {
3457 for (j = 0; j < ic->journal_section_sectors; j++) {
3458 int k;
3459 struct journal_sector *js = access_journal(ic, i, j);
3460
3461 k = find_commit_seq(ic, i, j, js->commit_id);
3462 if (k < 0)
3463 goto clear_journal;
3464 used_commit_ids[k] = true;
3465 max_commit_id_sections[k] = i;
3466 }
3467 if (journal_empty) {
3468 for (j = 0; j < ic->journal_section_entries; j++) {
3469 struct journal_entry *je = access_journal_entry(ic, i, j);
3470
3471 if (!journal_entry_is_unused(je)) {
3472 journal_empty = false;
3473 break;
3474 }
3475 }
3476 }
3477 }
3478
3479 if (!used_commit_ids[N_COMMIT_IDS - 1]) {
3480 unused = N_COMMIT_IDS - 1;
3481 while (unused && !used_commit_ids[unused - 1])
3482 unused--;
3483 } else {
3484 for (unused = 0; unused < N_COMMIT_IDS; unused++)
3485 if (!used_commit_ids[unused])
3486 break;
3487 if (unused == N_COMMIT_IDS) {
3488 dm_integrity_io_error(ic, "journal commit ids", -EIO);
3489 goto clear_journal;
3490 }
3491 }
3492 DEBUG_print("first unused commit seq %d [%d,%d,%d,%d]\n",
3493 unused, used_commit_ids[0], used_commit_ids[1],
3494 used_commit_ids[2], used_commit_ids[3]);
3495
3496 last_used = prev_commit_seq(unused);
3497 want_commit_seq = prev_commit_seq(last_used);
3498
3499 if (!used_commit_ids[want_commit_seq] && used_commit_ids[prev_commit_seq(want_commit_seq)])
3500 journal_empty = true;
3501
3502 write_start = max_commit_id_sections[last_used] + 1;
3503 if (unlikely(write_start >= ic->journal_sections))
3504 want_commit_seq = next_commit_seq(want_commit_seq);
3505 wraparound_section(ic, &write_start);
3506
3507 i = write_start;
3508 for (write_sections = 0; write_sections < ic->journal_sections; write_sections++) {
3509 for (j = 0; j < ic->journal_section_sectors; j++) {
3510 struct journal_sector *js = access_journal(ic, i, j);
3511
3512 if (js->commit_id != dm_integrity_commit_id(ic, i, j, want_commit_seq)) {
3513 /*
3514 * This could be caused by crash during writing.
3515 * We won't replay the inconsistent part of the
3516 * journal.
3517 */
3518 DEBUG_print("commit id mismatch at position (%u, %u): %d != %d\n",
3519 i, j, find_commit_seq(ic, i, j, js->commit_id), want_commit_seq);
3520 goto brk;
3521 }
3522 }
3523 i++;
3524 if (unlikely(i >= ic->journal_sections))
3525 want_commit_seq = next_commit_seq(want_commit_seq);
3526 wraparound_section(ic, &i);
3527 }
3528 brk:
3529
3530 if (!journal_empty) {
3531 DEBUG_print("replaying %u sections, starting at %u, commit seq %d\n",
3532 write_sections, write_start, want_commit_seq);
3533 do_journal_write(ic, write_start, write_sections, true);
3534 }
3535
3536 if (write_sections == ic->journal_sections && (ic->mode == 'J' || journal_empty)) {
3537 continue_section = write_start;
3538 ic->commit_seq = want_commit_seq;
3539 DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq);
3540 } else {
3541 unsigned int s;
3542 unsigned char erase_seq;
3543
3544 clear_journal:
3545 DEBUG_print("clearing journal\n");
3546
3547 erase_seq = prev_commit_seq(prev_commit_seq(last_used));
3548 s = write_start;
3549 init_journal(ic, s, 1, erase_seq);
3550 s++;
3551 wraparound_section(ic, &s);
3552 if (ic->journal_sections >= 2) {
3553 init_journal(ic, s, ic->journal_sections - 2, erase_seq);
3554 s += ic->journal_sections - 2;
3555 wraparound_section(ic, &s);
3556 init_journal(ic, s, 1, erase_seq);
3557 }
3558
3559 continue_section = 0;
3560 ic->commit_seq = next_commit_seq(erase_seq);
3561 }
3562
3563 ic->committed_section = continue_section;
3564 ic->n_committed_sections = 0;
3565
3566 ic->uncommitted_section = continue_section;
3567 ic->n_uncommitted_sections = 0;
3568
3569 ic->free_section = continue_section;
3570 ic->free_section_entry = 0;
3571 ic->free_sectors = ic->journal_entries;
3572
3573 ic->journal_tree_root = RB_ROOT;
3574 for (i = 0; i < ic->journal_entries; i++)
3575 init_journal_node(&ic->journal_tree[i]);
3576 }
3577
3578 static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic)
3579 {
3580 DEBUG_print("%s\n", __func__);
3581
3582 if (ic->mode == 'B') {
3583 ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1;
3584 ic->synchronous_mode = 1;
3585
3586 cancel_delayed_work_sync(&ic->bitmap_flush_work);
3587 queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0);
3588 flush_workqueue(ic->commit_wq);
3589 }
3590 }
3591
3592 static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, void *x)
3593 {
3594 struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier);
3595
3596 DEBUG_print("%s\n", __func__);
3597
3598 dm_integrity_enter_synchronous_mode(ic);
3599
3600 return NOTIFY_DONE;
3601 }
3602
3603 static void dm_integrity_postsuspend(struct dm_target *ti)
3604 {
3605 struct dm_integrity_c *ic = ti->private;
3606 int r;
3607
3608 WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier));
3609
3610 del_timer_sync(&ic->autocommit_timer);
3611
3612 if (ic->recalc_wq)
3613 drain_workqueue(ic->recalc_wq);
3614
3615 if (ic->mode == 'B')
3616 cancel_delayed_work_sync(&ic->bitmap_flush_work);
3617
3618 queue_work(ic->commit_wq, &ic->commit_work);
3619 drain_workqueue(ic->commit_wq);
3620
3621 if (ic->mode == 'J') {
3622 queue_work(ic->writer_wq, &ic->writer_work);
3623 drain_workqueue(ic->writer_wq);
3624 dm_integrity_flush_buffers(ic, true);
3625 if (ic->wrote_to_journal) {
3626 init_journal(ic, ic->free_section,
3627 ic->journal_sections - ic->free_section, ic->commit_seq);
3628 if (ic->free_section) {
3629 init_journal(ic, 0, ic->free_section,
3630 next_commit_seq(ic->commit_seq));
3631 }
3632 }
3633 }
3634
3635 if (ic->mode == 'B') {
3636 dm_integrity_flush_buffers(ic, true);
3637 #if 1
3638 /* set to 0 to test bitmap replay code */
3639 init_journal(ic, 0, ic->journal_sections, 0);
3640 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3641 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3642 if (unlikely(r))
3643 dm_integrity_io_error(ic, "writing superblock", r);
3644 #endif
3645 }
3646
3647 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
3648
3649 ic->journal_uptodate = true;
3650 }
3651
3652 static void dm_integrity_resume(struct dm_target *ti)
3653 {
3654 struct dm_integrity_c *ic = ti->private;
3655 __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors);
3656 int r;
3657
3658 DEBUG_print("resume\n");
3659
3660 ic->wrote_to_journal = false;
3661
3662 if (ic->provided_data_sectors != old_provided_data_sectors) {
3663 if (ic->provided_data_sectors > old_provided_data_sectors &&
3664 ic->mode == 'B' &&
3665 ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) {
3666 rw_journal_sectors(ic, REQ_OP_READ, 0,
3667 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3668 block_bitmap_op(ic, ic->journal, old_provided_data_sectors,
3669 ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET);
3670 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3671 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3672 }
3673
3674 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
3675 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3676 if (unlikely(r))
3677 dm_integrity_io_error(ic, "writing superblock", r);
3678 }
3679
3680 if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) {
3681 DEBUG_print("resume dirty_bitmap\n");
3682 rw_journal_sectors(ic, REQ_OP_READ, 0,
3683 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3684 if (ic->mode == 'B') {
3685 if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
3686 !ic->reset_recalculate_flag) {
3687 block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal);
3688 block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal);
3689 if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors,
3690 BITMAP_OP_TEST_ALL_CLEAR)) {
3691 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3692 ic->sb->recalc_sector = cpu_to_le64(0);
3693 }
3694 } else {
3695 DEBUG_print("non-matching blocks_per_bitmap_bit: %u, %u\n",
3696 ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit);
3697 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3698 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3699 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3700 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET);
3701 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3702 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3703 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3704 ic->sb->recalc_sector = cpu_to_le64(0);
3705 }
3706 } else {
3707 if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit &&
3708 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) ||
3709 ic->reset_recalculate_flag) {
3710 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3711 ic->sb->recalc_sector = cpu_to_le64(0);
3712 }
3713 init_journal(ic, 0, ic->journal_sections, 0);
3714 replay_journal(ic);
3715 ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3716 }
3717 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3718 if (unlikely(r))
3719 dm_integrity_io_error(ic, "writing superblock", r);
3720 } else {
3721 replay_journal(ic);
3722 if (ic->reset_recalculate_flag) {
3723 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
3724 ic->sb->recalc_sector = cpu_to_le64(0);
3725 }
3726 if (ic->mode == 'B') {
3727 ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP);
3728 ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit;
3729 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
3730 if (unlikely(r))
3731 dm_integrity_io_error(ic, "writing superblock", r);
3732
3733 block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3734 block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3735 block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR);
3736 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
3737 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) {
3738 block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector),
3739 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3740 block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector),
3741 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3742 block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector),
3743 ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET);
3744 }
3745 rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0,
3746 ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL);
3747 }
3748 }
3749
3750 DEBUG_print("testing recalc: %x\n", ic->sb->flags);
3751 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
3752 __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector);
3753
3754 DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors);
3755 if (recalc_pos < ic->provided_data_sectors) {
3756 queue_work(ic->recalc_wq, &ic->recalc_work);
3757 } else if (recalc_pos > ic->provided_data_sectors) {
3758 ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors);
3759 recalc_write_super(ic);
3760 }
3761 }
3762
3763 ic->reboot_notifier.notifier_call = dm_integrity_reboot;
3764 ic->reboot_notifier.next = NULL;
3765 ic->reboot_notifier.priority = INT_MAX - 1; /* be notified after md and before hardware drivers */
3766 WARN_ON(register_reboot_notifier(&ic->reboot_notifier));
3767
3768 #if 0
3769 /* set to 1 to stress test synchronous mode */
3770 dm_integrity_enter_synchronous_mode(ic);
3771 #endif
3772 }
3773
3774 static void dm_integrity_status(struct dm_target *ti, status_type_t type,
3775 unsigned int status_flags, char *result, unsigned int maxlen)
3776 {
3777 struct dm_integrity_c *ic = ti->private;
3778 unsigned int arg_count;
3779 size_t sz = 0;
3780
3781 switch (type) {
3782 case STATUSTYPE_INFO:
3783 DMEMIT("%llu %llu",
3784 (unsigned long long)atomic64_read(&ic->number_of_mismatches),
3785 ic->provided_data_sectors);
3786 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
3787 DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector));
3788 else
3789 DMEMIT(" -");
3790 break;
3791
3792 case STATUSTYPE_TABLE: {
3793 __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100;
3794
3795 watermark_percentage += ic->journal_entries / 2;
3796 do_div(watermark_percentage, ic->journal_entries);
3797 arg_count = 3;
3798 arg_count += !!ic->meta_dev;
3799 arg_count += ic->sectors_per_block != 1;
3800 arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING));
3801 arg_count += ic->reset_recalculate_flag;
3802 arg_count += ic->discard;
3803 arg_count += ic->mode == 'J';
3804 arg_count += ic->mode == 'J';
3805 arg_count += ic->mode == 'B';
3806 arg_count += ic->mode == 'B';
3807 arg_count += !!ic->internal_hash_alg.alg_string;
3808 arg_count += !!ic->journal_crypt_alg.alg_string;
3809 arg_count += !!ic->journal_mac_alg.alg_string;
3810 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0;
3811 arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0;
3812 arg_count += ic->legacy_recalculate;
3813 DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start,
3814 ic->tag_size, ic->mode, arg_count);
3815 if (ic->meta_dev)
3816 DMEMIT(" meta_device:%s", ic->meta_dev->name);
3817 if (ic->sectors_per_block != 1)
3818 DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT);
3819 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))
3820 DMEMIT(" recalculate");
3821 if (ic->reset_recalculate_flag)
3822 DMEMIT(" reset_recalculate");
3823 if (ic->discard)
3824 DMEMIT(" allow_discards");
3825 DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS);
3826 DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors);
3827 DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors);
3828 if (ic->mode == 'J') {
3829 DMEMIT(" journal_watermark:%u", (unsigned int)watermark_percentage);
3830 DMEMIT(" commit_time:%u", ic->autocommit_msec);
3831 }
3832 if (ic->mode == 'B') {
3833 DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit);
3834 DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval));
3835 }
3836 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0)
3837 DMEMIT(" fix_padding");
3838 if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0)
3839 DMEMIT(" fix_hmac");
3840 if (ic->legacy_recalculate)
3841 DMEMIT(" legacy_recalculate");
3842
3843 #define EMIT_ALG(a, n) \
3844 do { \
3845 if (ic->a.alg_string) { \
3846 DMEMIT(" %s:%s", n, ic->a.alg_string); \
3847 if (ic->a.key_string) \
3848 DMEMIT(":%s", ic->a.key_string);\
3849 } \
3850 } while (0)
3851 EMIT_ALG(internal_hash_alg, "internal_hash");
3852 EMIT_ALG(journal_crypt_alg, "journal_crypt");
3853 EMIT_ALG(journal_mac_alg, "journal_mac");
3854 break;
3855 }
3856 case STATUSTYPE_IMA:
3857 DMEMIT_TARGET_NAME_VERSION(ti->type);
3858 DMEMIT(",dev_name=%s,start=%llu,tag_size=%u,mode=%c",
3859 ic->dev->name, ic->start, ic->tag_size, ic->mode);
3860
3861 if (ic->meta_dev)
3862 DMEMIT(",meta_device=%s", ic->meta_dev->name);
3863 if (ic->sectors_per_block != 1)
3864 DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT);
3865
3866 DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ?
3867 'y' : 'n');
3868 DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n');
3869 DMEMIT(",fix_padding=%c",
3870 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n');
3871 DMEMIT(",fix_hmac=%c",
3872 ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n');
3873 DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n');
3874
3875 DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS);
3876 DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors);
3877 DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors);
3878 DMEMIT(";");
3879 break;
3880 }
3881 }
3882
3883 static int dm_integrity_iterate_devices(struct dm_target *ti,
3884 iterate_devices_callout_fn fn, void *data)
3885 {
3886 struct dm_integrity_c *ic = ti->private;
3887
3888 if (!ic->meta_dev)
3889 return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data);
3890 else
3891 return fn(ti, ic->dev, 0, ti->len, data);
3892 }
3893
3894 static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits)
3895 {
3896 struct dm_integrity_c *ic = ti->private;
3897
3898 if (ic->sectors_per_block > 1) {
3899 limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
3900 limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT;
3901 limits->io_min = ic->sectors_per_block << SECTOR_SHIFT;
3902 limits->dma_alignment = limits->logical_block_size - 1;
3903 limits->discard_granularity = ic->sectors_per_block << SECTOR_SHIFT;
3904 }
3905
3906 if (!ic->internal_hash) {
3907 struct blk_integrity *bi = &limits->integrity;
3908
3909 memset(bi, 0, sizeof(*bi));
3910 bi->tuple_size = ic->tag_size;
3911 bi->tag_size = bi->tuple_size;
3912 bi->interval_exp =
3913 ic->sb->log2_sectors_per_block + SECTOR_SHIFT;
3914 }
3915
3916 limits->max_integrity_segments = USHRT_MAX;
3917 }
3918
3919 static void calculate_journal_section_size(struct dm_integrity_c *ic)
3920 {
3921 unsigned int sector_space = JOURNAL_SECTOR_DATA;
3922
3923 ic->journal_sections = le32_to_cpu(ic->sb->journal_sections);
3924 ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size,
3925 JOURNAL_ENTRY_ROUNDUP);
3926
3927 if (ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC))
3928 sector_space -= JOURNAL_MAC_PER_SECTOR;
3929 ic->journal_entries_per_sector = sector_space / ic->journal_entry_size;
3930 ic->journal_section_entries = ic->journal_entries_per_sector * JOURNAL_BLOCK_SECTORS;
3931 ic->journal_section_sectors = (ic->journal_section_entries << ic->sb->log2_sectors_per_block) + JOURNAL_BLOCK_SECTORS;
3932 ic->journal_entries = ic->journal_section_entries * ic->journal_sections;
3933 }
3934
3935 static int calculate_device_limits(struct dm_integrity_c *ic)
3936 {
3937 __u64 initial_sectors;
3938
3939 calculate_journal_section_size(ic);
3940 initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections;
3941 if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX)
3942 return -EINVAL;
3943 ic->initial_sectors = initial_sectors;
3944
3945 if (ic->mode == 'I') {
3946 if (ic->initial_sectors + ic->provided_data_sectors > ic->meta_device_sectors)
3947 return -EINVAL;
3948 } else if (!ic->meta_dev) {
3949 sector_t last_sector, last_area, last_offset;
3950
3951 /* we have to maintain excessive padding for compatibility with existing volumes */
3952 __u64 metadata_run_padding =
3953 ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ?
3954 (__u64)(METADATA_PADDING_SECTORS << SECTOR_SHIFT) :
3955 (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS);
3956
3957 ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block),
3958 metadata_run_padding) >> SECTOR_SHIFT;
3959 if (!(ic->metadata_run & (ic->metadata_run - 1)))
3960 ic->log2_metadata_run = __ffs(ic->metadata_run);
3961 else
3962 ic->log2_metadata_run = -1;
3963
3964 get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset);
3965 last_sector = get_data_sector(ic, last_area, last_offset);
3966 if (last_sector < ic->start || last_sector >= ic->meta_device_sectors)
3967 return -EINVAL;
3968 } else {
3969 __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size;
3970
3971 meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1))
3972 >> (ic->log2_buffer_sectors + SECTOR_SHIFT);
3973 meta_size <<= ic->log2_buffer_sectors;
3974 if (ic->initial_sectors + meta_size < ic->initial_sectors ||
3975 ic->initial_sectors + meta_size > ic->meta_device_sectors)
3976 return -EINVAL;
3977 ic->metadata_run = 1;
3978 ic->log2_metadata_run = 0;
3979 }
3980
3981 return 0;
3982 }
3983
3984 static void get_provided_data_sectors(struct dm_integrity_c *ic)
3985 {
3986 if (!ic->meta_dev) {
3987 int test_bit;
3988
3989 ic->provided_data_sectors = 0;
3990 for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) {
3991 __u64 prev_data_sectors = ic->provided_data_sectors;
3992
3993 ic->provided_data_sectors |= (sector_t)1 << test_bit;
3994 if (calculate_device_limits(ic))
3995 ic->provided_data_sectors = prev_data_sectors;
3996 }
3997 } else {
3998 ic->provided_data_sectors = ic->data_device_sectors;
3999 ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1);
4000 }
4001 }
4002
4003 static int initialize_superblock(struct dm_integrity_c *ic,
4004 unsigned int journal_sectors, unsigned int interleave_sectors)
4005 {
4006 unsigned int journal_sections;
4007 int test_bit;
4008
4009 memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT);
4010 memcpy(ic->sb->magic, SB_MAGIC, 8);
4011 if (ic->mode == 'I')
4012 ic->sb->flags |= cpu_to_le32(SB_FLAG_INLINE);
4013 ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size);
4014 ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block);
4015 if (ic->journal_mac_alg.alg_string)
4016 ic->sb->flags |= cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC);
4017
4018 calculate_journal_section_size(ic);
4019 journal_sections = journal_sectors / ic->journal_section_sectors;
4020 if (!journal_sections)
4021 journal_sections = 1;
4022 if (ic->mode == 'I')
4023 journal_sections = 0;
4024
4025 if (ic->fix_hmac && (ic->internal_hash_alg.alg_string || ic->journal_mac_alg.alg_string)) {
4026 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_HMAC);
4027 get_random_bytes(ic->sb->salt, SALT_SIZE);
4028 }
4029
4030 if (!ic->meta_dev) {
4031 if (ic->fix_padding)
4032 ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING);
4033 ic->sb->journal_sections = cpu_to_le32(journal_sections);
4034 if (!interleave_sectors)
4035 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
4036 ic->sb->log2_interleave_sectors = __fls(interleave_sectors);
4037 ic->sb->log2_interleave_sectors = max_t(__u8, MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
4038 ic->sb->log2_interleave_sectors = min_t(__u8, MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors);
4039
4040 get_provided_data_sectors(ic);
4041 if (!ic->provided_data_sectors)
4042 return -EINVAL;
4043 } else {
4044 ic->sb->log2_interleave_sectors = 0;
4045
4046 get_provided_data_sectors(ic);
4047 if (!ic->provided_data_sectors)
4048 return -EINVAL;
4049
4050 try_smaller_buffer:
4051 ic->sb->journal_sections = cpu_to_le32(0);
4052 for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) {
4053 __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections);
4054 __u32 test_journal_sections = prev_journal_sections | (1U << test_bit);
4055
4056 if (test_journal_sections > journal_sections)
4057 continue;
4058 ic->sb->journal_sections = cpu_to_le32(test_journal_sections);
4059 if (calculate_device_limits(ic))
4060 ic->sb->journal_sections = cpu_to_le32(prev_journal_sections);
4061
4062 }
4063 if (!le32_to_cpu(ic->sb->journal_sections)) {
4064 if (ic->log2_buffer_sectors > 3) {
4065 ic->log2_buffer_sectors--;
4066 goto try_smaller_buffer;
4067 }
4068 return -EINVAL;
4069 }
4070 }
4071
4072 ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors);
4073
4074 sb_set_version(ic);
4075
4076 return 0;
4077 }
4078
4079 static void dm_integrity_free_page_list(struct page_list *pl)
4080 {
4081 unsigned int i;
4082
4083 if (!pl)
4084 return;
4085 for (i = 0; pl[i].page; i++)
4086 __free_page(pl[i].page);
4087 kvfree(pl);
4088 }
4089
4090 static struct page_list *dm_integrity_alloc_page_list(unsigned int n_pages)
4091 {
4092 struct page_list *pl;
4093 unsigned int i;
4094
4095 pl = kvmalloc_array(n_pages + 1, sizeof(struct page_list), GFP_KERNEL | __GFP_ZERO);
4096 if (!pl)
4097 return NULL;
4098
4099 for (i = 0; i < n_pages; i++) {
4100 pl[i].page = alloc_page(GFP_KERNEL);
4101 if (!pl[i].page) {
4102 dm_integrity_free_page_list(pl);
4103 return NULL;
4104 }
4105 if (i)
4106 pl[i - 1].next = &pl[i];
4107 }
4108 pl[i].page = NULL;
4109 pl[i].next = NULL;
4110
4111 return pl;
4112 }
4113
4114 static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl)
4115 {
4116 unsigned int i;
4117
4118 for (i = 0; i < ic->journal_sections; i++)
4119 kvfree(sl[i]);
4120 kvfree(sl);
4121 }
4122
4123 static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic,
4124 struct page_list *pl)
4125 {
4126 struct scatterlist **sl;
4127 unsigned int i;
4128
4129 sl = kvmalloc_array(ic->journal_sections,
4130 sizeof(struct scatterlist *),
4131 GFP_KERNEL | __GFP_ZERO);
4132 if (!sl)
4133 return NULL;
4134
4135 for (i = 0; i < ic->journal_sections; i++) {
4136 struct scatterlist *s;
4137 unsigned int start_index, start_offset;
4138 unsigned int end_index, end_offset;
4139 unsigned int n_pages;
4140 unsigned int idx;
4141
4142 page_list_location(ic, i, 0, &start_index, &start_offset);
4143 page_list_location(ic, i, ic->journal_section_sectors - 1,
4144 &end_index, &end_offset);
4145
4146 n_pages = (end_index - start_index + 1);
4147
4148 s = kvmalloc_array(n_pages, sizeof(struct scatterlist),
4149 GFP_KERNEL);
4150 if (!s) {
4151 dm_integrity_free_journal_scatterlist(ic, sl);
4152 return NULL;
4153 }
4154
4155 sg_init_table(s, n_pages);
4156 for (idx = start_index; idx <= end_index; idx++) {
4157 char *va = lowmem_page_address(pl[idx].page);
4158 unsigned int start = 0, end = PAGE_SIZE;
4159
4160 if (idx == start_index)
4161 start = start_offset;
4162 if (idx == end_index)
4163 end = end_offset + (1 << SECTOR_SHIFT);
4164 sg_set_buf(&s[idx - start_index], va + start, end - start);
4165 }
4166
4167 sl[i] = s;
4168 }
4169
4170 return sl;
4171 }
4172
4173 static void free_alg(struct alg_spec *a)
4174 {
4175 kfree_sensitive(a->alg_string);
4176 kfree_sensitive(a->key);
4177 memset(a, 0, sizeof(*a));
4178 }
4179
4180 static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval)
4181 {
4182 char *k;
4183
4184 free_alg(a);
4185
4186 a->alg_string = kstrdup(strchr(arg, ':') + 1, GFP_KERNEL);
4187 if (!a->alg_string)
4188 goto nomem;
4189
4190 k = strchr(a->alg_string, ':');
4191 if (k) {
4192 *k = 0;
4193 a->key_string = k + 1;
4194 if (strlen(a->key_string) & 1)
4195 goto inval;
4196
4197 a->key_size = strlen(a->key_string) / 2;
4198 a->key = kmalloc(a->key_size, GFP_KERNEL);
4199 if (!a->key)
4200 goto nomem;
4201 if (hex2bin(a->key, a->key_string, a->key_size))
4202 goto inval;
4203 }
4204
4205 return 0;
4206 inval:
4207 *error = error_inval;
4208 return -EINVAL;
4209 nomem:
4210 *error = "Out of memory for an argument";
4211 return -ENOMEM;
4212 }
4213
4214 static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error,
4215 char *error_alg, char *error_key)
4216 {
4217 int r;
4218
4219 if (a->alg_string) {
4220 *hash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4221 if (IS_ERR(*hash)) {
4222 *error = error_alg;
4223 r = PTR_ERR(*hash);
4224 *hash = NULL;
4225 return r;
4226 }
4227
4228 if (a->key) {
4229 r = crypto_shash_setkey(*hash, a->key, a->key_size);
4230 if (r) {
4231 *error = error_key;
4232 return r;
4233 }
4234 } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) {
4235 *error = error_key;
4236 return -ENOKEY;
4237 }
4238 }
4239
4240 return 0;
4241 }
4242
4243 static int create_journal(struct dm_integrity_c *ic, char **error)
4244 {
4245 int r = 0;
4246 unsigned int i;
4247 __u64 journal_pages, journal_desc_size, journal_tree_size;
4248 unsigned char *crypt_data = NULL, *crypt_iv = NULL;
4249 struct skcipher_request *req = NULL;
4250
4251 ic->commit_ids[0] = cpu_to_le64(0x1111111111111111ULL);
4252 ic->commit_ids[1] = cpu_to_le64(0x2222222222222222ULL);
4253 ic->commit_ids[2] = cpu_to_le64(0x3333333333333333ULL);
4254 ic->commit_ids[3] = cpu_to_le64(0x4444444444444444ULL);
4255
4256 journal_pages = roundup((__u64)ic->journal_sections * ic->journal_section_sectors,
4257 PAGE_SIZE >> SECTOR_SHIFT) >> (PAGE_SHIFT - SECTOR_SHIFT);
4258 journal_desc_size = journal_pages * sizeof(struct page_list);
4259 if (journal_pages >= totalram_pages() - totalhigh_pages() || journal_desc_size > ULONG_MAX) {
4260 *error = "Journal doesn't fit into memory";
4261 r = -ENOMEM;
4262 goto bad;
4263 }
4264 ic->journal_pages = journal_pages;
4265
4266 ic->journal = dm_integrity_alloc_page_list(ic->journal_pages);
4267 if (!ic->journal) {
4268 *error = "Could not allocate memory for journal";
4269 r = -ENOMEM;
4270 goto bad;
4271 }
4272 if (ic->journal_crypt_alg.alg_string) {
4273 unsigned int ivsize, blocksize;
4274 struct journal_completion comp;
4275
4276 comp.ic = ic;
4277 ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY);
4278 if (IS_ERR(ic->journal_crypt)) {
4279 *error = "Invalid journal cipher";
4280 r = PTR_ERR(ic->journal_crypt);
4281 ic->journal_crypt = NULL;
4282 goto bad;
4283 }
4284 ivsize = crypto_skcipher_ivsize(ic->journal_crypt);
4285 blocksize = crypto_skcipher_blocksize(ic->journal_crypt);
4286
4287 if (ic->journal_crypt_alg.key) {
4288 r = crypto_skcipher_setkey(ic->journal_crypt, ic->journal_crypt_alg.key,
4289 ic->journal_crypt_alg.key_size);
4290 if (r) {
4291 *error = "Error setting encryption key";
4292 goto bad;
4293 }
4294 }
4295 DEBUG_print("cipher %s, block size %u iv size %u\n",
4296 ic->journal_crypt_alg.alg_string, blocksize, ivsize);
4297
4298 ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages);
4299 if (!ic->journal_io) {
4300 *error = "Could not allocate memory for journal io";
4301 r = -ENOMEM;
4302 goto bad;
4303 }
4304
4305 if (blocksize == 1) {
4306 struct scatterlist *sg;
4307
4308 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
4309 if (!req) {
4310 *error = "Could not allocate crypt request";
4311 r = -ENOMEM;
4312 goto bad;
4313 }
4314
4315 crypt_iv = kzalloc(ivsize, GFP_KERNEL);
4316 if (!crypt_iv) {
4317 *error = "Could not allocate iv";
4318 r = -ENOMEM;
4319 goto bad;
4320 }
4321
4322 ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages);
4323 if (!ic->journal_xor) {
4324 *error = "Could not allocate memory for journal xor";
4325 r = -ENOMEM;
4326 goto bad;
4327 }
4328
4329 sg = kvmalloc_array(ic->journal_pages + 1,
4330 sizeof(struct scatterlist),
4331 GFP_KERNEL);
4332 if (!sg) {
4333 *error = "Unable to allocate sg list";
4334 r = -ENOMEM;
4335 goto bad;
4336 }
4337 sg_init_table(sg, ic->journal_pages + 1);
4338 for (i = 0; i < ic->journal_pages; i++) {
4339 char *va = lowmem_page_address(ic->journal_xor[i].page);
4340
4341 clear_page(va);
4342 sg_set_buf(&sg[i], va, PAGE_SIZE);
4343 }
4344 sg_set_buf(&sg[i], &ic->commit_ids, sizeof(ic->commit_ids));
4345
4346 skcipher_request_set_crypt(req, sg, sg,
4347 PAGE_SIZE * ic->journal_pages + sizeof(ic->commit_ids), crypt_iv);
4348 init_completion(&comp.comp);
4349 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
4350 if (do_crypt(true, req, &comp))
4351 wait_for_completion(&comp.comp);
4352 kvfree(sg);
4353 r = dm_integrity_failed(ic);
4354 if (r) {
4355 *error = "Unable to encrypt journal";
4356 goto bad;
4357 }
4358 DEBUG_bytes(lowmem_page_address(ic->journal_xor[0].page), 64, "xor data");
4359
4360 crypto_free_skcipher(ic->journal_crypt);
4361 ic->journal_crypt = NULL;
4362 } else {
4363 unsigned int crypt_len = roundup(ivsize, blocksize);
4364
4365 req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
4366 if (!req) {
4367 *error = "Could not allocate crypt request";
4368 r = -ENOMEM;
4369 goto bad;
4370 }
4371
4372 crypt_iv = kmalloc(ivsize, GFP_KERNEL);
4373 if (!crypt_iv) {
4374 *error = "Could not allocate iv";
4375 r = -ENOMEM;
4376 goto bad;
4377 }
4378
4379 crypt_data = kmalloc(crypt_len, GFP_KERNEL);
4380 if (!crypt_data) {
4381 *error = "Unable to allocate crypt data";
4382 r = -ENOMEM;
4383 goto bad;
4384 }
4385
4386 ic->journal_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal);
4387 if (!ic->journal_scatterlist) {
4388 *error = "Unable to allocate sg list";
4389 r = -ENOMEM;
4390 goto bad;
4391 }
4392 ic->journal_io_scatterlist = dm_integrity_alloc_journal_scatterlist(ic, ic->journal_io);
4393 if (!ic->journal_io_scatterlist) {
4394 *error = "Unable to allocate sg list";
4395 r = -ENOMEM;
4396 goto bad;
4397 }
4398 ic->sk_requests = kvmalloc_array(ic->journal_sections,
4399 sizeof(struct skcipher_request *),
4400 GFP_KERNEL | __GFP_ZERO);
4401 if (!ic->sk_requests) {
4402 *error = "Unable to allocate sk requests";
4403 r = -ENOMEM;
4404 goto bad;
4405 }
4406 for (i = 0; i < ic->journal_sections; i++) {
4407 struct scatterlist sg;
4408 struct skcipher_request *section_req;
4409 __le32 section_le = cpu_to_le32(i);
4410
4411 memset(crypt_iv, 0x00, ivsize);
4412 memset(crypt_data, 0x00, crypt_len);
4413 memcpy(crypt_data, &section_le, min_t(size_t, crypt_len, sizeof(section_le)));
4414
4415 sg_init_one(&sg, crypt_data, crypt_len);
4416 skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv);
4417 init_completion(&comp.comp);
4418 comp.in_flight = (atomic_t)ATOMIC_INIT(1);
4419 if (do_crypt(true, req, &comp))
4420 wait_for_completion(&comp.comp);
4421
4422 r = dm_integrity_failed(ic);
4423 if (r) {
4424 *error = "Unable to generate iv";
4425 goto bad;
4426 }
4427
4428 section_req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL);
4429 if (!section_req) {
4430 *error = "Unable to allocate crypt request";
4431 r = -ENOMEM;
4432 goto bad;
4433 }
4434 section_req->iv = kmalloc_array(ivsize, 2,
4435 GFP_KERNEL);
4436 if (!section_req->iv) {
4437 skcipher_request_free(section_req);
4438 *error = "Unable to allocate iv";
4439 r = -ENOMEM;
4440 goto bad;
4441 }
4442 memcpy(section_req->iv + ivsize, crypt_data, ivsize);
4443 section_req->cryptlen = (size_t)ic->journal_section_sectors << SECTOR_SHIFT;
4444 ic->sk_requests[i] = section_req;
4445 DEBUG_bytes(crypt_data, ivsize, "iv(%u)", i);
4446 }
4447 }
4448 }
4449
4450 for (i = 0; i < N_COMMIT_IDS; i++) {
4451 unsigned int j;
4452
4453 retest_commit_id:
4454 for (j = 0; j < i; j++) {
4455 if (ic->commit_ids[j] == ic->commit_ids[i]) {
4456 ic->commit_ids[i] = cpu_to_le64(le64_to_cpu(ic->commit_ids[i]) + 1);
4457 goto retest_commit_id;
4458 }
4459 }
4460 DEBUG_print("commit id %u: %016llx\n", i, ic->commit_ids[i]);
4461 }
4462
4463 journal_tree_size = (__u64)ic->journal_entries * sizeof(struct journal_node);
4464 if (journal_tree_size > ULONG_MAX) {
4465 *error = "Journal doesn't fit into memory";
4466 r = -ENOMEM;
4467 goto bad;
4468 }
4469 ic->journal_tree = kvmalloc(journal_tree_size, GFP_KERNEL);
4470 if (!ic->journal_tree) {
4471 *error = "Could not allocate memory for journal tree";
4472 r = -ENOMEM;
4473 }
4474 bad:
4475 kfree(crypt_data);
4476 kfree(crypt_iv);
4477 skcipher_request_free(req);
4478
4479 return r;
4480 }
4481
4482 /*
4483 * Construct a integrity mapping
4484 *
4485 * Arguments:
4486 * device
4487 * offset from the start of the device
4488 * tag size
4489 * D - direct writes, J - journal writes, B - bitmap mode, R - recovery mode
4490 * number of optional arguments
4491 * optional arguments:
4492 * journal_sectors
4493 * interleave_sectors
4494 * buffer_sectors
4495 * journal_watermark
4496 * commit_time
4497 * meta_device
4498 * block_size
4499 * sectors_per_bit
4500 * bitmap_flush_interval
4501 * internal_hash
4502 * journal_crypt
4503 * journal_mac
4504 * recalculate
4505 */
4506 static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
4507 {
4508 struct dm_integrity_c *ic;
4509 char dummy;
4510 int r;
4511 unsigned int extra_args;
4512 struct dm_arg_set as;
4513 static const struct dm_arg _args[] = {
4514 {0, 18, "Invalid number of feature args"},
4515 };
4516 unsigned int journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec;
4517 bool should_write_sb;
4518 __u64 threshold;
4519 unsigned long long start;
4520 __s8 log2_sectors_per_bitmap_bit = -1;
4521 __s8 log2_blocks_per_bitmap_bit;
4522 __u64 bits_in_journal;
4523 __u64 n_bitmap_bits;
4524
4525 #define DIRECT_ARGUMENTS 4
4526
4527 if (argc <= DIRECT_ARGUMENTS) {
4528 ti->error = "Invalid argument count";
4529 return -EINVAL;
4530 }
4531
4532 ic = kzalloc(sizeof(struct dm_integrity_c), GFP_KERNEL);
4533 if (!ic) {
4534 ti->error = "Cannot allocate integrity context";
4535 return -ENOMEM;
4536 }
4537 ti->private = ic;
4538 ti->per_io_data_size = sizeof(struct dm_integrity_io);
4539 ic->ti = ti;
4540
4541 ic->in_progress = RB_ROOT;
4542 INIT_LIST_HEAD(&ic->wait_list);
4543 init_waitqueue_head(&ic->endio_wait);
4544 bio_list_init(&ic->flush_bio_list);
4545 init_waitqueue_head(&ic->copy_to_journal_wait);
4546 init_completion(&ic->crypto_backoff);
4547 atomic64_set(&ic->number_of_mismatches, 0);
4548 ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL;
4549
4550 r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev);
4551 if (r) {
4552 ti->error = "Device lookup failed";
4553 goto bad;
4554 }
4555
4556 if (sscanf(argv[1], "%llu%c", &start, &dummy) != 1 || start != (sector_t)start) {
4557 ti->error = "Invalid starting offset";
4558 r = -EINVAL;
4559 goto bad;
4560 }
4561 ic->start = start;
4562
4563 if (strcmp(argv[2], "-")) {
4564 if (sscanf(argv[2], "%u%c", &ic->tag_size, &dummy) != 1 || !ic->tag_size) {
4565 ti->error = "Invalid tag size";
4566 r = -EINVAL;
4567 goto bad;
4568 }
4569 }
4570
4571 if (!strcmp(argv[3], "J") || !strcmp(argv[3], "B") ||
4572 !strcmp(argv[3], "D") || !strcmp(argv[3], "R") ||
4573 !strcmp(argv[3], "I")) {
4574 ic->mode = argv[3][0];
4575 } else {
4576 ti->error = "Invalid mode (expecting J, B, D, R, I)";
4577 r = -EINVAL;
4578 goto bad;
4579 }
4580
4581 journal_sectors = 0;
4582 interleave_sectors = DEFAULT_INTERLEAVE_SECTORS;
4583 buffer_sectors = DEFAULT_BUFFER_SECTORS;
4584 journal_watermark = DEFAULT_JOURNAL_WATERMARK;
4585 sync_msec = DEFAULT_SYNC_MSEC;
4586 ic->sectors_per_block = 1;
4587
4588 as.argc = argc - DIRECT_ARGUMENTS;
4589 as.argv = argv + DIRECT_ARGUMENTS;
4590 r = dm_read_arg_group(_args, &as, &extra_args, &ti->error);
4591 if (r)
4592 goto bad;
4593
4594 while (extra_args--) {
4595 const char *opt_string;
4596 unsigned int val;
4597 unsigned long long llval;
4598
4599 opt_string = dm_shift_arg(&as);
4600 if (!opt_string) {
4601 r = -EINVAL;
4602 ti->error = "Not enough feature arguments";
4603 goto bad;
4604 }
4605 if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1)
4606 journal_sectors = val ? val : 1;
4607 else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1)
4608 interleave_sectors = val;
4609 else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1)
4610 buffer_sectors = val;
4611 else if (sscanf(opt_string, "journal_watermark:%u%c", &val, &dummy) == 1 && val <= 100)
4612 journal_watermark = val;
4613 else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1)
4614 sync_msec = val;
4615 else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) {
4616 if (ic->meta_dev) {
4617 dm_put_device(ti, ic->meta_dev);
4618 ic->meta_dev = NULL;
4619 }
4620 r = dm_get_device(ti, strchr(opt_string, ':') + 1,
4621 dm_table_get_mode(ti->table), &ic->meta_dev);
4622 if (r) {
4623 ti->error = "Device lookup failed";
4624 goto bad;
4625 }
4626 } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) {
4627 if (val < 1 << SECTOR_SHIFT ||
4628 val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT ||
4629 (val & (val - 1))) {
4630 r = -EINVAL;
4631 ti->error = "Invalid block_size argument";
4632 goto bad;
4633 }
4634 ic->sectors_per_block = val >> SECTOR_SHIFT;
4635 } else if (sscanf(opt_string, "sectors_per_bit:%llu%c", &llval, &dummy) == 1) {
4636 log2_sectors_per_bitmap_bit = !llval ? 0 : __ilog2_u64(llval);
4637 } else if (sscanf(opt_string, "bitmap_flush_interval:%u%c", &val, &dummy) == 1) {
4638 if ((uint64_t)val >= (uint64_t)UINT_MAX * 1000 / HZ) {
4639 r = -EINVAL;
4640 ti->error = "Invalid bitmap_flush_interval argument";
4641 goto bad;
4642 }
4643 ic->bitmap_flush_interval = msecs_to_jiffies(val);
4644 } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) {
4645 r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error,
4646 "Invalid internal_hash argument");
4647 if (r)
4648 goto bad;
4649 } else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) {
4650 r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error,
4651 "Invalid journal_crypt argument");
4652 if (r)
4653 goto bad;
4654 } else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) {
4655 r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error,
4656 "Invalid journal_mac argument");
4657 if (r)
4658 goto bad;
4659 } else if (!strcmp(opt_string, "recalculate")) {
4660 ic->recalculate_flag = true;
4661 } else if (!strcmp(opt_string, "reset_recalculate")) {
4662 ic->recalculate_flag = true;
4663 ic->reset_recalculate_flag = true;
4664 } else if (!strcmp(opt_string, "allow_discards")) {
4665 ic->discard = true;
4666 } else if (!strcmp(opt_string, "fix_padding")) {
4667 ic->fix_padding = true;
4668 } else if (!strcmp(opt_string, "fix_hmac")) {
4669 ic->fix_hmac = true;
4670 } else if (!strcmp(opt_string, "legacy_recalculate")) {
4671 ic->legacy_recalculate = true;
4672 } else {
4673 r = -EINVAL;
4674 ti->error = "Invalid argument";
4675 goto bad;
4676 }
4677 }
4678
4679 ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev);
4680 if (!ic->meta_dev)
4681 ic->meta_device_sectors = ic->data_device_sectors;
4682 else
4683 ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev);
4684
4685 if (!journal_sectors) {
4686 journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS,
4687 ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR);
4688 }
4689
4690 if (!buffer_sectors)
4691 buffer_sectors = 1;
4692 ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT);
4693
4694 r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error,
4695 "Invalid internal hash", "Error setting internal hash key");
4696 if (r)
4697 goto bad;
4698
4699 r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error,
4700 "Invalid journal mac", "Error setting journal mac key");
4701 if (r)
4702 goto bad;
4703
4704 if (!ic->tag_size) {
4705 if (!ic->internal_hash) {
4706 ti->error = "Unknown tag size";
4707 r = -EINVAL;
4708 goto bad;
4709 }
4710 ic->tag_size = crypto_shash_digestsize(ic->internal_hash);
4711 }
4712 if (ic->tag_size > MAX_TAG_SIZE) {
4713 ti->error = "Too big tag size";
4714 r = -EINVAL;
4715 goto bad;
4716 }
4717 if (!(ic->tag_size & (ic->tag_size - 1)))
4718 ic->log2_tag_size = __ffs(ic->tag_size);
4719 else
4720 ic->log2_tag_size = -1;
4721
4722 if (ic->mode == 'I') {
4723 struct blk_integrity *bi;
4724 if (ic->meta_dev) {
4725 r = -EINVAL;
4726 ti->error = "Metadata device not supported in inline mode";
4727 goto bad;
4728 }
4729 if (!ic->internal_hash_alg.alg_string) {
4730 r = -EINVAL;
4731 ti->error = "Internal hash not set in inline mode";
4732 goto bad;
4733 }
4734 if (ic->journal_crypt_alg.alg_string || ic->journal_mac_alg.alg_string) {
4735 r = -EINVAL;
4736 ti->error = "Journal crypt not supported in inline mode";
4737 goto bad;
4738 }
4739 if (ic->discard) {
4740 r = -EINVAL;
4741 ti->error = "Discards not supported in inline mode";
4742 goto bad;
4743 }
4744 bi = blk_get_integrity(ic->dev->bdev->bd_disk);
4745 if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) {
4746 r = -EINVAL;
4747 ti->error = "Integrity profile not supported";
4748 goto bad;
4749 }
4750 /*printk("tag_size: %u, tuple_size: %u\n", bi->tag_size, bi->tuple_size);*/
4751 if (bi->tuple_size < ic->tag_size) {
4752 r = -EINVAL;
4753 ti->error = "The integrity profile is smaller than tag size";
4754 goto bad;
4755 }
4756 if ((unsigned long)bi->tuple_size > PAGE_SIZE / 2) {
4757 r = -EINVAL;
4758 ti->error = "Too big tuple size";
4759 goto bad;
4760 }
4761 ic->tuple_size = bi->tuple_size;
4762 if (1 << bi->interval_exp != ic->sectors_per_block << SECTOR_SHIFT) {
4763 r = -EINVAL;
4764 ti->error = "Integrity profile sector size mismatch";
4765 goto bad;
4766 }
4767 }
4768
4769 if (ic->mode == 'B' && !ic->internal_hash) {
4770 r = -EINVAL;
4771 ti->error = "Bitmap mode can be only used with internal hash";
4772 goto bad;
4773 }
4774
4775 if (ic->discard && !ic->internal_hash) {
4776 r = -EINVAL;
4777 ti->error = "Discard can be only used with internal hash";
4778 goto bad;
4779 }
4780
4781 ic->autocommit_jiffies = msecs_to_jiffies(sync_msec);
4782 ic->autocommit_msec = sync_msec;
4783 timer_setup(&ic->autocommit_timer, autocommit_fn, 0);
4784
4785 ic->io = dm_io_client_create();
4786 if (IS_ERR(ic->io)) {
4787 r = PTR_ERR(ic->io);
4788 ic->io = NULL;
4789 ti->error = "Cannot allocate dm io";
4790 goto bad;
4791 }
4792
4793 r = mempool_init_slab_pool(&ic->journal_io_mempool, JOURNAL_IO_MEMPOOL, journal_io_cache);
4794 if (r) {
4795 ti->error = "Cannot allocate mempool";
4796 goto bad;
4797 }
4798
4799 r = mempool_init_page_pool(&ic->recheck_pool, 1, ic->mode == 'I' ? 1 : 0);
4800 if (r) {
4801 ti->error = "Cannot allocate mempool";
4802 goto bad;
4803 }
4804
4805 if (ic->mode == 'I') {
4806 r = bioset_init(&ic->recheck_bios, RECHECK_POOL_SIZE, 0, BIOSET_NEED_BVECS);
4807 if (r) {
4808 ti->error = "Cannot allocate bio set";
4809 goto bad;
4810 }
4811 r = bioset_integrity_create(&ic->recheck_bios, RECHECK_POOL_SIZE);
4812 if (r) {
4813 ti->error = "Cannot allocate bio integrity set";
4814 r = -ENOMEM;
4815 goto bad;
4816 }
4817 r = bioset_init(&ic->recalc_bios, 1, 0, BIOSET_NEED_BVECS);
4818 if (r) {
4819 ti->error = "Cannot allocate bio set";
4820 goto bad;
4821 }
4822 r = bioset_integrity_create(&ic->recalc_bios, 1);
4823 if (r) {
4824 ti->error = "Cannot allocate bio integrity set";
4825 r = -ENOMEM;
4826 goto bad;
4827 }
4828 }
4829
4830 ic->metadata_wq = alloc_workqueue("dm-integrity-metadata",
4831 WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE);
4832 if (!ic->metadata_wq) {
4833 ti->error = "Cannot allocate workqueue";
4834 r = -ENOMEM;
4835 goto bad;
4836 }
4837
4838 /*
4839 * If this workqueue weren't ordered, it would cause bio reordering
4840 * and reduced performance.
4841 */
4842 ic->wait_wq = alloc_ordered_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM);
4843 if (!ic->wait_wq) {
4844 ti->error = "Cannot allocate workqueue";
4845 r = -ENOMEM;
4846 goto bad;
4847 }
4848
4849 ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM,
4850 METADATA_WORKQUEUE_MAX_ACTIVE);
4851 if (!ic->offload_wq) {
4852 ti->error = "Cannot allocate workqueue";
4853 r = -ENOMEM;
4854 goto bad;
4855 }
4856
4857 ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1);
4858 if (!ic->commit_wq) {
4859 ti->error = "Cannot allocate workqueue";
4860 r = -ENOMEM;
4861 goto bad;
4862 }
4863 INIT_WORK(&ic->commit_work, integrity_commit);
4864
4865 if (ic->mode == 'J' || ic->mode == 'B') {
4866 ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1);
4867 if (!ic->writer_wq) {
4868 ti->error = "Cannot allocate workqueue";
4869 r = -ENOMEM;
4870 goto bad;
4871 }
4872 INIT_WORK(&ic->writer_work, integrity_writer);
4873 }
4874
4875 ic->sb = alloc_pages_exact(SB_SECTORS << SECTOR_SHIFT, GFP_KERNEL);
4876 if (!ic->sb) {
4877 r = -ENOMEM;
4878 ti->error = "Cannot allocate superblock area";
4879 goto bad;
4880 }
4881
4882 r = sync_rw_sb(ic, REQ_OP_READ);
4883 if (r) {
4884 ti->error = "Error reading superblock";
4885 goto bad;
4886 }
4887 should_write_sb = false;
4888 if (memcmp(ic->sb->magic, SB_MAGIC, 8)) {
4889 if (ic->mode != 'R') {
4890 if (memchr_inv(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT)) {
4891 r = -EINVAL;
4892 ti->error = "The device is not initialized";
4893 goto bad;
4894 }
4895 }
4896
4897 r = initialize_superblock(ic, journal_sectors, interleave_sectors);
4898 if (r) {
4899 ti->error = "Could not initialize superblock";
4900 goto bad;
4901 }
4902 if (ic->mode != 'R')
4903 should_write_sb = true;
4904 }
4905
4906 if (!ic->sb->version || ic->sb->version > SB_VERSION_6) {
4907 r = -EINVAL;
4908 ti->error = "Unknown version";
4909 goto bad;
4910 }
4911 if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) != (ic->mode == 'I')) {
4912 r = -EINVAL;
4913 ti->error = "Inline flag mismatch";
4914 goto bad;
4915 }
4916 if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) {
4917 r = -EINVAL;
4918 ti->error = "Tag size doesn't match the information in superblock";
4919 goto bad;
4920 }
4921 if (ic->sb->log2_sectors_per_block != __ffs(ic->sectors_per_block)) {
4922 r = -EINVAL;
4923 ti->error = "Block size doesn't match the information in superblock";
4924 goto bad;
4925 }
4926 if (ic->mode != 'I') {
4927 if (!le32_to_cpu(ic->sb->journal_sections)) {
4928 r = -EINVAL;
4929 ti->error = "Corrupted superblock, journal_sections is 0";
4930 goto bad;
4931 }
4932 } else {
4933 if (le32_to_cpu(ic->sb->journal_sections)) {
4934 r = -EINVAL;
4935 ti->error = "Corrupted superblock, journal_sections is not 0";
4936 goto bad;
4937 }
4938 }
4939 /* make sure that ti->max_io_len doesn't overflow */
4940 if (!ic->meta_dev) {
4941 if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS ||
4942 ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) {
4943 r = -EINVAL;
4944 ti->error = "Invalid interleave_sectors in the superblock";
4945 goto bad;
4946 }
4947 } else {
4948 if (ic->sb->log2_interleave_sectors) {
4949 r = -EINVAL;
4950 ti->error = "Invalid interleave_sectors in the superblock";
4951 goto bad;
4952 }
4953 }
4954 if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) {
4955 r = -EINVAL;
4956 ti->error = "Journal mac mismatch";
4957 goto bad;
4958 }
4959
4960 get_provided_data_sectors(ic);
4961 if (!ic->provided_data_sectors) {
4962 r = -EINVAL;
4963 ti->error = "The device is too small";
4964 goto bad;
4965 }
4966
4967 try_smaller_buffer:
4968 r = calculate_device_limits(ic);
4969 if (r) {
4970 if (ic->meta_dev) {
4971 if (ic->log2_buffer_sectors > 3) {
4972 ic->log2_buffer_sectors--;
4973 goto try_smaller_buffer;
4974 }
4975 }
4976 ti->error = "The device is too small";
4977 goto bad;
4978 }
4979
4980 if (log2_sectors_per_bitmap_bit < 0)
4981 log2_sectors_per_bitmap_bit = __fls(DEFAULT_SECTORS_PER_BITMAP_BIT);
4982 if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block)
4983 log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block;
4984
4985 bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3);
4986 if (bits_in_journal > UINT_MAX)
4987 bits_in_journal = UINT_MAX;
4988 if (bits_in_journal)
4989 while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit) - 1) >> log2_sectors_per_bitmap_bit)
4990 log2_sectors_per_bitmap_bit++;
4991
4992 log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block;
4993 ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit;
4994 if (should_write_sb)
4995 ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit;
4996
4997 n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block)
4998 + (((sector_t)1 << log2_blocks_per_bitmap_bit) - 1)) >> log2_blocks_per_bitmap_bit;
4999 ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8);
5000
5001 if (!ic->meta_dev)
5002 ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run));
5003
5004 if (ti->len > ic->provided_data_sectors) {
5005 r = -EINVAL;
5006 ti->error = "Not enough provided sectors for requested mapping size";
5007 goto bad;
5008 }
5009
5010 threshold = (__u64)ic->journal_entries * (100 - journal_watermark);
5011 threshold += 50;
5012 do_div(threshold, 100);
5013 ic->free_sectors_threshold = threshold;
5014
5015 DEBUG_print("initialized:\n");
5016 DEBUG_print(" integrity_tag_size %u\n", le16_to_cpu(ic->sb->integrity_tag_size));
5017 DEBUG_print(" journal_entry_size %u\n", ic->journal_entry_size);
5018 DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector);
5019 DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries);
5020 DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors);
5021 DEBUG_print(" journal_sections %u\n", (unsigned int)le32_to_cpu(ic->sb->journal_sections));
5022 DEBUG_print(" journal_entries %u\n", ic->journal_entries);
5023 DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors);
5024 DEBUG_print(" data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev));
5025 DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors);
5026 DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run);
5027 DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run);
5028 DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data_sectors);
5029 DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors);
5030 DEBUG_print(" bits_in_journal %llu\n", bits_in_journal);
5031
5032 if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) {
5033 ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING);
5034 ic->sb->recalc_sector = cpu_to_le64(0);
5035 }
5036
5037 if (ic->internal_hash) {
5038 ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1);
5039 if (!ic->recalc_wq) {
5040 ti->error = "Cannot allocate workqueue";
5041 r = -ENOMEM;
5042 goto bad;
5043 }
5044 INIT_WORK(&ic->recalc_work, ic->mode == 'I' ? integrity_recalc_inline : integrity_recalc);
5045 } else {
5046 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) {
5047 ti->error = "Recalculate can only be specified with internal_hash";
5048 r = -EINVAL;
5049 goto bad;
5050 }
5051 }
5052
5053 if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) &&
5054 le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors &&
5055 dm_integrity_disable_recalculate(ic)) {
5056 ti->error = "Recalculating with HMAC is disabled for security reasons - if you really need it, use the argument \"legacy_recalculate\"";
5057 r = -EOPNOTSUPP;
5058 goto bad;
5059 }
5060
5061 ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev,
5062 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0);
5063 if (IS_ERR(ic->bufio)) {
5064 r = PTR_ERR(ic->bufio);
5065 ti->error = "Cannot initialize dm-bufio";
5066 ic->bufio = NULL;
5067 goto bad;
5068 }
5069 dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors);
5070
5071 if (ic->mode != 'R' && ic->mode != 'I') {
5072 r = create_journal(ic, &ti->error);
5073 if (r)
5074 goto bad;
5075
5076 }
5077
5078 if (ic->mode == 'B') {
5079 unsigned int i;
5080 unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE);
5081
5082 ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages);
5083 if (!ic->recalc_bitmap) {
5084 r = -ENOMEM;
5085 goto bad;
5086 }
5087 ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages);
5088 if (!ic->may_write_bitmap) {
5089 r = -ENOMEM;
5090 goto bad;
5091 }
5092 ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL);
5093 if (!ic->bbs) {
5094 r = -ENOMEM;
5095 goto bad;
5096 }
5097 INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work);
5098 for (i = 0; i < ic->n_bitmap_blocks; i++) {
5099 struct bitmap_block_status *bbs = &ic->bbs[i];
5100 unsigned int sector, pl_index, pl_offset;
5101
5102 INIT_WORK(&bbs->work, bitmap_block_work);
5103 bbs->ic = ic;
5104 bbs->idx = i;
5105 bio_list_init(&bbs->bio_queue);
5106 spin_lock_init(&bbs->bio_queue_lock);
5107
5108 sector = i * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT);
5109 pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT);
5110 pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1);
5111
5112 bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset;
5113 }
5114 }
5115
5116 if (should_write_sb) {
5117 init_journal(ic, 0, ic->journal_sections, 0);
5118 r = dm_integrity_failed(ic);
5119 if (unlikely(r)) {
5120 ti->error = "Error initializing journal";
5121 goto bad;
5122 }
5123 r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA);
5124 if (r) {
5125 ti->error = "Error initializing superblock";
5126 goto bad;
5127 }
5128 ic->just_formatted = true;
5129 }
5130
5131 if (!ic->meta_dev && ic->mode != 'I') {
5132 r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors);
5133 if (r)
5134 goto bad;
5135 }
5136 if (ic->mode == 'B') {
5137 unsigned int max_io_len;
5138
5139 max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMAP_BLOCK_SIZE * 8);
5140 if (!max_io_len)
5141 max_io_len = 1U << 31;
5142 DEBUG_print("max_io_len: old %u, new %u\n", ti->max_io_len, max_io_len);
5143 if (!ti->max_io_len || ti->max_io_len > max_io_len) {
5144 r = dm_set_target_max_io_len(ti, max_io_len);
5145 if (r)
5146 goto bad;
5147 }
5148 }
5149
5150 ti->num_flush_bios = 1;
5151 ti->flush_supported = true;
5152 if (ic->discard)
5153 ti->num_discard_bios = 1;
5154
5155 if (ic->mode == 'I')
5156 ti->mempool_needs_integrity = true;
5157
5158 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1);
5159 return 0;
5160
5161 bad:
5162 dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0);
5163 dm_integrity_dtr(ti);
5164 return r;
5165 }
5166
5167 static void dm_integrity_dtr(struct dm_target *ti)
5168 {
5169 struct dm_integrity_c *ic = ti->private;
5170
5171 BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress));
5172 BUG_ON(!list_empty(&ic->wait_list));
5173
5174 if (ic->mode == 'B')
5175 cancel_delayed_work_sync(&ic->bitmap_flush_work);
5176 if (ic->metadata_wq)
5177 destroy_workqueue(ic->metadata_wq);
5178 if (ic->wait_wq)
5179 destroy_workqueue(ic->wait_wq);
5180 if (ic->offload_wq)
5181 destroy_workqueue(ic->offload_wq);
5182 if (ic->commit_wq)
5183 destroy_workqueue(ic->commit_wq);
5184 if (ic->writer_wq)
5185 destroy_workqueue(ic->writer_wq);
5186 if (ic->recalc_wq)
5187 destroy_workqueue(ic->recalc_wq);
5188 kvfree(ic->bbs);
5189 if (ic->bufio)
5190 dm_bufio_client_destroy(ic->bufio);
5191 bioset_exit(&ic->recalc_bios);
5192 bioset_exit(&ic->recheck_bios);
5193 mempool_exit(&ic->recheck_pool);
5194 mempool_exit(&ic->journal_io_mempool);
5195 if (ic->io)
5196 dm_io_client_destroy(ic->io);
5197 if (ic->dev)
5198 dm_put_device(ti, ic->dev);
5199 if (ic->meta_dev)
5200 dm_put_device(ti, ic->meta_dev);
5201 dm_integrity_free_page_list(ic->journal);
5202 dm_integrity_free_page_list(ic->journal_io);
5203 dm_integrity_free_page_list(ic->journal_xor);
5204 dm_integrity_free_page_list(ic->recalc_bitmap);
5205 dm_integrity_free_page_list(ic->may_write_bitmap);
5206 if (ic->journal_scatterlist)
5207 dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist);
5208 if (ic->journal_io_scatterlist)
5209 dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist);
5210 if (ic->sk_requests) {
5211 unsigned int i;
5212
5213 for (i = 0; i < ic->journal_sections; i++) {
5214 struct skcipher_request *req;
5215
5216 req = ic->sk_requests[i];
5217 if (req) {
5218 kfree_sensitive(req->iv);
5219 skcipher_request_free(req);
5220 }
5221 }
5222 kvfree(ic->sk_requests);
5223 }
5224 kvfree(ic->journal_tree);
5225 if (ic->sb)
5226 free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT);
5227
5228 if (ic->internal_hash)
5229 crypto_free_shash(ic->internal_hash);
5230 free_alg(&ic->internal_hash_alg);
5231
5232 if (ic->journal_crypt)
5233 crypto_free_skcipher(ic->journal_crypt);
5234 free_alg(&ic->journal_crypt_alg);
5235
5236 if (ic->journal_mac)
5237 crypto_free_shash(ic->journal_mac);
5238 free_alg(&ic->journal_mac_alg);
5239
5240 kfree(ic);
5241 dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1);
5242 }
5243
5244 static struct target_type integrity_target = {
5245 .name = "integrity",
5246 .version = {1, 13, 0},
5247 .module = THIS_MODULE,
5248 .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY,
5249 .ctr = dm_integrity_ctr,
5250 .dtr = dm_integrity_dtr,
5251 .map = dm_integrity_map,
5252 .end_io = dm_integrity_end_io,
5253 .postsuspend = dm_integrity_postsuspend,
5254 .resume = dm_integrity_resume,
5255 .status = dm_integrity_status,
5256 .iterate_devices = dm_integrity_iterate_devices,
5257 .io_hints = dm_integrity_io_hints,
5258 };
5259
5260 static int __init dm_integrity_init(void)
5261 {
5262 int r;
5263
5264 journal_io_cache = kmem_cache_create("integrity_journal_io",
5265 sizeof(struct journal_io), 0, 0, NULL);
5266 if (!journal_io_cache) {
5267 DMERR("can't allocate journal io cache");
5268 return -ENOMEM;
5269 }
5270
5271 r = dm_register_target(&integrity_target);
5272 if (r < 0) {
5273 kmem_cache_destroy(journal_io_cache);
5274 return r;
5275 }
5276
5277 return 0;
5278 }
5279
5280 static void __exit dm_integrity_exit(void)
5281 {
5282 dm_unregister_target(&integrity_target);
5283 kmem_cache_destroy(journal_io_cache);
5284 }
5285
5286 module_init(dm_integrity_init);
5287 module_exit(dm_integrity_exit);
5288
5289 MODULE_AUTHOR("Milan Broz");
5290 MODULE_AUTHOR("Mikulas Patocka");
5291 MODULE_DESCRIPTION(DM_NAME " target for integrity tags extension");
5292 MODULE_LICENSE("GPL");
Kernel DRM miscellaneous fixes and cross-tree changes