2 * Copyright (C) 2014 Facebook. All rights reserved.
4 * This file is released under the GPL.
7 #include <linux/device-mapper.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/blkdev.h>
12 #include <linux/bio.h>
13 #include <linux/slab.h>
14 #include <linux/kthread.h>
15 #include <linux/freezer.h>
17 #define DM_MSG_PREFIX "log-writes"
20 * This target will sequentially log all writes to the target device onto the
21 * log device. This is helpful for replaying writes to check for fs consistency
22 * at all times. This target provides a mechanism to mark specific events to
23 * check data at a later time. So for example you would:
27 * dmsetup message /dev/whatever mark mymark
30 * Then replay the log up to mymark and check the contents of the replay to
31 * verify it matches what was written.
33 * We log writes only after they have been flushed, this makes the log describe
34 * close to the order in which the data hits the actual disk, not its cache. So
35 * for example the following sequence (W means write, C means complete)
37 * Wa,Wb,Wc,Cc,Ca,FLUSH,FUAd,Cb,CFLUSH,CFUAd
39 * Would result in the log looking like this:
41 * c,a,flush,fuad,b,<other writes>,<next flush>
43 * This is meant to help expose problems where file systems do not properly wait
44 * on data being written before invoking a FLUSH. FUA bypasses cache so once it
45 * completes it is added to the log as it should be on disk.
47 * We treat DISCARDs as if they don't bypass cache so that they are logged in
48 * order of completion along with the normal writes. If we didn't do it this
49 * way we would process all the discards first and then write all the data, when
50 * in fact we want to do the data and the discard in the order that they
53 #define LOG_FLUSH_FLAG (1 << 0)
54 #define LOG_FUA_FLAG (1 << 1)
55 #define LOG_DISCARD_FLAG (1 << 2)
56 #define LOG_MARK_FLAG (1 << 3)
58 #define WRITE_LOG_VERSION 1ULL
59 #define WRITE_LOG_MAGIC 0x6a736677736872ULL
62 * The disk format for this is braindead simple.
64 * At byte 0 we have our super, followed by the following sequence for
67 * [ 1 sector ][ entry->nr_sectors ]
68 * [log_write_entry][ data written ]
70 * The log_write_entry takes up a full sector so we can have arbitrary length
71 * marks and it leaves us room for extra content in the future.
75 * Basic info about the log for userspace.
77 struct log_write_super
{
85 * sector - the sector we wrote.
86 * nr_sectors - the number of sectors we wrote.
87 * flags - flags for this log entry.
88 * data_len - the size of the data in this log entry, this is for private log
89 * entry stuff, the MARK data provided by userspace for example.
91 struct log_write_entry
{
100 struct dm_dev
*logdev
;
104 atomic_t pending_blocks
;
105 sector_t next_sector
;
107 bool logging_enabled
;
108 bool device_supports_discard
;
109 spinlock_t blocks_lock
;
110 struct list_head unflushed_blocks
;
111 struct list_head logging_blocks
;
112 wait_queue_head_t wait
;
113 struct task_struct
*log_kthread
;
116 struct pending_block
{
123 struct list_head list
;
124 struct bio_vec vecs
[0];
127 struct per_bio_data
{
128 struct pending_block
*block
;
131 static void put_pending_block(struct log_writes_c
*lc
)
133 if (atomic_dec_and_test(&lc
->pending_blocks
)) {
134 smp_mb__after_atomic();
135 if (waitqueue_active(&lc
->wait
))
140 static void put_io_block(struct log_writes_c
*lc
)
142 if (atomic_dec_and_test(&lc
->io_blocks
)) {
143 smp_mb__after_atomic();
144 if (waitqueue_active(&lc
->wait
))
149 static void log_end_io(struct bio
*bio
)
151 struct log_writes_c
*lc
= bio
->bi_private
;
152 struct bio_vec
*bvec
;
158 DMERR("Error writing log block, error=%d", bio
->bi_error
);
159 spin_lock_irqsave(&lc
->blocks_lock
, flags
);
160 lc
->logging_enabled
= false;
161 spin_unlock_irqrestore(&lc
->blocks_lock
, flags
);
164 bio_for_each_segment_all(bvec
, bio
, i
)
165 __free_page(bvec
->bv_page
);
172 * Meant to be called if there is an error, it will free all the pages
173 * associated with the block.
175 static void free_pending_block(struct log_writes_c
*lc
,
176 struct pending_block
*block
)
180 for (i
= 0; i
< block
->vec_cnt
; i
++) {
181 if (block
->vecs
[i
].bv_page
)
182 __free_page(block
->vecs
[i
].bv_page
);
186 put_pending_block(lc
);
189 static int write_metadata(struct log_writes_c
*lc
, void *entry
,
190 size_t entrylen
, void *data
, size_t datalen
,
198 bio
= bio_alloc(GFP_KERNEL
, 1);
200 DMERR("Couldn't alloc log bio");
203 bio
->bi_iter
.bi_size
= 0;
204 bio
->bi_iter
.bi_sector
= sector
;
205 bio
->bi_bdev
= lc
->logdev
->bdev
;
206 bio
->bi_end_io
= log_end_io
;
207 bio
->bi_private
= lc
;
209 page
= alloc_page(GFP_KERNEL
);
211 DMERR("Couldn't alloc log page");
216 ptr
= kmap_atomic(page
);
217 memcpy(ptr
, entry
, entrylen
);
219 memcpy(ptr
+ entrylen
, data
, datalen
);
220 memset(ptr
+ entrylen
+ datalen
, 0,
221 lc
->sectorsize
- entrylen
- datalen
);
224 ret
= bio_add_page(bio
, page
, lc
->sectorsize
, 0);
225 if (ret
!= lc
->sectorsize
) {
226 DMERR("Couldn't add page to the log block");
229 submit_bio(WRITE
, bio
);
239 static int log_one_block(struct log_writes_c
*lc
,
240 struct pending_block
*block
, sector_t sector
)
243 struct log_write_entry entry
;
247 entry
.sector
= cpu_to_le64(block
->sector
);
248 entry
.nr_sectors
= cpu_to_le64(block
->nr_sectors
);
249 entry
.flags
= cpu_to_le64(block
->flags
);
250 entry
.data_len
= cpu_to_le64(block
->datalen
);
251 if (write_metadata(lc
, &entry
, sizeof(entry
), block
->data
,
252 block
->datalen
, sector
)) {
253 free_pending_block(lc
, block
);
261 bio
= bio_alloc(GFP_KERNEL
, block
->vec_cnt
);
263 DMERR("Couldn't alloc log bio");
266 atomic_inc(&lc
->io_blocks
);
267 bio
->bi_iter
.bi_size
= 0;
268 bio
->bi_iter
.bi_sector
= sector
;
269 bio
->bi_bdev
= lc
->logdev
->bdev
;
270 bio
->bi_end_io
= log_end_io
;
271 bio
->bi_private
= lc
;
273 for (i
= 0; i
< block
->vec_cnt
; i
++) {
275 * The page offset is always 0 because we allocate a new page
276 * for every bvec in the original bio for simplicity sake.
278 ret
= bio_add_page(bio
, block
->vecs
[i
].bv_page
,
279 block
->vecs
[i
].bv_len
, 0);
280 if (ret
!= block
->vecs
[i
].bv_len
) {
281 atomic_inc(&lc
->io_blocks
);
282 submit_bio(WRITE
, bio
);
283 bio
= bio_alloc(GFP_KERNEL
, block
->vec_cnt
- i
);
285 DMERR("Couldn't alloc log bio");
288 bio
->bi_iter
.bi_size
= 0;
289 bio
->bi_iter
.bi_sector
= sector
;
290 bio
->bi_bdev
= lc
->logdev
->bdev
;
291 bio
->bi_end_io
= log_end_io
;
292 bio
->bi_private
= lc
;
294 ret
= bio_add_page(bio
, block
->vecs
[i
].bv_page
,
295 block
->vecs
[i
].bv_len
, 0);
296 if (ret
!= block
->vecs
[i
].bv_len
) {
297 DMERR("Couldn't add page on new bio?");
302 sector
+= block
->vecs
[i
].bv_len
>> SECTOR_SHIFT
;
304 submit_bio(WRITE
, bio
);
308 put_pending_block(lc
);
311 free_pending_block(lc
, block
);
316 static int log_super(struct log_writes_c
*lc
)
318 struct log_write_super super
;
320 super
.magic
= cpu_to_le64(WRITE_LOG_MAGIC
);
321 super
.version
= cpu_to_le64(WRITE_LOG_VERSION
);
322 super
.nr_entries
= cpu_to_le64(lc
->logged_entries
);
323 super
.sectorsize
= cpu_to_le32(lc
->sectorsize
);
325 if (write_metadata(lc
, &super
, sizeof(super
), NULL
, 0, 0)) {
326 DMERR("Couldn't write super");
333 static inline sector_t
logdev_last_sector(struct log_writes_c
*lc
)
335 return i_size_read(lc
->logdev
->bdev
->bd_inode
) >> SECTOR_SHIFT
;
338 static int log_writes_kthread(void *arg
)
340 struct log_writes_c
*lc
= (struct log_writes_c
*)arg
;
343 while (!kthread_should_stop()) {
345 bool logging_enabled
;
346 struct pending_block
*block
= NULL
;
349 spin_lock_irq(&lc
->blocks_lock
);
350 if (!list_empty(&lc
->logging_blocks
)) {
351 block
= list_first_entry(&lc
->logging_blocks
,
352 struct pending_block
, list
);
353 list_del_init(&block
->list
);
354 if (!lc
->logging_enabled
)
357 sector
= lc
->next_sector
;
358 if (block
->flags
& LOG_DISCARD_FLAG
)
361 lc
->next_sector
+= block
->nr_sectors
+ 1;
364 * Apparently the size of the device may not be known
365 * right away, so handle this properly.
368 lc
->end_sector
= logdev_last_sector(lc
);
369 if (lc
->end_sector
&&
370 lc
->next_sector
>= lc
->end_sector
) {
371 DMERR("Ran out of space on the logdev");
372 lc
->logging_enabled
= false;
375 lc
->logged_entries
++;
376 atomic_inc(&lc
->io_blocks
);
378 super
= (block
->flags
& (LOG_FUA_FLAG
| LOG_MARK_FLAG
));
380 atomic_inc(&lc
->io_blocks
);
383 logging_enabled
= lc
->logging_enabled
;
384 spin_unlock_irq(&lc
->blocks_lock
);
386 if (logging_enabled
) {
387 ret
= log_one_block(lc
, block
, sector
);
391 spin_lock_irq(&lc
->blocks_lock
);
392 lc
->logging_enabled
= false;
393 spin_unlock_irq(&lc
->blocks_lock
);
396 free_pending_block(lc
, block
);
400 if (!try_to_freeze()) {
401 set_current_state(TASK_INTERRUPTIBLE
);
402 if (!kthread_should_stop() &&
403 !atomic_read(&lc
->pending_blocks
))
405 __set_current_state(TASK_RUNNING
);
412 * Construct a log-writes mapping:
413 * log-writes <dev_path> <log_dev_path>
415 static int log_writes_ctr(struct dm_target
*ti
, unsigned int argc
, char **argv
)
417 struct log_writes_c
*lc
;
418 struct dm_arg_set as
;
419 const char *devname
, *logdevname
;
426 ti
->error
= "Invalid argument count";
430 lc
= kzalloc(sizeof(struct log_writes_c
), GFP_KERNEL
);
432 ti
->error
= "Cannot allocate context";
435 spin_lock_init(&lc
->blocks_lock
);
436 INIT_LIST_HEAD(&lc
->unflushed_blocks
);
437 INIT_LIST_HEAD(&lc
->logging_blocks
);
438 init_waitqueue_head(&lc
->wait
);
439 lc
->sectorsize
= 1 << SECTOR_SHIFT
;
440 atomic_set(&lc
->io_blocks
, 0);
441 atomic_set(&lc
->pending_blocks
, 0);
443 devname
= dm_shift_arg(&as
);
444 ret
= dm_get_device(ti
, devname
, dm_table_get_mode(ti
->table
), &lc
->dev
);
446 ti
->error
= "Device lookup failed";
450 logdevname
= dm_shift_arg(&as
);
451 ret
= dm_get_device(ti
, logdevname
, dm_table_get_mode(ti
->table
),
454 ti
->error
= "Log device lookup failed";
455 dm_put_device(ti
, lc
->dev
);
460 lc
->log_kthread
= kthread_run(log_writes_kthread
, lc
, "log-write");
461 if (!lc
->log_kthread
) {
462 ti
->error
= "Couldn't alloc kthread";
463 dm_put_device(ti
, lc
->dev
);
464 dm_put_device(ti
, lc
->logdev
);
468 /* We put the super at sector 0, start logging at sector 1 */
470 lc
->logging_enabled
= true;
471 lc
->end_sector
= logdev_last_sector(lc
);
472 lc
->device_supports_discard
= true;
474 ti
->num_flush_bios
= 1;
475 ti
->flush_supported
= true;
476 ti
->num_discard_bios
= 1;
477 ti
->discards_supported
= true;
478 ti
->per_bio_data_size
= sizeof(struct per_bio_data
);
487 static int log_mark(struct log_writes_c
*lc
, char *data
)
489 struct pending_block
*block
;
490 size_t maxsize
= lc
->sectorsize
- sizeof(struct log_write_entry
);
492 block
= kzalloc(sizeof(struct pending_block
), GFP_KERNEL
);
494 DMERR("Error allocating pending block");
498 block
->data
= kstrndup(data
, maxsize
, GFP_KERNEL
);
500 DMERR("Error copying mark data");
504 atomic_inc(&lc
->pending_blocks
);
505 block
->datalen
= strlen(block
->data
);
506 block
->flags
|= LOG_MARK_FLAG
;
507 spin_lock_irq(&lc
->blocks_lock
);
508 list_add_tail(&block
->list
, &lc
->logging_blocks
);
509 spin_unlock_irq(&lc
->blocks_lock
);
510 wake_up_process(lc
->log_kthread
);
514 static void log_writes_dtr(struct dm_target
*ti
)
516 struct log_writes_c
*lc
= ti
->private;
518 spin_lock_irq(&lc
->blocks_lock
);
519 list_splice_init(&lc
->unflushed_blocks
, &lc
->logging_blocks
);
520 spin_unlock_irq(&lc
->blocks_lock
);
523 * This is just nice to have since it'll update the super to include the
524 * unflushed blocks, if it fails we don't really care.
526 log_mark(lc
, "dm-log-writes-end");
527 wake_up_process(lc
->log_kthread
);
528 wait_event(lc
->wait
, !atomic_read(&lc
->io_blocks
) &&
529 !atomic_read(&lc
->pending_blocks
));
530 kthread_stop(lc
->log_kthread
);
532 WARN_ON(!list_empty(&lc
->logging_blocks
));
533 WARN_ON(!list_empty(&lc
->unflushed_blocks
));
534 dm_put_device(ti
, lc
->dev
);
535 dm_put_device(ti
, lc
->logdev
);
539 static void normal_map_bio(struct dm_target
*ti
, struct bio
*bio
)
541 struct log_writes_c
*lc
= ti
->private;
543 bio
->bi_bdev
= lc
->dev
->bdev
;
546 static int log_writes_map(struct dm_target
*ti
, struct bio
*bio
)
548 struct log_writes_c
*lc
= ti
->private;
549 struct per_bio_data
*pb
= dm_per_bio_data(bio
, sizeof(struct per_bio_data
));
550 struct pending_block
*block
;
551 struct bvec_iter iter
;
555 bool flush_bio
= (bio
->bi_rw
& REQ_FLUSH
);
556 bool fua_bio
= (bio
->bi_rw
& REQ_FUA
);
557 bool discard_bio
= (bio
->bi_rw
& REQ_DISCARD
);
561 /* Don't bother doing anything if logging has been disabled */
562 if (!lc
->logging_enabled
)
566 * Map reads as normal.
568 if (bio_data_dir(bio
) == READ
)
571 /* No sectors and not a flush? Don't care */
572 if (!bio_sectors(bio
) && !flush_bio
)
576 * Discards will have bi_size set but there's no actual data, so just
577 * allocate the size of the pending block.
580 alloc_size
= sizeof(struct pending_block
);
582 alloc_size
= sizeof(struct pending_block
) + sizeof(struct bio_vec
) * bio_segments(bio
);
584 block
= kzalloc(alloc_size
, GFP_NOIO
);
586 DMERR("Error allocating pending block");
587 spin_lock_irq(&lc
->blocks_lock
);
588 lc
->logging_enabled
= false;
589 spin_unlock_irq(&lc
->blocks_lock
);
592 INIT_LIST_HEAD(&block
->list
);
594 atomic_inc(&lc
->pending_blocks
);
597 block
->flags
|= LOG_FLUSH_FLAG
;
599 block
->flags
|= LOG_FUA_FLAG
;
601 block
->flags
|= LOG_DISCARD_FLAG
;
603 block
->sector
= bio
->bi_iter
.bi_sector
;
604 block
->nr_sectors
= bio_sectors(bio
);
606 /* We don't need the data, just submit */
608 WARN_ON(flush_bio
|| fua_bio
);
609 if (lc
->device_supports_discard
)
612 return DM_MAPIO_SUBMITTED
;
615 /* Flush bio, splice the unflushed blocks onto this list and submit */
616 if (flush_bio
&& !bio_sectors(bio
)) {
617 spin_lock_irq(&lc
->blocks_lock
);
618 list_splice_init(&lc
->unflushed_blocks
, &block
->list
);
619 spin_unlock_irq(&lc
->blocks_lock
);
624 * We will write this bio somewhere else way later so we need to copy
625 * the actual contents into new pages so we know the data will always be
628 * We do this because this could be a bio from O_DIRECT in which case we
629 * can't just hold onto the page until some later point, we have to
630 * manually copy the contents.
632 bio_for_each_segment(bv
, bio
, iter
) {
636 page
= alloc_page(GFP_NOIO
);
638 DMERR("Error allocing page");
639 free_pending_block(lc
, block
);
640 spin_lock_irq(&lc
->blocks_lock
);
641 lc
->logging_enabled
= false;
642 spin_unlock_irq(&lc
->blocks_lock
);
646 src
= kmap_atomic(bv
.bv_page
);
647 dst
= kmap_atomic(page
);
648 memcpy(dst
, src
+ bv
.bv_offset
, bv
.bv_len
);
651 block
->vecs
[i
].bv_page
= page
;
652 block
->vecs
[i
].bv_len
= bv
.bv_len
;
657 /* Had a flush with data in it, weird */
659 spin_lock_irq(&lc
->blocks_lock
);
660 list_splice_init(&lc
->unflushed_blocks
, &block
->list
);
661 spin_unlock_irq(&lc
->blocks_lock
);
664 normal_map_bio(ti
, bio
);
665 return DM_MAPIO_REMAPPED
;
668 static int normal_end_io(struct dm_target
*ti
, struct bio
*bio
, int error
)
670 struct log_writes_c
*lc
= ti
->private;
671 struct per_bio_data
*pb
= dm_per_bio_data(bio
, sizeof(struct per_bio_data
));
673 if (bio_data_dir(bio
) == WRITE
&& pb
->block
) {
674 struct pending_block
*block
= pb
->block
;
677 spin_lock_irqsave(&lc
->blocks_lock
, flags
);
678 if (block
->flags
& LOG_FLUSH_FLAG
) {
679 list_splice_tail_init(&block
->list
, &lc
->logging_blocks
);
680 list_add_tail(&block
->list
, &lc
->logging_blocks
);
681 wake_up_process(lc
->log_kthread
);
682 } else if (block
->flags
& LOG_FUA_FLAG
) {
683 list_add_tail(&block
->list
, &lc
->logging_blocks
);
684 wake_up_process(lc
->log_kthread
);
686 list_add_tail(&block
->list
, &lc
->unflushed_blocks
);
687 spin_unlock_irqrestore(&lc
->blocks_lock
, flags
);
694 * INFO format: <logged entries> <highest allocated sector>
696 static void log_writes_status(struct dm_target
*ti
, status_type_t type
,
697 unsigned status_flags
, char *result
,
701 struct log_writes_c
*lc
= ti
->private;
704 case STATUSTYPE_INFO
:
705 DMEMIT("%llu %llu", lc
->logged_entries
,
706 (unsigned long long)lc
->next_sector
- 1);
707 if (!lc
->logging_enabled
)
708 DMEMIT(" logging_disabled");
711 case STATUSTYPE_TABLE
:
712 DMEMIT("%s %s", lc
->dev
->name
, lc
->logdev
->name
);
717 static int log_writes_prepare_ioctl(struct dm_target
*ti
,
718 struct block_device
**bdev
, fmode_t
*mode
)
720 struct log_writes_c
*lc
= ti
->private;
721 struct dm_dev
*dev
= lc
->dev
;
725 * Only pass ioctls through if the device sizes match exactly.
727 if (ti
->len
!= i_size_read(dev
->bdev
->bd_inode
) >> SECTOR_SHIFT
)
732 static int log_writes_iterate_devices(struct dm_target
*ti
,
733 iterate_devices_callout_fn fn
,
736 struct log_writes_c
*lc
= ti
->private;
738 return fn(ti
, lc
->dev
, 0, ti
->len
, data
);
742 * Messages supported:
743 * mark <mark data> - specify the marked data.
745 static int log_writes_message(struct dm_target
*ti
, unsigned argc
, char **argv
)
748 struct log_writes_c
*lc
= ti
->private;
751 DMWARN("Invalid log-writes message arguments, expect 2 arguments, got %d", argc
);
755 if (!strcasecmp(argv
[0], "mark"))
756 r
= log_mark(lc
, argv
[1]);
758 DMWARN("Unrecognised log writes target message received: %s", argv
[0]);
763 static void log_writes_io_hints(struct dm_target
*ti
, struct queue_limits
*limits
)
765 struct log_writes_c
*lc
= ti
->private;
766 struct request_queue
*q
= bdev_get_queue(lc
->dev
->bdev
);
768 if (!q
|| !blk_queue_discard(q
)) {
769 lc
->device_supports_discard
= false;
770 limits
->discard_granularity
= 1 << SECTOR_SHIFT
;
771 limits
->max_discard_sectors
= (UINT_MAX
>> SECTOR_SHIFT
);
775 static struct target_type log_writes_target
= {
776 .name
= "log-writes",
777 .version
= {1, 0, 0},
778 .module
= THIS_MODULE
,
779 .ctr
= log_writes_ctr
,
780 .dtr
= log_writes_dtr
,
781 .map
= log_writes_map
,
782 .end_io
= normal_end_io
,
783 .status
= log_writes_status
,
784 .prepare_ioctl
= log_writes_prepare_ioctl
,
785 .message
= log_writes_message
,
786 .iterate_devices
= log_writes_iterate_devices
,
787 .io_hints
= log_writes_io_hints
,
790 static int __init
dm_log_writes_init(void)
792 int r
= dm_register_target(&log_writes_target
);
795 DMERR("register failed %d", r
);
800 static void __exit
dm_log_writes_exit(void)
802 dm_unregister_target(&log_writes_target
);
805 module_init(dm_log_writes_init
);
806 module_exit(dm_log_writes_exit
);
808 MODULE_DESCRIPTION(DM_NAME
" log writes target");
809 MODULE_AUTHOR("Josef Bacik <jbacik@fb.com>");
810 MODULE_LICENSE("GPL");