?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
treewide: remove redundant IS_ERR() before error code check
[linux/fpc-iii.git] / drivers / md / dm-writecache.c
blobb9e27e37a94373c1cbc8ca52f3b64c0d72d46228
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2018 Red Hat. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8 #include <linux/device-mapper.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/vmalloc.h>
12 #include <linux/kthread.h>
13 #include <linux/dm-io.h>
14 #include <linux/dm-kcopyd.h>
15 #include <linux/dax.h>
16 #include <linux/pfn_t.h>
17 #include <linux/libnvdimm.h>
18
19 #define DM_MSG_PREFIX "writecache"
20
21 #define HIGH_WATERMARK 50
22 #define LOW_WATERMARK 45
23 #define MAX_WRITEBACK_JOBS 0
24 #define ENDIO_LATENCY 16
25 #define WRITEBACK_LATENCY 64
26 #define AUTOCOMMIT_BLOCKS_SSD 65536
27 #define AUTOCOMMIT_BLOCKS_PMEM 64
28 #define AUTOCOMMIT_MSEC 1000
29
30 #define BITMAP_GRANULARITY 65536
31 #if BITMAP_GRANULARITY < PAGE_SIZE
32 #undef BITMAP_GRANULARITY
33 #define BITMAP_GRANULARITY PAGE_SIZE
34 #endif
35
36 #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER)
37 #define DM_WRITECACHE_HAS_PMEM
38 #endif
39
40 #ifdef DM_WRITECACHE_HAS_PMEM
41 #define pmem_assign(dest, src) \
42 do { \
43 typeof(dest) uniq = (src); \
44 memcpy_flushcache(&(dest), &uniq, sizeof(dest)); \
45 } while (0)
46 #else
47 #define pmem_assign(dest, src) ((dest) = (src))
48 #endif
49
50 #if defined(__HAVE_ARCH_MEMCPY_MCSAFE) && defined(DM_WRITECACHE_HAS_PMEM)
51 #define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
52 #endif
53
54 #define MEMORY_SUPERBLOCK_MAGIC 0x23489321
55 #define MEMORY_SUPERBLOCK_VERSION 1
56
57 struct wc_memory_entry {
58 __le64 original_sector;
59 __le64 seq_count;
60 };
61
62 struct wc_memory_superblock {
63 union {
64 struct {
65 __le32 magic;
66 __le32 version;
67 __le32 block_size;
68 __le32 pad;
69 __le64 n_blocks;
70 __le64 seq_count;
71 };
72 __le64 padding[8];
73 };
74 struct wc_memory_entry entries[0];
75 };
76
77 struct wc_entry {
78 struct rb_node rb_node;
79 struct list_head lru;
80 unsigned short wc_list_contiguous;
81 bool write_in_progress
82 #if BITS_PER_LONG == 64
83 :1
84 #endif
85 ;
86 unsigned long index
87 #if BITS_PER_LONG == 64
88 :47
89 #endif
90 ;
91 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
92 uint64_t original_sector;
93 uint64_t seq_count;
94 #endif
95 };
96
97 #ifdef DM_WRITECACHE_HAS_PMEM
98 #define WC_MODE_PMEM(wc) ((wc)->pmem_mode)
99 #define WC_MODE_FUA(wc) ((wc)->writeback_fua)
100 #else
101 #define WC_MODE_PMEM(wc) false
102 #define WC_MODE_FUA(wc) false
103 #endif
104 #define WC_MODE_SORT_FREELIST(wc) (!WC_MODE_PMEM(wc))
105
106 struct dm_writecache {
107 struct mutex lock;
108 struct list_head lru;
109 union {
110 struct list_head freelist;
111 struct {
112 struct rb_root freetree;
113 struct wc_entry *current_free;
114 };
115 };
116 struct rb_root tree;
117
118 size_t freelist_size;
119 size_t writeback_size;
120 size_t freelist_high_watermark;
121 size_t freelist_low_watermark;
122
123 unsigned uncommitted_blocks;
124 unsigned autocommit_blocks;
125 unsigned max_writeback_jobs;
126
127 int error;
128
129 unsigned long autocommit_jiffies;
130 struct timer_list autocommit_timer;
131 struct wait_queue_head freelist_wait;
132
133 atomic_t bio_in_progress[2];
134 struct wait_queue_head bio_in_progress_wait[2];
135
136 struct dm_target *ti;
137 struct dm_dev *dev;
138 struct dm_dev *ssd_dev;
139 sector_t start_sector;
140 void *memory_map;
141 uint64_t memory_map_size;
142 size_t metadata_sectors;
143 size_t n_blocks;
144 uint64_t seq_count;
145 void *block_start;
146 struct wc_entry *entries;
147 unsigned block_size;
148 unsigned char block_size_bits;
149
150 bool pmem_mode:1;
151 bool writeback_fua:1;
152
153 bool overwrote_committed:1;
154 bool memory_vmapped:1;
155
156 bool high_wm_percent_set:1;
157 bool low_wm_percent_set:1;
158 bool max_writeback_jobs_set:1;
159 bool autocommit_blocks_set:1;
160 bool autocommit_time_set:1;
161 bool writeback_fua_set:1;
162 bool flush_on_suspend:1;
163
164 unsigned writeback_all;
165 struct workqueue_struct *writeback_wq;
166 struct work_struct writeback_work;
167 struct work_struct flush_work;
168
169 struct dm_io_client *dm_io;
170
171 raw_spinlock_t endio_list_lock;
172 struct list_head endio_list;
173 struct task_struct *endio_thread;
174
175 struct task_struct *flush_thread;
176 struct bio_list flush_list;
177
178 struct dm_kcopyd_client *dm_kcopyd;
179 unsigned long *dirty_bitmap;
180 unsigned dirty_bitmap_size;
181
182 struct bio_set bio_set;
183 mempool_t copy_pool;
184 };
185
186 #define WB_LIST_INLINE 16
187
188 struct writeback_struct {
189 struct list_head endio_entry;
190 struct dm_writecache *wc;
191 struct wc_entry **wc_list;
192 unsigned wc_list_n;
193 struct wc_entry *wc_list_inline[WB_LIST_INLINE];
194 struct bio bio;
195 };
196
197 struct copy_struct {
198 struct list_head endio_entry;
199 struct dm_writecache *wc;
200 struct wc_entry *e;
201 unsigned n_entries;
202 int error;
203 };
204
205 DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,
206 "A percentage of time allocated for data copying");
207
208 static void wc_lock(struct dm_writecache *wc)
209 {
210 mutex_lock(&wc->lock);
211 }
212
213 static void wc_unlock(struct dm_writecache *wc)
214 {
215 mutex_unlock(&wc->lock);
216 }
217
218 #ifdef DM_WRITECACHE_HAS_PMEM
219 static int persistent_memory_claim(struct dm_writecache *wc)
220 {
221 int r;
222 loff_t s;
223 long p, da;
224 pfn_t pfn;
225 int id;
226 struct page **pages;
227
228 wc->memory_vmapped = false;
229
230 if (!wc->ssd_dev->dax_dev) {
231 r = -EOPNOTSUPP;
232 goto err1;
233 }
234 s = wc->memory_map_size;
235 p = s >> PAGE_SHIFT;
236 if (!p) {
237 r = -EINVAL;
238 goto err1;
239 }
240 if (p != s >> PAGE_SHIFT) {
241 r = -EOVERFLOW;
242 goto err1;
243 }
244
245 id = dax_read_lock();
246
247 da = dax_direct_access(wc->ssd_dev->dax_dev, 0, p, &wc->memory_map, &pfn);
248 if (da < 0) {
249 wc->memory_map = NULL;
250 r = da;
251 goto err2;
252 }
253 if (!pfn_t_has_page(pfn)) {
254 wc->memory_map = NULL;
255 r = -EOPNOTSUPP;
256 goto err2;
257 }
258 if (da != p) {
259 long i;
260 wc->memory_map = NULL;
261 pages = kvmalloc_array(p, sizeof(struct page *), GFP_KERNEL);
262 if (!pages) {
263 r = -ENOMEM;
264 goto err2;
265 }
266 i = 0;
267 do {
268 long daa;
269 daa = dax_direct_access(wc->ssd_dev->dax_dev, i, p - i,
270 NULL, &pfn);
271 if (daa <= 0) {
272 r = daa ? daa : -EINVAL;
273 goto err3;
274 }
275 if (!pfn_t_has_page(pfn)) {
276 r = -EOPNOTSUPP;
277 goto err3;
278 }
279 while (daa-- && i < p) {
280 pages[i++] = pfn_t_to_page(pfn);
281 pfn.val++;
282 }
283 } while (i < p);
284 wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);
285 if (!wc->memory_map) {
286 r = -ENOMEM;
287 goto err3;
288 }
289 kvfree(pages);
290 wc->memory_vmapped = true;
291 }
292
293 dax_read_unlock(id);
294
295 wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;
296 wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT;
297
298 return 0;
299 err3:
300 kvfree(pages);
301 err2:
302 dax_read_unlock(id);
303 err1:
304 return r;
305 }
306 #else
307 static int persistent_memory_claim(struct dm_writecache *wc)
308 {
309 BUG();
310 }
311 #endif
312
313 static void persistent_memory_release(struct dm_writecache *wc)
314 {
315 if (wc->memory_vmapped)
316 vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));
317 }
318
319 static struct page *persistent_memory_page(void *addr)
320 {
321 if (is_vmalloc_addr(addr))
322 return vmalloc_to_page(addr);
323 else
324 return virt_to_page(addr);
325 }
326
327 static unsigned persistent_memory_page_offset(void *addr)
328 {
329 return (unsigned long)addr & (PAGE_SIZE - 1);
330 }
331
332 static void persistent_memory_flush_cache(void *ptr, size_t size)
333 {
334 if (is_vmalloc_addr(ptr))
335 flush_kernel_vmap_range(ptr, size);
336 }
337
338 static void persistent_memory_invalidate_cache(void *ptr, size_t size)
339 {
340 if (is_vmalloc_addr(ptr))
341 invalidate_kernel_vmap_range(ptr, size);
342 }
343
344 static struct wc_memory_superblock *sb(struct dm_writecache *wc)
345 {
346 return wc->memory_map;
347 }
348
349 static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
350 {
351 return &sb(wc)->entries[e->index];
352 }
353
354 static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
355 {
356 return (char *)wc->block_start + (e->index << wc->block_size_bits);
357 }
358
359 static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
360 {
361 return wc->start_sector + wc->metadata_sectors +
362 ((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));
363 }
364
365 static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
366 {
367 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
368 return e->original_sector;
369 #else
370 return le64_to_cpu(memory_entry(wc, e)->original_sector);
371 #endif
372 }
373
374 static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
375 {
376 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
377 return e->seq_count;
378 #else
379 return le64_to_cpu(memory_entry(wc, e)->seq_count);
380 #endif
381 }
382
383 static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
384 {
385 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
386 e->seq_count = -1;
387 #endif
388 pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));
389 }
390
391 static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
392 uint64_t original_sector, uint64_t seq_count)
393 {
394 struct wc_memory_entry me;
395 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
396 e->original_sector = original_sector;
397 e->seq_count = seq_count;
398 #endif
399 me.original_sector = cpu_to_le64(original_sector);
400 me.seq_count = cpu_to_le64(seq_count);
401 pmem_assign(*memory_entry(wc, e), me);
402 }
403
404 #define writecache_error(wc, err, msg, arg...) \
405 do { \
406 if (!cmpxchg(&(wc)->error, 0, err)) \
407 DMERR(msg, ##arg); \
408 wake_up(&(wc)->freelist_wait); \
409 } while (0)
410
411 #define writecache_has_error(wc) (unlikely(READ_ONCE((wc)->error)))
412
413 static void writecache_flush_all_metadata(struct dm_writecache *wc)
414 {
415 if (!WC_MODE_PMEM(wc))
416 memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);
417 }
418
419 static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
420 {
421 if (!WC_MODE_PMEM(wc))
422 __set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,
423 wc->dirty_bitmap);
424 }
425
426 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);
427
428 struct io_notify {
429 struct dm_writecache *wc;
430 struct completion c;
431 atomic_t count;
432 };
433
434 static void writecache_notify_io(unsigned long error, void *context)
435 {
436 struct io_notify *endio = context;
437
438 if (unlikely(error != 0))
439 writecache_error(endio->wc, -EIO, "error writing metadata");
440 BUG_ON(atomic_read(&endio->count) <= 0);
441 if (atomic_dec_and_test(&endio->count))
442 complete(&endio->c);
443 }
444
445 static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
446 {
447 wait_event(wc->bio_in_progress_wait[direction],
448 !atomic_read(&wc->bio_in_progress[direction]));
449 }
450
451 static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
452 {
453 struct dm_io_region region;
454 struct dm_io_request req;
455 struct io_notify endio = {
456 wc,
457 COMPLETION_INITIALIZER_ONSTACK(endio.c),
458 ATOMIC_INIT(1),
459 };
460 unsigned bitmap_bits = wc->dirty_bitmap_size * 8;
461 unsigned i = 0;
462
463 while (1) {
464 unsigned j;
465 i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);
466 if (unlikely(i == bitmap_bits))
467 break;
468 j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i);
469
470 region.bdev = wc->ssd_dev->bdev;
471 region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
472 region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
473
474 if (unlikely(region.sector >= wc->metadata_sectors))
475 break;
476 if (unlikely(region.sector + region.count > wc->metadata_sectors))
477 region.count = wc->metadata_sectors - region.sector;
478
479 region.sector += wc->start_sector;
480 atomic_inc(&endio.count);
481 req.bi_op = REQ_OP_WRITE;
482 req.bi_op_flags = REQ_SYNC;
483 req.mem.type = DM_IO_VMA;
484 req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
485 req.client = wc->dm_io;
486 req.notify.fn = writecache_notify_io;
487 req.notify.context = &endio;
488
489 /* writing via async dm-io (implied by notify.fn above) won't return an error */
490 (void) dm_io(&req, 1, &region, NULL);
491 i = j;
492 }
493
494 writecache_notify_io(0, &endio);
495 wait_for_completion_io(&endio.c);
496
497 if (wait_for_ios)
498 writecache_wait_for_ios(wc, WRITE);
499
500 writecache_disk_flush(wc, wc->ssd_dev);
501
502 memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
503 }
504
505 static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
506 {
507 if (WC_MODE_PMEM(wc))
508 wmb();
509 else
510 ssd_commit_flushed(wc, wait_for_ios);
511 }
512
513 static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
514 {
515 int r;
516 struct dm_io_region region;
517 struct dm_io_request req;
518
519 region.bdev = dev->bdev;
520 region.sector = 0;
521 region.count = 0;
522 req.bi_op = REQ_OP_WRITE;
523 req.bi_op_flags = REQ_PREFLUSH;
524 req.mem.type = DM_IO_KMEM;
525 req.mem.ptr.addr = NULL;
526 req.client = wc->dm_io;
527 req.notify.fn = NULL;
528
529 r = dm_io(&req, 1, &region, NULL);
530 if (unlikely(r))
531 writecache_error(wc, r, "error flushing metadata: %d", r);
532 }
533
534 #define WFE_RETURN_FOLLOWING 1
535 #define WFE_LOWEST_SEQ 2
536
537 static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
538 uint64_t block, int flags)
539 {
540 struct wc_entry *e;
541 struct rb_node *node = wc->tree.rb_node;
542
543 if (unlikely(!node))
544 return NULL;
545
546 while (1) {
547 e = container_of(node, struct wc_entry, rb_node);
548 if (read_original_sector(wc, e) == block)
549 break;
550
551 node = (read_original_sector(wc, e) >= block ?
552 e->rb_node.rb_left : e->rb_node.rb_right);
553 if (unlikely(!node)) {
554 if (!(flags & WFE_RETURN_FOLLOWING))
555 return NULL;
556 if (read_original_sector(wc, e) >= block) {
557 return e;
558 } else {
559 node = rb_next(&e->rb_node);
560 if (unlikely(!node))
561 return NULL;
562 e = container_of(node, struct wc_entry, rb_node);
563 return e;
564 }
565 }
566 }
567
568 while (1) {
569 struct wc_entry *e2;
570 if (flags & WFE_LOWEST_SEQ)
571 node = rb_prev(&e->rb_node);
572 else
573 node = rb_next(&e->rb_node);
574 if (unlikely(!node))
575 return e;
576 e2 = container_of(node, struct wc_entry, rb_node);
577 if (read_original_sector(wc, e2) != block)
578 return e;
579 e = e2;
580 }
581 }
582
583 static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
584 {
585 struct wc_entry *e;
586 struct rb_node **node = &wc->tree.rb_node, *parent = NULL;
587
588 while (*node) {
589 e = container_of(*node, struct wc_entry, rb_node);
590 parent = &e->rb_node;
591 if (read_original_sector(wc, e) > read_original_sector(wc, ins))
592 node = &parent->rb_left;
593 else
594 node = &parent->rb_right;
595 }
596 rb_link_node(&ins->rb_node, parent, node);
597 rb_insert_color(&ins->rb_node, &wc->tree);
598 list_add(&ins->lru, &wc->lru);
599 }
600
601 static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
602 {
603 list_del(&e->lru);
604 rb_erase(&e->rb_node, &wc->tree);
605 }
606
607 static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
608 {
609 if (WC_MODE_SORT_FREELIST(wc)) {
610 struct rb_node **node = &wc->freetree.rb_node, *parent = NULL;
611 if (unlikely(!*node))
612 wc->current_free = e;
613 while (*node) {
614 parent = *node;
615 if (&e->rb_node < *node)
616 node = &parent->rb_left;
617 else
618 node = &parent->rb_right;
619 }
620 rb_link_node(&e->rb_node, parent, node);
621 rb_insert_color(&e->rb_node, &wc->freetree);
622 } else {
623 list_add_tail(&e->lru, &wc->freelist);
624 }
625 wc->freelist_size++;
626 }
627
628 static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
629 {
630 struct wc_entry *e;
631
632 if (WC_MODE_SORT_FREELIST(wc)) {
633 struct rb_node *next;
634 if (unlikely(!wc->current_free))
635 return NULL;
636 e = wc->current_free;
637 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
638 return NULL;
639 next = rb_next(&e->rb_node);
640 rb_erase(&e->rb_node, &wc->freetree);
641 if (unlikely(!next))
642 next = rb_first(&wc->freetree);
643 wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL;
644 } else {
645 if (unlikely(list_empty(&wc->freelist)))
646 return NULL;
647 e = container_of(wc->freelist.next, struct wc_entry, lru);
648 if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
649 return NULL;
650 list_del(&e->lru);
651 }
652 wc->freelist_size--;
653 if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
654 queue_work(wc->writeback_wq, &wc->writeback_work);
655
656 return e;
657 }
658
659 static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
660 {
661 writecache_unlink(wc, e);
662 writecache_add_to_freelist(wc, e);
663 clear_seq_count(wc, e);
664 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
665 if (unlikely(waitqueue_active(&wc->freelist_wait)))
666 wake_up(&wc->freelist_wait);
667 }
668
669 static void writecache_wait_on_freelist(struct dm_writecache *wc)
670 {
671 DEFINE_WAIT(wait);
672
673 prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE);
674 wc_unlock(wc);
675 io_schedule();
676 finish_wait(&wc->freelist_wait, &wait);
677 wc_lock(wc);
678 }
679
680 static void writecache_poison_lists(struct dm_writecache *wc)
681 {
682 /*
683 * Catch incorrect access to these values while the device is suspended.
684 */
685 memset(&wc->tree, -1, sizeof wc->tree);
686 wc->lru.next = LIST_POISON1;
687 wc->lru.prev = LIST_POISON2;
688 wc->freelist.next = LIST_POISON1;
689 wc->freelist.prev = LIST_POISON2;
690 }
691
692 static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
693 {
694 writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
695 if (WC_MODE_PMEM(wc))
696 writecache_flush_region(wc, memory_data(wc, e), wc->block_size);
697 }
698
699 static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
700 {
701 return read_seq_count(wc, e) < wc->seq_count;
702 }
703
704 static void writecache_flush(struct dm_writecache *wc)
705 {
706 struct wc_entry *e, *e2;
707 bool need_flush_after_free;
708
709 wc->uncommitted_blocks = 0;
710 del_timer(&wc->autocommit_timer);
711
712 if (list_empty(&wc->lru))
713 return;
714
715 e = container_of(wc->lru.next, struct wc_entry, lru);
716 if (writecache_entry_is_committed(wc, e)) {
717 if (wc->overwrote_committed) {
718 writecache_wait_for_ios(wc, WRITE);
719 writecache_disk_flush(wc, wc->ssd_dev);
720 wc->overwrote_committed = false;
721 }
722 return;
723 }
724 while (1) {
725 writecache_flush_entry(wc, e);
726 if (unlikely(e->lru.next == &wc->lru))
727 break;
728 e2 = container_of(e->lru.next, struct wc_entry, lru);
729 if (writecache_entry_is_committed(wc, e2))
730 break;
731 e = e2;
732 cond_resched();
733 }
734 writecache_commit_flushed(wc, true);
735
736 wc->seq_count++;
737 pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
738 writecache_flush_region(wc, &sb(wc)->seq_count, sizeof sb(wc)->seq_count);
739 writecache_commit_flushed(wc, false);
740
741 wc->overwrote_committed = false;
742
743 need_flush_after_free = false;
744 while (1) {
745 /* Free another committed entry with lower seq-count */
746 struct rb_node *rb_node = rb_prev(&e->rb_node);
747
748 if (rb_node) {
749 e2 = container_of(rb_node, struct wc_entry, rb_node);
750 if (read_original_sector(wc, e2) == read_original_sector(wc, e) &&
751 likely(!e2->write_in_progress)) {
752 writecache_free_entry(wc, e2);
753 need_flush_after_free = true;
754 }
755 }
756 if (unlikely(e->lru.prev == &wc->lru))
757 break;
758 e = container_of(e->lru.prev, struct wc_entry, lru);
759 cond_resched();
760 }
761
762 if (need_flush_after_free)
763 writecache_commit_flushed(wc, false);
764 }
765
766 static void writecache_flush_work(struct work_struct *work)
767 {
768 struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work);
769
770 wc_lock(wc);
771 writecache_flush(wc);
772 wc_unlock(wc);
773 }
774
775 static void writecache_autocommit_timer(struct timer_list *t)
776 {
777 struct dm_writecache *wc = from_timer(wc, t, autocommit_timer);
778 if (!writecache_has_error(wc))
779 queue_work(wc->writeback_wq, &wc->flush_work);
780 }
781
782 static void writecache_schedule_autocommit(struct dm_writecache *wc)
783 {
784 if (!timer_pending(&wc->autocommit_timer))
785 mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies);
786 }
787
788 static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
789 {
790 struct wc_entry *e;
791 bool discarded_something = false;
792
793 e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ);
794 if (unlikely(!e))
795 return;
796
797 while (read_original_sector(wc, e) < end) {
798 struct rb_node *node = rb_next(&e->rb_node);
799
800 if (likely(!e->write_in_progress)) {
801 if (!discarded_something) {
802 writecache_wait_for_ios(wc, READ);
803 writecache_wait_for_ios(wc, WRITE);
804 discarded_something = true;
805 }
806 writecache_free_entry(wc, e);
807 }
808
809 if (unlikely(!node))
810 break;
811
812 e = container_of(node, struct wc_entry, rb_node);
813 }
814
815 if (discarded_something)
816 writecache_commit_flushed(wc, false);
817 }
818
819 static bool writecache_wait_for_writeback(struct dm_writecache *wc)
820 {
821 if (wc->writeback_size) {
822 writecache_wait_on_freelist(wc);
823 return true;
824 }
825 return false;
826 }
827
828 static void writecache_suspend(struct dm_target *ti)
829 {
830 struct dm_writecache *wc = ti->private;
831 bool flush_on_suspend;
832
833 del_timer_sync(&wc->autocommit_timer);
834
835 wc_lock(wc);
836 writecache_flush(wc);
837 flush_on_suspend = wc->flush_on_suspend;
838 if (flush_on_suspend) {
839 wc->flush_on_suspend = false;
840 wc->writeback_all++;
841 queue_work(wc->writeback_wq, &wc->writeback_work);
842 }
843 wc_unlock(wc);
844
845 flush_workqueue(wc->writeback_wq);
846
847 wc_lock(wc);
848 if (flush_on_suspend)
849 wc->writeback_all--;
850 while (writecache_wait_for_writeback(wc));
851
852 if (WC_MODE_PMEM(wc))
853 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
854
855 writecache_poison_lists(wc);
856
857 wc_unlock(wc);
858 }
859
860 static int writecache_alloc_entries(struct dm_writecache *wc)
861 {
862 size_t b;
863
864 if (wc->entries)
865 return 0;
866 wc->entries = vmalloc(array_size(sizeof(struct wc_entry), wc->n_blocks));
867 if (!wc->entries)
868 return -ENOMEM;
869 for (b = 0; b < wc->n_blocks; b++) {
870 struct wc_entry *e = &wc->entries[b];
871 e->index = b;
872 e->write_in_progress = false;
873 }
874
875 return 0;
876 }
877
878 static void writecache_resume(struct dm_target *ti)
879 {
880 struct dm_writecache *wc = ti->private;
881 size_t b;
882 bool need_flush = false;
883 __le64 sb_seq_count;
884 int r;
885
886 wc_lock(wc);
887
888 if (WC_MODE_PMEM(wc))
889 persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size);
890
891 wc->tree = RB_ROOT;
892 INIT_LIST_HEAD(&wc->lru);
893 if (WC_MODE_SORT_FREELIST(wc)) {
894 wc->freetree = RB_ROOT;
895 wc->current_free = NULL;
896 } else {
897 INIT_LIST_HEAD(&wc->freelist);
898 }
899 wc->freelist_size = 0;
900
901 r = memcpy_mcsafe(&sb_seq_count, &sb(wc)->seq_count, sizeof(uint64_t));
902 if (r) {
903 writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
904 sb_seq_count = cpu_to_le64(0);
905 }
906 wc->seq_count = le64_to_cpu(sb_seq_count);
907
908 #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
909 for (b = 0; b < wc->n_blocks; b++) {
910 struct wc_entry *e = &wc->entries[b];
911 struct wc_memory_entry wme;
912 if (writecache_has_error(wc)) {
913 e->original_sector = -1;
914 e->seq_count = -1;
915 continue;
916 }
917 r = memcpy_mcsafe(&wme, memory_entry(wc, e), sizeof(struct wc_memory_entry));
918 if (r) {
919 writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
920 (unsigned long)b, r);
921 e->original_sector = -1;
922 e->seq_count = -1;
923 } else {
924 e->original_sector = le64_to_cpu(wme.original_sector);
925 e->seq_count = le64_to_cpu(wme.seq_count);
926 }
927 }
928 #endif
929 for (b = 0; b < wc->n_blocks; b++) {
930 struct wc_entry *e = &wc->entries[b];
931 if (!writecache_entry_is_committed(wc, e)) {
932 if (read_seq_count(wc, e) != -1) {
933 erase_this:
934 clear_seq_count(wc, e);
935 need_flush = true;
936 }
937 writecache_add_to_freelist(wc, e);
938 } else {
939 struct wc_entry *old;
940
941 old = writecache_find_entry(wc, read_original_sector(wc, e), 0);
942 if (!old) {
943 writecache_insert_entry(wc, e);
944 } else {
945 if (read_seq_count(wc, old) == read_seq_count(wc, e)) {
946 writecache_error(wc, -EINVAL,
947 "two identical entries, position %llu, sector %llu, sequence %llu",
948 (unsigned long long)b, (unsigned long long)read_original_sector(wc, e),
949 (unsigned long long)read_seq_count(wc, e));
950 }
951 if (read_seq_count(wc, old) > read_seq_count(wc, e)) {
952 goto erase_this;
953 } else {
954 writecache_free_entry(wc, old);
955 writecache_insert_entry(wc, e);
956 need_flush = true;
957 }
958 }
959 }
960 cond_resched();
961 }
962
963 if (need_flush) {
964 writecache_flush_all_metadata(wc);
965 writecache_commit_flushed(wc, false);
966 }
967
968 wc_unlock(wc);
969 }
970
971 static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
972 {
973 if (argc != 1)
974 return -EINVAL;
975
976 wc_lock(wc);
977 if (dm_suspended(wc->ti)) {
978 wc_unlock(wc);
979 return -EBUSY;
980 }
981 if (writecache_has_error(wc)) {
982 wc_unlock(wc);
983 return -EIO;
984 }
985
986 writecache_flush(wc);
987 wc->writeback_all++;
988 queue_work(wc->writeback_wq, &wc->writeback_work);
989 wc_unlock(wc);
990
991 flush_workqueue(wc->writeback_wq);
992
993 wc_lock(wc);
994 wc->writeback_all--;
995 if (writecache_has_error(wc)) {
996 wc_unlock(wc);
997 return -EIO;
998 }
999 wc_unlock(wc);
1000
1001 return 0;
1002 }
1003
1004 static int process_flush_on_suspend_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
1005 {
1006 if (argc != 1)
1007 return -EINVAL;
1008
1009 wc_lock(wc);
1010 wc->flush_on_suspend = true;
1011 wc_unlock(wc);
1012
1013 return 0;
1014 }
1015
1016 static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
1017 char *result, unsigned maxlen)
1018 {
1019 int r = -EINVAL;
1020 struct dm_writecache *wc = ti->private;
1021
1022 if (!strcasecmp(argv[0], "flush"))
1023 r = process_flush_mesg(argc, argv, wc);
1024 else if (!strcasecmp(argv[0], "flush_on_suspend"))
1025 r = process_flush_on_suspend_mesg(argc, argv, wc);
1026 else
1027 DMERR("unrecognised message received: %s", argv[0]);
1028
1029 return r;
1030 }
1031
1032 static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
1033 {
1034 void *buf;
1035 unsigned long flags;
1036 unsigned size;
1037 int rw = bio_data_dir(bio);
1038 unsigned remaining_size = wc->block_size;
1039
1040 do {
1041 struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
1042 buf = bvec_kmap_irq(&bv, &flags);
1043 size = bv.bv_len;
1044 if (unlikely(size > remaining_size))
1045 size = remaining_size;
1046
1047 if (rw == READ) {
1048 int r;
1049 r = memcpy_mcsafe(buf, data, size);
1050 flush_dcache_page(bio_page(bio));
1051 if (unlikely(r)) {
1052 writecache_error(wc, r, "hardware memory error when reading data: %d", r);
1053 bio->bi_status = BLK_STS_IOERR;
1054 }
1055 } else {
1056 flush_dcache_page(bio_page(bio));
1057 memcpy_flushcache(data, buf, size);
1058 }
1059
1060 bvec_kunmap_irq(buf, &flags);
1061
1062 data = (char *)data + size;
1063 remaining_size -= size;
1064 bio_advance(bio, size);
1065 } while (unlikely(remaining_size));
1066 }
1067
1068 static int writecache_flush_thread(void *data)
1069 {
1070 struct dm_writecache *wc = data;
1071
1072 while (1) {
1073 struct bio *bio;
1074
1075 wc_lock(wc);
1076 bio = bio_list_pop(&wc->flush_list);
1077 if (!bio) {
1078 set_current_state(TASK_INTERRUPTIBLE);
1079 wc_unlock(wc);
1080
1081 if (unlikely(kthread_should_stop())) {
1082 set_current_state(TASK_RUNNING);
1083 break;
1084 }
1085
1086 schedule();
1087 continue;
1088 }
1089
1090 if (bio_op(bio) == REQ_OP_DISCARD) {
1091 writecache_discard(wc, bio->bi_iter.bi_sector,
1092 bio_end_sector(bio));
1093 wc_unlock(wc);
1094 bio_set_dev(bio, wc->dev->bdev);
1095 generic_make_request(bio);
1096 } else {
1097 writecache_flush(wc);
1098 wc_unlock(wc);
1099 if (writecache_has_error(wc))
1100 bio->bi_status = BLK_STS_IOERR;
1101 bio_endio(bio);
1102 }
1103 }
1104
1105 return 0;
1106 }
1107
1108 static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
1109 {
1110 if (bio_list_empty(&wc->flush_list))
1111 wake_up_process(wc->flush_thread);
1112 bio_list_add(&wc->flush_list, bio);
1113 }
1114
1115 static int writecache_map(struct dm_target *ti, struct bio *bio)
1116 {
1117 struct wc_entry *e;
1118 struct dm_writecache *wc = ti->private;
1119
1120 bio->bi_private = NULL;
1121
1122 wc_lock(wc);
1123
1124 if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
1125 if (writecache_has_error(wc))
1126 goto unlock_error;
1127 if (WC_MODE_PMEM(wc)) {
1128 writecache_flush(wc);
1129 if (writecache_has_error(wc))
1130 goto unlock_error;
1131 goto unlock_submit;
1132 } else {
1133 writecache_offload_bio(wc, bio);
1134 goto unlock_return;
1135 }
1136 }
1137
1138 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1139
1140 if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
1141 (wc->block_size / 512 - 1)) != 0)) {
1142 DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
1143 (unsigned long long)bio->bi_iter.bi_sector,
1144 bio->bi_iter.bi_size, wc->block_size);
1145 goto unlock_error;
1146 }
1147
1148 if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
1149 if (writecache_has_error(wc))
1150 goto unlock_error;
1151 if (WC_MODE_PMEM(wc)) {
1152 writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
1153 goto unlock_remap_origin;
1154 } else {
1155 writecache_offload_bio(wc, bio);
1156 goto unlock_return;
1157 }
1158 }
1159
1160 if (bio_data_dir(bio) == READ) {
1161 read_next_block:
1162 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
1163 if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
1164 if (WC_MODE_PMEM(wc)) {
1165 bio_copy_block(wc, bio, memory_data(wc, e));
1166 if (bio->bi_iter.bi_size)
1167 goto read_next_block;
1168 goto unlock_submit;
1169 } else {
1170 dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
1171 bio_set_dev(bio, wc->ssd_dev->bdev);
1172 bio->bi_iter.bi_sector = cache_sector(wc, e);
1173 if (!writecache_entry_is_committed(wc, e))
1174 writecache_wait_for_ios(wc, WRITE);
1175 goto unlock_remap;
1176 }
1177 } else {
1178 if (e) {
1179 sector_t next_boundary =
1180 read_original_sector(wc, e) - bio->bi_iter.bi_sector;
1181 if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
1182 dm_accept_partial_bio(bio, next_boundary);
1183 }
1184 }
1185 goto unlock_remap_origin;
1186 }
1187 } else {
1188 do {
1189 if (writecache_has_error(wc))
1190 goto unlock_error;
1191 e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
1192 if (e) {
1193 if (!writecache_entry_is_committed(wc, e))
1194 goto bio_copy;
1195 if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
1196 wc->overwrote_committed = true;
1197 goto bio_copy;
1198 }
1199 }
1200 e = writecache_pop_from_freelist(wc, (sector_t)-1);
1201 if (unlikely(!e)) {
1202 writecache_wait_on_freelist(wc);
1203 continue;
1204 }
1205 write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
1206 writecache_insert_entry(wc, e);
1207 wc->uncommitted_blocks++;
1208 bio_copy:
1209 if (WC_MODE_PMEM(wc)) {
1210 bio_copy_block(wc, bio, memory_data(wc, e));
1211 } else {
1212 unsigned bio_size = wc->block_size;
1213 sector_t start_cache_sec = cache_sector(wc, e);
1214 sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
1215
1216 while (bio_size < bio->bi_iter.bi_size) {
1217 struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
1218 if (!f)
1219 break;
1220 write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
1221 (bio_size >> SECTOR_SHIFT), wc->seq_count);
1222 writecache_insert_entry(wc, f);
1223 wc->uncommitted_blocks++;
1224 bio_size += wc->block_size;
1225 current_cache_sec += wc->block_size >> SECTOR_SHIFT;
1226 }
1227
1228 bio_set_dev(bio, wc->ssd_dev->bdev);
1229 bio->bi_iter.bi_sector = start_cache_sec;
1230 dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
1231
1232 if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
1233 wc->uncommitted_blocks = 0;
1234 queue_work(wc->writeback_wq, &wc->flush_work);
1235 } else {
1236 writecache_schedule_autocommit(wc);
1237 }
1238 goto unlock_remap;
1239 }
1240 } while (bio->bi_iter.bi_size);
1241
1242 if (unlikely(bio->bi_opf & REQ_FUA ||
1243 wc->uncommitted_blocks >= wc->autocommit_blocks))
1244 writecache_flush(wc);
1245 else
1246 writecache_schedule_autocommit(wc);
1247 goto unlock_submit;
1248 }
1249
1250 unlock_remap_origin:
1251 bio_set_dev(bio, wc->dev->bdev);
1252 wc_unlock(wc);
1253 return DM_MAPIO_REMAPPED;
1254
1255 unlock_remap:
1256 /* make sure that writecache_end_io decrements bio_in_progress: */
1257 bio->bi_private = (void *)1;
1258 atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
1259 wc_unlock(wc);
1260 return DM_MAPIO_REMAPPED;
1261
1262 unlock_submit:
1263 wc_unlock(wc);
1264 bio_endio(bio);
1265 return DM_MAPIO_SUBMITTED;
1266
1267 unlock_return:
1268 wc_unlock(wc);
1269 return DM_MAPIO_SUBMITTED;
1270
1271 unlock_error:
1272 wc_unlock(wc);
1273 bio_io_error(bio);
1274 return DM_MAPIO_SUBMITTED;
1275 }
1276
1277 static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
1278 {
1279 struct dm_writecache *wc = ti->private;
1280
1281 if (bio->bi_private != NULL) {
1282 int dir = bio_data_dir(bio);
1283 if (atomic_dec_and_test(&wc->bio_in_progress[dir]))
1284 if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir])))
1285 wake_up(&wc->bio_in_progress_wait[dir]);
1286 }
1287 return 0;
1288 }
1289
1290 static int writecache_iterate_devices(struct dm_target *ti,
1291 iterate_devices_callout_fn fn, void *data)
1292 {
1293 struct dm_writecache *wc = ti->private;
1294
1295 return fn(ti, wc->dev, 0, ti->len, data);
1296 }
1297
1298 static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
1299 {
1300 struct dm_writecache *wc = ti->private;
1301
1302 if (limits->logical_block_size < wc->block_size)
1303 limits->logical_block_size = wc->block_size;
1304
1305 if (limits->physical_block_size < wc->block_size)
1306 limits->physical_block_size = wc->block_size;
1307
1308 if (limits->io_min < wc->block_size)
1309 limits->io_min = wc->block_size;
1310 }
1311
1312
1313 static void writecache_writeback_endio(struct bio *bio)
1314 {
1315 struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio);
1316 struct dm_writecache *wc = wb->wc;
1317 unsigned long flags;
1318
1319 raw_spin_lock_irqsave(&wc->endio_list_lock, flags);
1320 if (unlikely(list_empty(&wc->endio_list)))
1321 wake_up_process(wc->endio_thread);
1322 list_add_tail(&wb->endio_entry, &wc->endio_list);
1323 raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags);
1324 }
1325
1326 static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
1327 {
1328 struct copy_struct *c = ptr;
1329 struct dm_writecache *wc = c->wc;
1330
1331 c->error = likely(!(read_err | write_err)) ? 0 : -EIO;
1332
1333 raw_spin_lock_irq(&wc->endio_list_lock);
1334 if (unlikely(list_empty(&wc->endio_list)))
1335 wake_up_process(wc->endio_thread);
1336 list_add_tail(&c->endio_entry, &wc->endio_list);
1337 raw_spin_unlock_irq(&wc->endio_list_lock);
1338 }
1339
1340 static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
1341 {
1342 unsigned i;
1343 struct writeback_struct *wb;
1344 struct wc_entry *e;
1345 unsigned long n_walked = 0;
1346
1347 do {
1348 wb = list_entry(list->next, struct writeback_struct, endio_entry);
1349 list_del(&wb->endio_entry);
1350
1351 if (unlikely(wb->bio.bi_status != BLK_STS_OK))
1352 writecache_error(wc, blk_status_to_errno(wb->bio.bi_status),
1353 "write error %d", wb->bio.bi_status);
1354 i = 0;
1355 do {
1356 e = wb->wc_list[i];
1357 BUG_ON(!e->write_in_progress);
1358 e->write_in_progress = false;
1359 INIT_LIST_HEAD(&e->lru);
1360 if (!writecache_has_error(wc))
1361 writecache_free_entry(wc, e);
1362 BUG_ON(!wc->writeback_size);
1363 wc->writeback_size--;
1364 n_walked++;
1365 if (unlikely(n_walked >= ENDIO_LATENCY)) {
1366 writecache_commit_flushed(wc, false);
1367 wc_unlock(wc);
1368 wc_lock(wc);
1369 n_walked = 0;
1370 }
1371 } while (++i < wb->wc_list_n);
1372
1373 if (wb->wc_list != wb->wc_list_inline)
1374 kfree(wb->wc_list);
1375 bio_put(&wb->bio);
1376 } while (!list_empty(list));
1377 }
1378
1379 static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
1380 {
1381 struct copy_struct *c;
1382 struct wc_entry *e;
1383
1384 do {
1385 c = list_entry(list->next, struct copy_struct, endio_entry);
1386 list_del(&c->endio_entry);
1387
1388 if (unlikely(c->error))
1389 writecache_error(wc, c->error, "copy error");
1390
1391 e = c->e;
1392 do {
1393 BUG_ON(!e->write_in_progress);
1394 e->write_in_progress = false;
1395 INIT_LIST_HEAD(&e->lru);
1396 if (!writecache_has_error(wc))
1397 writecache_free_entry(wc, e);
1398
1399 BUG_ON(!wc->writeback_size);
1400 wc->writeback_size--;
1401 e++;
1402 } while (--c->n_entries);
1403 mempool_free(c, &wc->copy_pool);
1404 } while (!list_empty(list));
1405 }
1406
1407 static int writecache_endio_thread(void *data)
1408 {
1409 struct dm_writecache *wc = data;
1410
1411 while (1) {
1412 struct list_head list;
1413
1414 raw_spin_lock_irq(&wc->endio_list_lock);
1415 if (!list_empty(&wc->endio_list))
1416 goto pop_from_list;
1417 set_current_state(TASK_INTERRUPTIBLE);
1418 raw_spin_unlock_irq(&wc->endio_list_lock);
1419
1420 if (unlikely(kthread_should_stop())) {
1421 set_current_state(TASK_RUNNING);
1422 break;
1423 }
1424
1425 schedule();
1426
1427 continue;
1428
1429 pop_from_list:
1430 list = wc->endio_list;
1431 list.next->prev = list.prev->next = &list;
1432 INIT_LIST_HEAD(&wc->endio_list);
1433 raw_spin_unlock_irq(&wc->endio_list_lock);
1434
1435 if (!WC_MODE_FUA(wc))
1436 writecache_disk_flush(wc, wc->dev);
1437
1438 wc_lock(wc);
1439
1440 if (WC_MODE_PMEM(wc)) {
1441 __writecache_endio_pmem(wc, &list);
1442 } else {
1443 __writecache_endio_ssd(wc, &list);
1444 writecache_wait_for_ios(wc, READ);
1445 }
1446
1447 writecache_commit_flushed(wc, false);
1448
1449 wc_unlock(wc);
1450 }
1451
1452 return 0;
1453 }
1454
1455 static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e, gfp_t gfp)
1456 {
1457 struct dm_writecache *wc = wb->wc;
1458 unsigned block_size = wc->block_size;
1459 void *address = memory_data(wc, e);
1460
1461 persistent_memory_flush_cache(address, block_size);
1462 return bio_add_page(&wb->bio, persistent_memory_page(address),
1463 block_size, persistent_memory_page_offset(address)) != 0;
1464 }
1465
1466 struct writeback_list {
1467 struct list_head list;
1468 size_t size;
1469 };
1470
1471 static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
1472 {
1473 if (unlikely(wc->max_writeback_jobs)) {
1474 if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) {
1475 wc_lock(wc);
1476 while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs)
1477 writecache_wait_on_freelist(wc);
1478 wc_unlock(wc);
1479 }
1480 }
1481 cond_resched();
1482 }
1483
1484 static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
1485 {
1486 struct wc_entry *e, *f;
1487 struct bio *bio;
1488 struct writeback_struct *wb;
1489 unsigned max_pages;
1490
1491 while (wbl->size) {
1492 wbl->size--;
1493 e = container_of(wbl->list.prev, struct wc_entry, lru);
1494 list_del(&e->lru);
1495
1496 max_pages = e->wc_list_contiguous;
1497
1498 bio = bio_alloc_bioset(GFP_NOIO, max_pages, &wc->bio_set);
1499 wb = container_of(bio, struct writeback_struct, bio);
1500 wb->wc = wc;
1501 bio->bi_end_io = writecache_writeback_endio;
1502 bio_set_dev(bio, wc->dev->bdev);
1503 bio->bi_iter.bi_sector = read_original_sector(wc, e);
1504 if (max_pages <= WB_LIST_INLINE ||
1505 unlikely(!(wb->wc_list = kmalloc_array(max_pages, sizeof(struct wc_entry *),
1506 GFP_NOIO | __GFP_NORETRY |
1507 __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
1508 wb->wc_list = wb->wc_list_inline;
1509 max_pages = WB_LIST_INLINE;
1510 }
1511
1512 BUG_ON(!wc_add_block(wb, e, GFP_NOIO));
1513
1514 wb->wc_list[0] = e;
1515 wb->wc_list_n = 1;
1516
1517 while (wbl->size && wb->wc_list_n < max_pages) {
1518 f = container_of(wbl->list.prev, struct wc_entry, lru);
1519 if (read_original_sector(wc, f) !=
1520 read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
1521 break;
1522 if (!wc_add_block(wb, f, GFP_NOWAIT | __GFP_NOWARN))
1523 break;
1524 wbl->size--;
1525 list_del(&f->lru);
1526 wb->wc_list[wb->wc_list_n++] = f;
1527 e = f;
1528 }
1529 bio_set_op_attrs(bio, REQ_OP_WRITE, WC_MODE_FUA(wc) * REQ_FUA);
1530 if (writecache_has_error(wc)) {
1531 bio->bi_status = BLK_STS_IOERR;
1532 bio_endio(bio);
1533 } else {
1534 submit_bio(bio);
1535 }
1536
1537 __writeback_throttle(wc, wbl);
1538 }
1539 }
1540
1541 static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
1542 {
1543 struct wc_entry *e, *f;
1544 struct dm_io_region from, to;
1545 struct copy_struct *c;
1546
1547 while (wbl->size) {
1548 unsigned n_sectors;
1549
1550 wbl->size--;
1551 e = container_of(wbl->list.prev, struct wc_entry, lru);
1552 list_del(&e->lru);
1553
1554 n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT);
1555
1556 from.bdev = wc->ssd_dev->bdev;
1557 from.sector = cache_sector(wc, e);
1558 from.count = n_sectors;
1559 to.bdev = wc->dev->bdev;
1560 to.sector = read_original_sector(wc, e);
1561 to.count = n_sectors;
1562
1563 c = mempool_alloc(&wc->copy_pool, GFP_NOIO);
1564 c->wc = wc;
1565 c->e = e;
1566 c->n_entries = e->wc_list_contiguous;
1567
1568 while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) {
1569 wbl->size--;
1570 f = container_of(wbl->list.prev, struct wc_entry, lru);
1571 BUG_ON(f != e + 1);
1572 list_del(&f->lru);
1573 e = f;
1574 }
1575
1576 dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c);
1577
1578 __writeback_throttle(wc, wbl);
1579 }
1580 }
1581
1582 static void writecache_writeback(struct work_struct *work)
1583 {
1584 struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work);
1585 struct blk_plug plug;
1586 struct wc_entry *f, *uninitialized_var(g), *e = NULL;
1587 struct rb_node *node, *next_node;
1588 struct list_head skipped;
1589 struct writeback_list wbl;
1590 unsigned long n_walked;
1591
1592 wc_lock(wc);
1593 restart:
1594 if (writecache_has_error(wc)) {
1595 wc_unlock(wc);
1596 return;
1597 }
1598
1599 if (unlikely(wc->writeback_all)) {
1600 if (writecache_wait_for_writeback(wc))
1601 goto restart;
1602 }
1603
1604 if (wc->overwrote_committed) {
1605 writecache_wait_for_ios(wc, WRITE);
1606 }
1607
1608 n_walked = 0;
1609 INIT_LIST_HEAD(&skipped);
1610 INIT_LIST_HEAD(&wbl.list);
1611 wbl.size = 0;
1612 while (!list_empty(&wc->lru) &&
1613 (wc->writeback_all ||
1614 wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark)) {
1615
1616 n_walked++;
1617 if (unlikely(n_walked > WRITEBACK_LATENCY) &&
1618 likely(!wc->writeback_all) && likely(!dm_suspended(wc->ti))) {
1619 queue_work(wc->writeback_wq, &wc->writeback_work);
1620 break;
1621 }
1622
1623 if (unlikely(wc->writeback_all)) {
1624 if (unlikely(!e)) {
1625 writecache_flush(wc);
1626 e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node);
1627 } else
1628 e = g;
1629 } else
1630 e = container_of(wc->lru.prev, struct wc_entry, lru);
1631 BUG_ON(e->write_in_progress);
1632 if (unlikely(!writecache_entry_is_committed(wc, e))) {
1633 writecache_flush(wc);
1634 }
1635 node = rb_prev(&e->rb_node);
1636 if (node) {
1637 f = container_of(node, struct wc_entry, rb_node);
1638 if (unlikely(read_original_sector(wc, f) ==
1639 read_original_sector(wc, e))) {
1640 BUG_ON(!f->write_in_progress);
1641 list_del(&e->lru);
1642 list_add(&e->lru, &skipped);
1643 cond_resched();
1644 continue;
1645 }
1646 }
1647 wc->writeback_size++;
1648 list_del(&e->lru);
1649 list_add(&e->lru, &wbl.list);
1650 wbl.size++;
1651 e->write_in_progress = true;
1652 e->wc_list_contiguous = 1;
1653
1654 f = e;
1655
1656 while (1) {
1657 next_node = rb_next(&f->rb_node);
1658 if (unlikely(!next_node))
1659 break;
1660 g = container_of(next_node, struct wc_entry, rb_node);
1661 if (unlikely(read_original_sector(wc, g) ==
1662 read_original_sector(wc, f))) {
1663 f = g;
1664 continue;
1665 }
1666 if (read_original_sector(wc, g) !=
1667 read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT))
1668 break;
1669 if (unlikely(g->write_in_progress))
1670 break;
1671 if (unlikely(!writecache_entry_is_committed(wc, g)))
1672 break;
1673
1674 if (!WC_MODE_PMEM(wc)) {
1675 if (g != f + 1)
1676 break;
1677 }
1678
1679 n_walked++;
1680 //if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all))
1681 // break;
1682
1683 wc->writeback_size++;
1684 list_del(&g->lru);
1685 list_add(&g->lru, &wbl.list);
1686 wbl.size++;
1687 g->write_in_progress = true;
1688 g->wc_list_contiguous = BIO_MAX_PAGES;
1689 f = g;
1690 e->wc_list_contiguous++;
1691 if (unlikely(e->wc_list_contiguous == BIO_MAX_PAGES)) {
1692 if (unlikely(wc->writeback_all)) {
1693 next_node = rb_next(&f->rb_node);
1694 if (likely(next_node))
1695 g = container_of(next_node, struct wc_entry, rb_node);
1696 }
1697 break;
1698 }
1699 }
1700 cond_resched();
1701 }
1702
1703 if (!list_empty(&skipped)) {
1704 list_splice_tail(&skipped, &wc->lru);
1705 /*
1706 * If we didn't do any progress, we must wait until some
1707 * writeback finishes to avoid burning CPU in a loop
1708 */
1709 if (unlikely(!wbl.size))
1710 writecache_wait_for_writeback(wc);
1711 }
1712
1713 wc_unlock(wc);
1714
1715 blk_start_plug(&plug);
1716
1717 if (WC_MODE_PMEM(wc))
1718 __writecache_writeback_pmem(wc, &wbl);
1719 else
1720 __writecache_writeback_ssd(wc, &wbl);
1721
1722 blk_finish_plug(&plug);
1723
1724 if (unlikely(wc->writeback_all)) {
1725 wc_lock(wc);
1726 while (writecache_wait_for_writeback(wc));
1727 wc_unlock(wc);
1728 }
1729 }
1730
1731 static int calculate_memory_size(uint64_t device_size, unsigned block_size,
1732 size_t *n_blocks_p, size_t *n_metadata_blocks_p)
1733 {
1734 uint64_t n_blocks, offset;
1735 struct wc_entry e;
1736
1737 n_blocks = device_size;
1738 do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
1739
1740 while (1) {
1741 if (!n_blocks)
1742 return -ENOSPC;
1743 /* Verify the following entries[n_blocks] won't overflow */
1744 if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) /
1745 sizeof(struct wc_memory_entry)))
1746 return -EFBIG;
1747 offset = offsetof(struct wc_memory_superblock, entries[n_blocks]);
1748 offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1);
1749 if (offset + n_blocks * block_size <= device_size)
1750 break;
1751 n_blocks--;
1752 }
1753
1754 /* check if the bit field overflows */
1755 e.index = n_blocks;
1756 if (e.index != n_blocks)
1757 return -EFBIG;
1758
1759 if (n_blocks_p)
1760 *n_blocks_p = n_blocks;
1761 if (n_metadata_blocks_p)
1762 *n_metadata_blocks_p = offset >> __ffs(block_size);
1763 return 0;
1764 }
1765
1766 static int init_memory(struct dm_writecache *wc)
1767 {
1768 size_t b;
1769 int r;
1770
1771 r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL);
1772 if (r)
1773 return r;
1774
1775 r = writecache_alloc_entries(wc);
1776 if (r)
1777 return r;
1778
1779 for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
1780 pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
1781 pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
1782 pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
1783 pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
1784 pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
1785
1786 for (b = 0; b < wc->n_blocks; b++)
1787 write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
1788
1789 writecache_flush_all_metadata(wc);
1790 writecache_commit_flushed(wc, false);
1791 pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
1792 writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
1793 writecache_commit_flushed(wc, false);
1794
1795 return 0;
1796 }
1797
1798 static void writecache_dtr(struct dm_target *ti)
1799 {
1800 struct dm_writecache *wc = ti->private;
1801
1802 if (!wc)
1803 return;
1804
1805 if (wc->endio_thread)
1806 kthread_stop(wc->endio_thread);
1807
1808 if (wc->flush_thread)
1809 kthread_stop(wc->flush_thread);
1810
1811 bioset_exit(&wc->bio_set);
1812
1813 mempool_exit(&wc->copy_pool);
1814
1815 if (wc->writeback_wq)
1816 destroy_workqueue(wc->writeback_wq);
1817
1818 if (wc->dev)
1819 dm_put_device(ti, wc->dev);
1820
1821 if (wc->ssd_dev)
1822 dm_put_device(ti, wc->ssd_dev);
1823
1824 if (wc->entries)
1825 vfree(wc->entries);
1826
1827 if (wc->memory_map) {
1828 if (WC_MODE_PMEM(wc))
1829 persistent_memory_release(wc);
1830 else
1831 vfree(wc->memory_map);
1832 }
1833
1834 if (wc->dm_kcopyd)
1835 dm_kcopyd_client_destroy(wc->dm_kcopyd);
1836
1837 if (wc->dm_io)
1838 dm_io_client_destroy(wc->dm_io);
1839
1840 if (wc->dirty_bitmap)
1841 vfree(wc->dirty_bitmap);
1842
1843 kfree(wc);
1844 }
1845
1846 static int writecache_ctr(struct dm_target *ti, unsigned argc, char **argv)
1847 {
1848 struct dm_writecache *wc;
1849 struct dm_arg_set as;
1850 const char *string;
1851 unsigned opt_params;
1852 size_t offset, data_size;
1853 int i, r;
1854 char dummy;
1855 int high_wm_percent = HIGH_WATERMARK;
1856 int low_wm_percent = LOW_WATERMARK;
1857 uint64_t x;
1858 struct wc_memory_superblock s;
1859
1860 static struct dm_arg _args[] = {
1861 {0, 10, "Invalid number of feature args"},
1862 };
1863
1864 as.argc = argc;
1865 as.argv = argv;
1866
1867 wc = kzalloc(sizeof(struct dm_writecache), GFP_KERNEL);
1868 if (!wc) {
1869 ti->error = "Cannot allocate writecache structure";
1870 r = -ENOMEM;
1871 goto bad;
1872 }
1873 ti->private = wc;
1874 wc->ti = ti;
1875
1876 mutex_init(&wc->lock);
1877 writecache_poison_lists(wc);
1878 init_waitqueue_head(&wc->freelist_wait);
1879 timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
1880
1881 for (i = 0; i < 2; i++) {
1882 atomic_set(&wc->bio_in_progress[i], 0);
1883 init_waitqueue_head(&wc->bio_in_progress_wait[i]);
1884 }
1885
1886 wc->dm_io = dm_io_client_create();
1887 if (IS_ERR(wc->dm_io)) {
1888 r = PTR_ERR(wc->dm_io);
1889 ti->error = "Unable to allocate dm-io client";
1890 wc->dm_io = NULL;
1891 goto bad;
1892 }
1893
1894 wc->writeback_wq = alloc_workqueue("writecache-writeback", WQ_MEM_RECLAIM, 1);
1895 if (!wc->writeback_wq) {
1896 r = -ENOMEM;
1897 ti->error = "Could not allocate writeback workqueue";
1898 goto bad;
1899 }
1900 INIT_WORK(&wc->writeback_work, writecache_writeback);
1901 INIT_WORK(&wc->flush_work, writecache_flush_work);
1902
1903 raw_spin_lock_init(&wc->endio_list_lock);
1904 INIT_LIST_HEAD(&wc->endio_list);
1905 wc->endio_thread = kthread_create(writecache_endio_thread, wc, "writecache_endio");
1906 if (IS_ERR(wc->endio_thread)) {
1907 r = PTR_ERR(wc->endio_thread);
1908 wc->endio_thread = NULL;
1909 ti->error = "Couldn't spawn endio thread";
1910 goto bad;
1911 }
1912 wake_up_process(wc->endio_thread);
1913
1914 /*
1915 * Parse the mode (pmem or ssd)
1916 */
1917 string = dm_shift_arg(&as);
1918 if (!string)
1919 goto bad_arguments;
1920
1921 if (!strcasecmp(string, "s")) {
1922 wc->pmem_mode = false;
1923 } else if (!strcasecmp(string, "p")) {
1924 #ifdef DM_WRITECACHE_HAS_PMEM
1925 wc->pmem_mode = true;
1926 wc->writeback_fua = true;
1927 #else
1928 /*
1929 * If the architecture doesn't support persistent memory or
1930 * the kernel doesn't support any DAX drivers, this driver can
1931 * only be used in SSD-only mode.
1932 */
1933 r = -EOPNOTSUPP;
1934 ti->error = "Persistent memory or DAX not supported on this system";
1935 goto bad;
1936 #endif
1937 } else {
1938 goto bad_arguments;
1939 }
1940
1941 if (WC_MODE_PMEM(wc)) {
1942 r = bioset_init(&wc->bio_set, BIO_POOL_SIZE,
1943 offsetof(struct writeback_struct, bio),
1944 BIOSET_NEED_BVECS);
1945 if (r) {
1946 ti->error = "Could not allocate bio set";
1947 goto bad;
1948 }
1949 } else {
1950 r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct));
1951 if (r) {
1952 ti->error = "Could not allocate mempool";
1953 goto bad;
1954 }
1955 }
1956
1957 /*
1958 * Parse the origin data device
1959 */
1960 string = dm_shift_arg(&as);
1961 if (!string)
1962 goto bad_arguments;
1963 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev);
1964 if (r) {
1965 ti->error = "Origin data device lookup failed";
1966 goto bad;
1967 }
1968
1969 /*
1970 * Parse cache data device (be it pmem or ssd)
1971 */
1972 string = dm_shift_arg(&as);
1973 if (!string)
1974 goto bad_arguments;
1975
1976 r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev);
1977 if (r) {
1978 ti->error = "Cache data device lookup failed";
1979 goto bad;
1980 }
1981 wc->memory_map_size = i_size_read(wc->ssd_dev->bdev->bd_inode);
1982
1983 /*
1984 * Parse the cache block size
1985 */
1986 string = dm_shift_arg(&as);
1987 if (!string)
1988 goto bad_arguments;
1989 if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 ||
1990 wc->block_size < 512 || wc->block_size > PAGE_SIZE ||
1991 (wc->block_size & (wc->block_size - 1))) {
1992 r = -EINVAL;
1993 ti->error = "Invalid block size";
1994 goto bad;
1995 }
1996 wc->block_size_bits = __ffs(wc->block_size);
1997
1998 wc->max_writeback_jobs = MAX_WRITEBACK_JOBS;
1999 wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM;
2000 wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC);
2001
2002 /*
2003 * Parse optional arguments
2004 */
2005 r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
2006 if (r)
2007 goto bad;
2008
2009 while (opt_params) {
2010 string = dm_shift_arg(&as), opt_params--;
2011 if (!strcasecmp(string, "start_sector") && opt_params >= 1) {
2012 unsigned long long start_sector;
2013 string = dm_shift_arg(&as), opt_params--;
2014 if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1)
2015 goto invalid_optional;
2016 wc->start_sector = start_sector;
2017 if (wc->start_sector != start_sector ||
2018 wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT)
2019 goto invalid_optional;
2020 } else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) {
2021 string = dm_shift_arg(&as), opt_params--;
2022 if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1)
2023 goto invalid_optional;
2024 if (high_wm_percent < 0 || high_wm_percent > 100)
2025 goto invalid_optional;
2026 wc->high_wm_percent_set = true;
2027 } else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) {
2028 string = dm_shift_arg(&as), opt_params--;
2029 if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1)
2030 goto invalid_optional;
2031 if (low_wm_percent < 0 || low_wm_percent > 100)
2032 goto invalid_optional;
2033 wc->low_wm_percent_set = true;
2034 } else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) {
2035 string = dm_shift_arg(&as), opt_params--;
2036 if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1)
2037 goto invalid_optional;
2038 wc->max_writeback_jobs_set = true;
2039 } else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) {
2040 string = dm_shift_arg(&as), opt_params--;
2041 if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1)
2042 goto invalid_optional;
2043 wc->autocommit_blocks_set = true;
2044 } else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) {
2045 unsigned autocommit_msecs;
2046 string = dm_shift_arg(&as), opt_params--;
2047 if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
2048 goto invalid_optional;
2049 if (autocommit_msecs > 3600000)
2050 goto invalid_optional;
2051 wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
2052 wc->autocommit_time_set = true;
2053 } else if (!strcasecmp(string, "fua")) {
2054 if (WC_MODE_PMEM(wc)) {
2055 wc->writeback_fua = true;
2056 wc->writeback_fua_set = true;
2057 } else goto invalid_optional;
2058 } else if (!strcasecmp(string, "nofua")) {
2059 if (WC_MODE_PMEM(wc)) {
2060 wc->writeback_fua = false;
2061 wc->writeback_fua_set = true;
2062 } else goto invalid_optional;
2063 } else {
2064 invalid_optional:
2065 r = -EINVAL;
2066 ti->error = "Invalid optional argument";
2067 goto bad;
2068 }
2069 }
2070
2071 if (high_wm_percent < low_wm_percent) {
2072 r = -EINVAL;
2073 ti->error = "High watermark must be greater than or equal to low watermark";
2074 goto bad;
2075 }
2076
2077 if (WC_MODE_PMEM(wc)) {
2078 r = persistent_memory_claim(wc);
2079 if (r) {
2080 ti->error = "Unable to map persistent memory for cache";
2081 goto bad;
2082 }
2083 } else {
2084 struct dm_io_region region;
2085 struct dm_io_request req;
2086 size_t n_blocks, n_metadata_blocks;
2087 uint64_t n_bitmap_bits;
2088
2089 wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT;
2090
2091 bio_list_init(&wc->flush_list);
2092 wc->flush_thread = kthread_create(writecache_flush_thread, wc, "dm_writecache_flush");
2093 if (IS_ERR(wc->flush_thread)) {
2094 r = PTR_ERR(wc->flush_thread);
2095 wc->flush_thread = NULL;
2096 ti->error = "Couldn't spawn flush thread";
2097 goto bad;
2098 }
2099 wake_up_process(wc->flush_thread);
2100
2101 r = calculate_memory_size(wc->memory_map_size, wc->block_size,
2102 &n_blocks, &n_metadata_blocks);
2103 if (r) {
2104 ti->error = "Invalid device size";
2105 goto bad;
2106 }
2107
2108 n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) +
2109 BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY;
2110 /* this is limitation of test_bit functions */
2111 if (n_bitmap_bits > 1U << 31) {
2112 r = -EFBIG;
2113 ti->error = "Invalid device size";
2114 goto bad;
2115 }
2116
2117 wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits);
2118 if (!wc->memory_map) {
2119 r = -ENOMEM;
2120 ti->error = "Unable to allocate memory for metadata";
2121 goto bad;
2122 }
2123
2124 wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle);
2125 if (IS_ERR(wc->dm_kcopyd)) {
2126 r = PTR_ERR(wc->dm_kcopyd);
2127 ti->error = "Unable to allocate dm-kcopyd client";
2128 wc->dm_kcopyd = NULL;
2129 goto bad;
2130 }
2131
2132 wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT);
2133 wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) /
2134 BITS_PER_LONG * sizeof(unsigned long);
2135 wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size);
2136 if (!wc->dirty_bitmap) {
2137 r = -ENOMEM;
2138 ti->error = "Unable to allocate dirty bitmap";
2139 goto bad;
2140 }
2141
2142 region.bdev = wc->ssd_dev->bdev;
2143 region.sector = wc->start_sector;
2144 region.count = wc->metadata_sectors;
2145 req.bi_op = REQ_OP_READ;
2146 req.bi_op_flags = REQ_SYNC;
2147 req.mem.type = DM_IO_VMA;
2148 req.mem.ptr.vma = (char *)wc->memory_map;
2149 req.client = wc->dm_io;
2150 req.notify.fn = NULL;
2151
2152 r = dm_io(&req, 1, &region, NULL);
2153 if (r) {
2154 ti->error = "Unable to read metadata";
2155 goto bad;
2156 }
2157 }
2158
2159 r = memcpy_mcsafe(&s, sb(wc), sizeof(struct wc_memory_superblock));
2160 if (r) {
2161 ti->error = "Hardware memory error when reading superblock";
2162 goto bad;
2163 }
2164 if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) {
2165 r = init_memory(wc);
2166 if (r) {
2167 ti->error = "Unable to initialize device";
2168 goto bad;
2169 }
2170 r = memcpy_mcsafe(&s, sb(wc), sizeof(struct wc_memory_superblock));
2171 if (r) {
2172 ti->error = "Hardware memory error when reading superblock";
2173 goto bad;
2174 }
2175 }
2176
2177 if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) {
2178 ti->error = "Invalid magic in the superblock";
2179 r = -EINVAL;
2180 goto bad;
2181 }
2182
2183 if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) {
2184 ti->error = "Invalid version in the superblock";
2185 r = -EINVAL;
2186 goto bad;
2187 }
2188
2189 if (le32_to_cpu(s.block_size) != wc->block_size) {
2190 ti->error = "Block size does not match superblock";
2191 r = -EINVAL;
2192 goto bad;
2193 }
2194
2195 wc->n_blocks = le64_to_cpu(s.n_blocks);
2196
2197 offset = wc->n_blocks * sizeof(struct wc_memory_entry);
2198 if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) {
2199 overflow:
2200 ti->error = "Overflow in size calculation";
2201 r = -EINVAL;
2202 goto bad;
2203 }
2204 offset += sizeof(struct wc_memory_superblock);
2205 if (offset < sizeof(struct wc_memory_superblock))
2206 goto overflow;
2207 offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1);
2208 data_size = wc->n_blocks * (size_t)wc->block_size;
2209 if (!offset || (data_size / wc->block_size != wc->n_blocks) ||
2210 (offset + data_size < offset))
2211 goto overflow;
2212 if (offset + data_size > wc->memory_map_size) {
2213 ti->error = "Memory area is too small";
2214 r = -EINVAL;
2215 goto bad;
2216 }
2217
2218 wc->metadata_sectors = offset >> SECTOR_SHIFT;
2219 wc->block_start = (char *)sb(wc) + offset;
2220
2221 x = (uint64_t)wc->n_blocks * (100 - high_wm_percent);
2222 x += 50;
2223 do_div(x, 100);
2224 wc->freelist_high_watermark = x;
2225 x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
2226 x += 50;
2227 do_div(x, 100);
2228 wc->freelist_low_watermark = x;
2229
2230 r = writecache_alloc_entries(wc);
2231 if (r) {
2232 ti->error = "Cannot allocate memory";
2233 goto bad;
2234 }
2235
2236 ti->num_flush_bios = 1;
2237 ti->flush_supported = true;
2238 ti->num_discard_bios = 1;
2239
2240 if (WC_MODE_PMEM(wc))
2241 persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
2242
2243 return 0;
2244
2245 bad_arguments:
2246 r = -EINVAL;
2247 ti->error = "Bad arguments";
2248 bad:
2249 writecache_dtr(ti);
2250 return r;
2251 }
2252
2253 static void writecache_status(struct dm_target *ti, status_type_t type,
2254 unsigned status_flags, char *result, unsigned maxlen)
2255 {
2256 struct dm_writecache *wc = ti->private;
2257 unsigned extra_args;
2258 unsigned sz = 0;
2259 uint64_t x;
2260
2261 switch (type) {
2262 case STATUSTYPE_INFO:
2263 DMEMIT("%ld %llu %llu %llu", writecache_has_error(wc),
2264 (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
2265 (unsigned long long)wc->writeback_size);
2266 break;
2267 case STATUSTYPE_TABLE:
2268 DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
2269 wc->dev->name, wc->ssd_dev->name, wc->block_size);
2270 extra_args = 0;
2271 if (wc->start_sector)
2272 extra_args += 2;
2273 if (wc->high_wm_percent_set)
2274 extra_args += 2;
2275 if (wc->low_wm_percent_set)
2276 extra_args += 2;
2277 if (wc->max_writeback_jobs_set)
2278 extra_args += 2;
2279 if (wc->autocommit_blocks_set)
2280 extra_args += 2;
2281 if (wc->autocommit_time_set)
2282 extra_args += 2;
2283 if (wc->writeback_fua_set)
2284 extra_args++;
2285
2286 DMEMIT("%u", extra_args);
2287 if (wc->start_sector)
2288 DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
2289 if (wc->high_wm_percent_set) {
2290 x = (uint64_t)wc->freelist_high_watermark * 100;
2291 x += wc->n_blocks / 2;
2292 do_div(x, (size_t)wc->n_blocks);
2293 DMEMIT(" high_watermark %u", 100 - (unsigned)x);
2294 }
2295 if (wc->low_wm_percent_set) {
2296 x = (uint64_t)wc->freelist_low_watermark * 100;
2297 x += wc->n_blocks / 2;
2298 do_div(x, (size_t)wc->n_blocks);
2299 DMEMIT(" low_watermark %u", 100 - (unsigned)x);
2300 }
2301 if (wc->max_writeback_jobs_set)
2302 DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
2303 if (wc->autocommit_blocks_set)
2304 DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
2305 if (wc->autocommit_time_set)
2306 DMEMIT(" autocommit_time %u", jiffies_to_msecs(wc->autocommit_jiffies));
2307 if (wc->writeback_fua_set)
2308 DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
2309 break;
2310 }
2311 }
2312
2313 static struct target_type writecache_target = {
2314 .name = "writecache",
2315 .version = {1, 1, 1},
2316 .module = THIS_MODULE,
2317 .ctr = writecache_ctr,
2318 .dtr = writecache_dtr,
2319 .status = writecache_status,
2320 .postsuspend = writecache_suspend,
2321 .resume = writecache_resume,
2322 .message = writecache_message,
2323 .map = writecache_map,
2324 .end_io = writecache_end_io,
2325 .iterate_devices = writecache_iterate_devices,
2326 .io_hints = writecache_io_hints,
2327 };
2328
2329 static int __init dm_writecache_init(void)
2330 {
2331 int r;
2332
2333 r = dm_register_target(&writecache_target);
2334 if (r < 0) {
2335 DMERR("register failed %d", r);
2336 return r;
2337 }
2338
2339 return 0;
2340 }
2341
2342 static void __exit dm_writecache_exit(void)
2343 {
2344 dm_unregister_target(&writecache_target);
2345 }
2346
2347 module_init(dm_writecache_init);
2348 module_exit(dm_writecache_exit);
2349
2350 MODULE_DESCRIPTION(DM_NAME " writecache target");
2351 MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
2352 MODULE_LICENSE("GPL");
Linux with added FPC-III support