?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
hwmon: (adt7475) fan stall prevention
[linux/fpc-iii.git] / drivers / md / bitmap.c
blobf4eace5ea184095eb0c170c4f3f1647f72b8c537
1 /*
2 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
3 *
4 * bitmap_create - sets up the bitmap structure
5 * bitmap_destroy - destroys the bitmap structure
6 *
7 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
8 * - added disk storage for bitmap
9 * - changes to allow various bitmap chunk sizes
10 */
11
12 /*
13 * Still to do:
14 *
15 * flush after percent set rather than just time based. (maybe both).
16 */
17
18 #include <linux/blkdev.h>
19 #include <linux/module.h>
20 #include <linux/errno.h>
21 #include <linux/slab.h>
22 #include <linux/init.h>
23 #include <linux/timer.h>
24 #include <linux/sched.h>
25 #include <linux/list.h>
26 #include <linux/file.h>
27 #include <linux/mount.h>
28 #include <linux/buffer_head.h>
29 #include <linux/seq_file.h>
30 #include <trace/events/block.h>
31 #include "md.h"
32 #include "bitmap.h"
33
34 static inline char *bmname(struct bitmap *bitmap)
35 {
36 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
37 }
38
39 /*
40 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
41 *
42 * 1) check to see if this page is allocated, if it's not then try to alloc
43 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
44 * page pointer directly as a counter
45 *
46 * if we find our page, we increment the page's refcount so that it stays
47 * allocated while we're using it
48 */
49 static int bitmap_checkpage(struct bitmap_counts *bitmap,
50 unsigned long page, int create, int no_hijack)
51 __releases(bitmap->lock)
52 __acquires(bitmap->lock)
53 {
54 unsigned char *mappage;
55
56 if (page >= bitmap->pages) {
57 /* This can happen if bitmap_start_sync goes beyond
58 * End-of-device while looking for a whole page.
59 * It is harmless.
60 */
61 return -EINVAL;
62 }
63
64 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
65 return 0;
66
67 if (bitmap->bp[page].map) /* page is already allocated, just return */
68 return 0;
69
70 if (!create)
71 return -ENOENT;
72
73 /* this page has not been allocated yet */
74
75 spin_unlock_irq(&bitmap->lock);
76 /* It is possible that this is being called inside a
77 * prepare_to_wait/finish_wait loop from raid5c:make_request().
78 * In general it is not permitted to sleep in that context as it
79 * can cause the loop to spin freely.
80 * That doesn't apply here as we can only reach this point
81 * once with any loop.
82 * When this function completes, either bp[page].map or
83 * bp[page].hijacked. In either case, this function will
84 * abort before getting to this point again. So there is
85 * no risk of a free-spin, and so it is safe to assert
86 * that sleeping here is allowed.
87 */
88 sched_annotate_sleep();
89 mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
90 spin_lock_irq(&bitmap->lock);
91
92 if (mappage == NULL) {
93 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
94 /* We don't support hijack for cluster raid */
95 if (no_hijack)
96 return -ENOMEM;
97 /* failed - set the hijacked flag so that we can use the
98 * pointer as a counter */
99 if (!bitmap->bp[page].map)
100 bitmap->bp[page].hijacked = 1;
101 } else if (bitmap->bp[page].map ||
102 bitmap->bp[page].hijacked) {
103 /* somebody beat us to getting the page */
104 kfree(mappage);
105 } else {
106
107 /* no page was in place and we have one, so install it */
108
109 bitmap->bp[page].map = mappage;
110 bitmap->missing_pages--;
111 }
112 return 0;
113 }
114
115 /* if page is completely empty, put it back on the free list, or dealloc it */
116 /* if page was hijacked, unmark the flag so it might get alloced next time */
117 /* Note: lock should be held when calling this */
118 static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
119 {
120 char *ptr;
121
122 if (bitmap->bp[page].count) /* page is still busy */
123 return;
124
125 /* page is no longer in use, it can be released */
126
127 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
128 bitmap->bp[page].hijacked = 0;
129 bitmap->bp[page].map = NULL;
130 } else {
131 /* normal case, free the page */
132 ptr = bitmap->bp[page].map;
133 bitmap->bp[page].map = NULL;
134 bitmap->missing_pages++;
135 kfree(ptr);
136 }
137 }
138
139 /*
140 * bitmap file handling - read and write the bitmap file and its superblock
141 */
142
143 /*
144 * basic page I/O operations
145 */
146
147 /* IO operations when bitmap is stored near all superblocks */
148 static int read_sb_page(struct mddev *mddev, loff_t offset,
149 struct page *page,
150 unsigned long index, int size)
151 {
152 /* choose a good rdev and read the page from there */
153
154 struct md_rdev *rdev;
155 sector_t target;
156
157 rdev_for_each(rdev, mddev) {
158 if (! test_bit(In_sync, &rdev->flags)
159 || test_bit(Faulty, &rdev->flags))
160 continue;
161
162 target = offset + index * (PAGE_SIZE/512);
163
164 if (sync_page_io(rdev, target,
165 roundup(size, bdev_logical_block_size(rdev->bdev)),
166 page, REQ_OP_READ, 0, true)) {
167 page->index = index;
168 return 0;
169 }
170 }
171 return -EIO;
172 }
173
174 static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
175 {
176 /* Iterate the disks of an mddev, using rcu to protect access to the
177 * linked list, and raising the refcount of devices we return to ensure
178 * they don't disappear while in use.
179 * As devices are only added or removed when raid_disk is < 0 and
180 * nr_pending is 0 and In_sync is clear, the entries we return will
181 * still be in the same position on the list when we re-enter
182 * list_for_each_entry_continue_rcu.
183 *
184 * Note that if entered with 'rdev == NULL' to start at the
185 * beginning, we temporarily assign 'rdev' to an address which
186 * isn't really an rdev, but which can be used by
187 * list_for_each_entry_continue_rcu() to find the first entry.
188 */
189 rcu_read_lock();
190 if (rdev == NULL)
191 /* start at the beginning */
192 rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
193 else {
194 /* release the previous rdev and start from there. */
195 rdev_dec_pending(rdev, mddev);
196 }
197 list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
198 if (rdev->raid_disk >= 0 &&
199 !test_bit(Faulty, &rdev->flags)) {
200 /* this is a usable devices */
201 atomic_inc(&rdev->nr_pending);
202 rcu_read_unlock();
203 return rdev;
204 }
205 }
206 rcu_read_unlock();
207 return NULL;
208 }
209
210 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
211 {
212 struct md_rdev *rdev;
213 struct block_device *bdev;
214 struct mddev *mddev = bitmap->mddev;
215 struct bitmap_storage *store = &bitmap->storage;
216
217 restart:
218 rdev = NULL;
219 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
220 int size = PAGE_SIZE;
221 loff_t offset = mddev->bitmap_info.offset;
222
223 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
224
225 if (page->index == store->file_pages-1) {
226 int last_page_size = store->bytes & (PAGE_SIZE-1);
227 if (last_page_size == 0)
228 last_page_size = PAGE_SIZE;
229 size = roundup(last_page_size,
230 bdev_logical_block_size(bdev));
231 }
232 /* Just make sure we aren't corrupting data or
233 * metadata
234 */
235 if (mddev->external) {
236 /* Bitmap could be anywhere. */
237 if (rdev->sb_start + offset + (page->index
238 * (PAGE_SIZE/512))
239 > rdev->data_offset
240 &&
241 rdev->sb_start + offset
242 < (rdev->data_offset + mddev->dev_sectors
243 + (PAGE_SIZE/512)))
244 goto bad_alignment;
245 } else if (offset < 0) {
246 /* DATA BITMAP METADATA */
247 if (offset
248 + (long)(page->index * (PAGE_SIZE/512))
249 + size/512 > 0)
250 /* bitmap runs in to metadata */
251 goto bad_alignment;
252 if (rdev->data_offset + mddev->dev_sectors
253 > rdev->sb_start + offset)
254 /* data runs in to bitmap */
255 goto bad_alignment;
256 } else if (rdev->sb_start < rdev->data_offset) {
257 /* METADATA BITMAP DATA */
258 if (rdev->sb_start
259 + offset
260 + page->index*(PAGE_SIZE/512) + size/512
261 > rdev->data_offset)
262 /* bitmap runs in to data */
263 goto bad_alignment;
264 } else {
265 /* DATA METADATA BITMAP - no problems */
266 }
267 md_super_write(mddev, rdev,
268 rdev->sb_start + offset
269 + page->index * (PAGE_SIZE/512),
270 size,
271 page);
272 }
273
274 if (wait && md_super_wait(mddev) < 0)
275 goto restart;
276 return 0;
277
278 bad_alignment:
279 return -EINVAL;
280 }
281
282 static void bitmap_file_kick(struct bitmap *bitmap);
283 /*
284 * write out a page to a file
285 */
286 static void write_page(struct bitmap *bitmap, struct page *page, int wait)
287 {
288 struct buffer_head *bh;
289
290 if (bitmap->storage.file == NULL) {
291 switch (write_sb_page(bitmap, page, wait)) {
292 case -EINVAL:
293 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
294 }
295 } else {
296
297 bh = page_buffers(page);
298
299 while (bh && bh->b_blocknr) {
300 atomic_inc(&bitmap->pending_writes);
301 set_buffer_locked(bh);
302 set_buffer_mapped(bh);
303 submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
304 bh = bh->b_this_page;
305 }
306
307 if (wait)
308 wait_event(bitmap->write_wait,
309 atomic_read(&bitmap->pending_writes)==0);
310 }
311 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
312 bitmap_file_kick(bitmap);
313 }
314
315 static void end_bitmap_write(struct buffer_head *bh, int uptodate)
316 {
317 struct bitmap *bitmap = bh->b_private;
318
319 if (!uptodate)
320 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
321 if (atomic_dec_and_test(&bitmap->pending_writes))
322 wake_up(&bitmap->write_wait);
323 }
324
325 /* copied from buffer.c */
326 static void
327 __clear_page_buffers(struct page *page)
328 {
329 ClearPagePrivate(page);
330 set_page_private(page, 0);
331 put_page(page);
332 }
333 static void free_buffers(struct page *page)
334 {
335 struct buffer_head *bh;
336
337 if (!PagePrivate(page))
338 return;
339
340 bh = page_buffers(page);
341 while (bh) {
342 struct buffer_head *next = bh->b_this_page;
343 free_buffer_head(bh);
344 bh = next;
345 }
346 __clear_page_buffers(page);
347 put_page(page);
348 }
349
350 /* read a page from a file.
351 * We both read the page, and attach buffers to the page to record the
352 * address of each block (using bmap). These addresses will be used
353 * to write the block later, completely bypassing the filesystem.
354 * This usage is similar to how swap files are handled, and allows us
355 * to write to a file with no concerns of memory allocation failing.
356 */
357 static int read_page(struct file *file, unsigned long index,
358 struct bitmap *bitmap,
359 unsigned long count,
360 struct page *page)
361 {
362 int ret = 0;
363 struct inode *inode = file_inode(file);
364 struct buffer_head *bh;
365 sector_t block;
366
367 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
368 (unsigned long long)index << PAGE_SHIFT);
369
370 bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
371 if (!bh) {
372 ret = -ENOMEM;
373 goto out;
374 }
375 attach_page_buffers(page, bh);
376 block = index << (PAGE_SHIFT - inode->i_blkbits);
377 while (bh) {
378 if (count == 0)
379 bh->b_blocknr = 0;
380 else {
381 bh->b_blocknr = bmap(inode, block);
382 if (bh->b_blocknr == 0) {
383 /* Cannot use this file! */
384 ret = -EINVAL;
385 goto out;
386 }
387 bh->b_bdev = inode->i_sb->s_bdev;
388 if (count < (1<<inode->i_blkbits))
389 count = 0;
390 else
391 count -= (1<<inode->i_blkbits);
392
393 bh->b_end_io = end_bitmap_write;
394 bh->b_private = bitmap;
395 atomic_inc(&bitmap->pending_writes);
396 set_buffer_locked(bh);
397 set_buffer_mapped(bh);
398 submit_bh(REQ_OP_READ, 0, bh);
399 }
400 block++;
401 bh = bh->b_this_page;
402 }
403 page->index = index;
404
405 wait_event(bitmap->write_wait,
406 atomic_read(&bitmap->pending_writes)==0);
407 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
408 ret = -EIO;
409 out:
410 if (ret)
411 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
412 (int)PAGE_SIZE,
413 (unsigned long long)index << PAGE_SHIFT,
414 ret);
415 return ret;
416 }
417
418 /*
419 * bitmap file superblock operations
420 */
421
422 /*
423 * bitmap_wait_writes() should be called before writing any bitmap
424 * blocks, to ensure previous writes, particularly from
425 * bitmap_daemon_work(), have completed.
426 */
427 static void bitmap_wait_writes(struct bitmap *bitmap)
428 {
429 if (bitmap->storage.file)
430 wait_event(bitmap->write_wait,
431 atomic_read(&bitmap->pending_writes)==0);
432 else
433 /* Note that we ignore the return value. The writes
434 * might have failed, but that would just mean that
435 * some bits which should be cleared haven't been,
436 * which is safe. The relevant bitmap blocks will
437 * probably get written again, but there is no great
438 * loss if they aren't.
439 */
440 md_super_wait(bitmap->mddev);
441 }
442
443
444 /* update the event counter and sync the superblock to disk */
445 void bitmap_update_sb(struct bitmap *bitmap)
446 {
447 bitmap_super_t *sb;
448
449 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
450 return;
451 if (bitmap->mddev->bitmap_info.external)
452 return;
453 if (!bitmap->storage.sb_page) /* no superblock */
454 return;
455 sb = kmap_atomic(bitmap->storage.sb_page);
456 sb->events = cpu_to_le64(bitmap->mddev->events);
457 if (bitmap->mddev->events < bitmap->events_cleared)
458 /* rocking back to read-only */
459 bitmap->events_cleared = bitmap->mddev->events;
460 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
461 sb->state = cpu_to_le32(bitmap->flags);
462 /* Just in case these have been changed via sysfs: */
463 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
464 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
465 /* This might have been changed by a reshape */
466 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
467 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
468 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
469 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
470 bitmap_info.space);
471 kunmap_atomic(sb);
472 write_page(bitmap, bitmap->storage.sb_page, 1);
473 }
474 EXPORT_SYMBOL(bitmap_update_sb);
475
476 /* print out the bitmap file superblock */
477 void bitmap_print_sb(struct bitmap *bitmap)
478 {
479 bitmap_super_t *sb;
480
481 if (!bitmap || !bitmap->storage.sb_page)
482 return;
483 sb = kmap_atomic(bitmap->storage.sb_page);
484 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
485 pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
486 pr_debug(" version: %d\n", le32_to_cpu(sb->version));
487 pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
488 le32_to_cpu(*(__u32 *)(sb->uuid+0)),
489 le32_to_cpu(*(__u32 *)(sb->uuid+4)),
490 le32_to_cpu(*(__u32 *)(sb->uuid+8)),
491 le32_to_cpu(*(__u32 *)(sb->uuid+12)));
492 pr_debug(" events: %llu\n",
493 (unsigned long long) le64_to_cpu(sb->events));
494 pr_debug("events cleared: %llu\n",
495 (unsigned long long) le64_to_cpu(sb->events_cleared));
496 pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
497 pr_debug(" chunksize: %d B\n", le32_to_cpu(sb->chunksize));
498 pr_debug(" daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
499 pr_debug(" sync size: %llu KB\n",
500 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
501 pr_debug("max write behind: %d\n", le32_to_cpu(sb->write_behind));
502 kunmap_atomic(sb);
503 }
504
505 /*
506 * bitmap_new_disk_sb
507 * @bitmap
508 *
509 * This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
510 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
511 * This function verifies 'bitmap_info' and populates the on-disk bitmap
512 * structure, which is to be written to disk.
513 *
514 * Returns: 0 on success, -Exxx on error
515 */
516 static int bitmap_new_disk_sb(struct bitmap *bitmap)
517 {
518 bitmap_super_t *sb;
519 unsigned long chunksize, daemon_sleep, write_behind;
520
521 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
522 if (bitmap->storage.sb_page == NULL)
523 return -ENOMEM;
524 bitmap->storage.sb_page->index = 0;
525
526 sb = kmap_atomic(bitmap->storage.sb_page);
527
528 sb->magic = cpu_to_le32(BITMAP_MAGIC);
529 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
530
531 chunksize = bitmap->mddev->bitmap_info.chunksize;
532 BUG_ON(!chunksize);
533 if (!is_power_of_2(chunksize)) {
534 kunmap_atomic(sb);
535 pr_warn("bitmap chunksize not a power of 2\n");
536 return -EINVAL;
537 }
538 sb->chunksize = cpu_to_le32(chunksize);
539
540 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
541 if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
542 pr_debug("Choosing daemon_sleep default (5 sec)\n");
543 daemon_sleep = 5 * HZ;
544 }
545 sb->daemon_sleep = cpu_to_le32(daemon_sleep);
546 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
547
548 /*
549 * FIXME: write_behind for RAID1. If not specified, what
550 * is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
551 */
552 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
553 if (write_behind > COUNTER_MAX)
554 write_behind = COUNTER_MAX / 2;
555 sb->write_behind = cpu_to_le32(write_behind);
556 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
557
558 /* keep the array size field of the bitmap superblock up to date */
559 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
560
561 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
562
563 set_bit(BITMAP_STALE, &bitmap->flags);
564 sb->state = cpu_to_le32(bitmap->flags);
565 bitmap->events_cleared = bitmap->mddev->events;
566 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
567 bitmap->mddev->bitmap_info.nodes = 0;
568
569 kunmap_atomic(sb);
570
571 return 0;
572 }
573
574 /* read the superblock from the bitmap file and initialize some bitmap fields */
575 static int bitmap_read_sb(struct bitmap *bitmap)
576 {
577 char *reason = NULL;
578 bitmap_super_t *sb;
579 unsigned long chunksize, daemon_sleep, write_behind;
580 unsigned long long events;
581 int nodes = 0;
582 unsigned long sectors_reserved = 0;
583 int err = -EINVAL;
584 struct page *sb_page;
585 loff_t offset = bitmap->mddev->bitmap_info.offset;
586
587 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
588 chunksize = 128 * 1024 * 1024;
589 daemon_sleep = 5 * HZ;
590 write_behind = 0;
591 set_bit(BITMAP_STALE, &bitmap->flags);
592 err = 0;
593 goto out_no_sb;
594 }
595 /* page 0 is the superblock, read it... */
596 sb_page = alloc_page(GFP_KERNEL);
597 if (!sb_page)
598 return -ENOMEM;
599 bitmap->storage.sb_page = sb_page;
600
601 re_read:
602 /* If cluster_slot is set, the cluster is setup */
603 if (bitmap->cluster_slot >= 0) {
604 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
605
606 sector_div(bm_blocks,
607 bitmap->mddev->bitmap_info.chunksize >> 9);
608 /* bits to bytes */
609 bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
610 /* to 4k blocks */
611 bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
612 offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
613 pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
614 bitmap->cluster_slot, offset);
615 }
616
617 if (bitmap->storage.file) {
618 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
619 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
620
621 err = read_page(bitmap->storage.file, 0,
622 bitmap, bytes, sb_page);
623 } else {
624 err = read_sb_page(bitmap->mddev,
625 offset,
626 sb_page,
627 0, sizeof(bitmap_super_t));
628 }
629 if (err)
630 return err;
631
632 err = -EINVAL;
633 sb = kmap_atomic(sb_page);
634
635 chunksize = le32_to_cpu(sb->chunksize);
636 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
637 write_behind = le32_to_cpu(sb->write_behind);
638 sectors_reserved = le32_to_cpu(sb->sectors_reserved);
639 /* Setup nodes/clustername only if bitmap version is
640 * cluster-compatible
641 */
642 if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
643 nodes = le32_to_cpu(sb->nodes);
644 strlcpy(bitmap->mddev->bitmap_info.cluster_name,
645 sb->cluster_name, 64);
646 }
647
648 /* verify that the bitmap-specific fields are valid */
649 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
650 reason = "bad magic";
651 else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
652 le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
653 reason = "unrecognized superblock version";
654 else if (chunksize < 512)
655 reason = "bitmap chunksize too small";
656 else if (!is_power_of_2(chunksize))
657 reason = "bitmap chunksize not a power of 2";
658 else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
659 reason = "daemon sleep period out of range";
660 else if (write_behind > COUNTER_MAX)
661 reason = "write-behind limit out of range (0 - 16383)";
662 if (reason) {
663 pr_warn("%s: invalid bitmap file superblock: %s\n",
664 bmname(bitmap), reason);
665 goto out;
666 }
667
668 /* keep the array size field of the bitmap superblock up to date */
669 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
670
671 if (bitmap->mddev->persistent) {
672 /*
673 * We have a persistent array superblock, so compare the
674 * bitmap's UUID and event counter to the mddev's
675 */
676 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
677 pr_warn("%s: bitmap superblock UUID mismatch\n",
678 bmname(bitmap));
679 goto out;
680 }
681 events = le64_to_cpu(sb->events);
682 if (!nodes && (events < bitmap->mddev->events)) {
683 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
684 bmname(bitmap), events,
685 (unsigned long long) bitmap->mddev->events);
686 set_bit(BITMAP_STALE, &bitmap->flags);
687 }
688 }
689
690 /* assign fields using values from superblock */
691 bitmap->flags |= le32_to_cpu(sb->state);
692 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
693 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
694 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
695 strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
696 err = 0;
697
698 out:
699 kunmap_atomic(sb);
700 /* Assigning chunksize is required for "re_read" */
701 bitmap->mddev->bitmap_info.chunksize = chunksize;
702 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
703 err = md_setup_cluster(bitmap->mddev, nodes);
704 if (err) {
705 pr_warn("%s: Could not setup cluster service (%d)\n",
706 bmname(bitmap), err);
707 goto out_no_sb;
708 }
709 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
710 goto re_read;
711 }
712
713
714 out_no_sb:
715 if (test_bit(BITMAP_STALE, &bitmap->flags))
716 bitmap->events_cleared = bitmap->mddev->events;
717 bitmap->mddev->bitmap_info.chunksize = chunksize;
718 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
719 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
720 bitmap->mddev->bitmap_info.nodes = nodes;
721 if (bitmap->mddev->bitmap_info.space == 0 ||
722 bitmap->mddev->bitmap_info.space > sectors_reserved)
723 bitmap->mddev->bitmap_info.space = sectors_reserved;
724 if (err) {
725 bitmap_print_sb(bitmap);
726 if (bitmap->cluster_slot < 0)
727 md_cluster_stop(bitmap->mddev);
728 }
729 return err;
730 }
731
732 /*
733 * general bitmap file operations
734 */
735
736 /*
737 * on-disk bitmap:
738 *
739 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
740 * file a page at a time. There's a superblock at the start of the file.
741 */
742 /* calculate the index of the page that contains this bit */
743 static inline unsigned long file_page_index(struct bitmap_storage *store,
744 unsigned long chunk)
745 {
746 if (store->sb_page)
747 chunk += sizeof(bitmap_super_t) << 3;
748 return chunk >> PAGE_BIT_SHIFT;
749 }
750
751 /* calculate the (bit) offset of this bit within a page */
752 static inline unsigned long file_page_offset(struct bitmap_storage *store,
753 unsigned long chunk)
754 {
755 if (store->sb_page)
756 chunk += sizeof(bitmap_super_t) << 3;
757 return chunk & (PAGE_BITS - 1);
758 }
759
760 /*
761 * return a pointer to the page in the filemap that contains the given bit
762 *
763 */
764 static inline struct page *filemap_get_page(struct bitmap_storage *store,
765 unsigned long chunk)
766 {
767 if (file_page_index(store, chunk) >= store->file_pages)
768 return NULL;
769 return store->filemap[file_page_index(store, chunk)];
770 }
771
772 static int bitmap_storage_alloc(struct bitmap_storage *store,
773 unsigned long chunks, int with_super,
774 int slot_number)
775 {
776 int pnum, offset = 0;
777 unsigned long num_pages;
778 unsigned long bytes;
779
780 bytes = DIV_ROUND_UP(chunks, 8);
781 if (with_super)
782 bytes += sizeof(bitmap_super_t);
783
784 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
785 offset = slot_number * num_pages;
786
787 store->filemap = kmalloc(sizeof(struct page *)
788 * num_pages, GFP_KERNEL);
789 if (!store->filemap)
790 return -ENOMEM;
791
792 if (with_super && !store->sb_page) {
793 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
794 if (store->sb_page == NULL)
795 return -ENOMEM;
796 }
797
798 pnum = 0;
799 if (store->sb_page) {
800 store->filemap[0] = store->sb_page;
801 pnum = 1;
802 store->sb_page->index = offset;
803 }
804
805 for ( ; pnum < num_pages; pnum++) {
806 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
807 if (!store->filemap[pnum]) {
808 store->file_pages = pnum;
809 return -ENOMEM;
810 }
811 store->filemap[pnum]->index = pnum + offset;
812 }
813 store->file_pages = pnum;
814
815 /* We need 4 bits per page, rounded up to a multiple
816 * of sizeof(unsigned long) */
817 store->filemap_attr = kzalloc(
818 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
819 GFP_KERNEL);
820 if (!store->filemap_attr)
821 return -ENOMEM;
822
823 store->bytes = bytes;
824
825 return 0;
826 }
827
828 static void bitmap_file_unmap(struct bitmap_storage *store)
829 {
830 struct page **map, *sb_page;
831 int pages;
832 struct file *file;
833
834 file = store->file;
835 map = store->filemap;
836 pages = store->file_pages;
837 sb_page = store->sb_page;
838
839 while (pages--)
840 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
841 free_buffers(map[pages]);
842 kfree(map);
843 kfree(store->filemap_attr);
844
845 if (sb_page)
846 free_buffers(sb_page);
847
848 if (file) {
849 struct inode *inode = file_inode(file);
850 invalidate_mapping_pages(inode->i_mapping, 0, -1);
851 fput(file);
852 }
853 }
854
855 /*
856 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
857 * then it is no longer reliable, so we stop using it and we mark the file
858 * as failed in the superblock
859 */
860 static void bitmap_file_kick(struct bitmap *bitmap)
861 {
862 char *path, *ptr = NULL;
863
864 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
865 bitmap_update_sb(bitmap);
866
867 if (bitmap->storage.file) {
868 path = kmalloc(PAGE_SIZE, GFP_KERNEL);
869 if (path)
870 ptr = file_path(bitmap->storage.file,
871 path, PAGE_SIZE);
872
873 pr_warn("%s: kicking failed bitmap file %s from array!\n",
874 bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
875
876 kfree(path);
877 } else
878 pr_warn("%s: disabling internal bitmap due to errors\n",
879 bmname(bitmap));
880 }
881 }
882
883 enum bitmap_page_attr {
884 BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
885 BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
886 * i.e. counter is 1 or 2. */
887 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
888 };
889
890 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
891 enum bitmap_page_attr attr)
892 {
893 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
894 }
895
896 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
897 enum bitmap_page_attr attr)
898 {
899 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
900 }
901
902 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
903 enum bitmap_page_attr attr)
904 {
905 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
906 }
907
908 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
909 enum bitmap_page_attr attr)
910 {
911 return test_and_clear_bit((pnum<<2) + attr,
912 bitmap->storage.filemap_attr);
913 }
914 /*
915 * bitmap_file_set_bit -- called before performing a write to the md device
916 * to set (and eventually sync) a particular bit in the bitmap file
917 *
918 * we set the bit immediately, then we record the page number so that
919 * when an unplug occurs, we can flush the dirty pages out to disk
920 */
921 static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
922 {
923 unsigned long bit;
924 struct page *page;
925 void *kaddr;
926 unsigned long chunk = block >> bitmap->counts.chunkshift;
927 struct bitmap_storage *store = &bitmap->storage;
928 unsigned long node_offset = 0;
929
930 if (mddev_is_clustered(bitmap->mddev))
931 node_offset = bitmap->cluster_slot * store->file_pages;
932
933 page = filemap_get_page(&bitmap->storage, chunk);
934 if (!page)
935 return;
936 bit = file_page_offset(&bitmap->storage, chunk);
937
938 /* set the bit */
939 kaddr = kmap_atomic(page);
940 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
941 set_bit(bit, kaddr);
942 else
943 set_bit_le(bit, kaddr);
944 kunmap_atomic(kaddr);
945 pr_debug("set file bit %lu page %lu\n", bit, page->index);
946 /* record page number so it gets flushed to disk when unplug occurs */
947 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_DIRTY);
948 }
949
950 static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
951 {
952 unsigned long bit;
953 struct page *page;
954 void *paddr;
955 unsigned long chunk = block >> bitmap->counts.chunkshift;
956 struct bitmap_storage *store = &bitmap->storage;
957 unsigned long node_offset = 0;
958
959 if (mddev_is_clustered(bitmap->mddev))
960 node_offset = bitmap->cluster_slot * store->file_pages;
961
962 page = filemap_get_page(&bitmap->storage, chunk);
963 if (!page)
964 return;
965 bit = file_page_offset(&bitmap->storage, chunk);
966 paddr = kmap_atomic(page);
967 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
968 clear_bit(bit, paddr);
969 else
970 clear_bit_le(bit, paddr);
971 kunmap_atomic(paddr);
972 if (!test_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
973 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_PENDING);
974 bitmap->allclean = 0;
975 }
976 }
977
978 static int bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
979 {
980 unsigned long bit;
981 struct page *page;
982 void *paddr;
983 unsigned long chunk = block >> bitmap->counts.chunkshift;
984 int set = 0;
985
986 page = filemap_get_page(&bitmap->storage, chunk);
987 if (!page)
988 return -EINVAL;
989 bit = file_page_offset(&bitmap->storage, chunk);
990 paddr = kmap_atomic(page);
991 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
992 set = test_bit(bit, paddr);
993 else
994 set = test_bit_le(bit, paddr);
995 kunmap_atomic(paddr);
996 return set;
997 }
998
999
1000 /* this gets called when the md device is ready to unplug its underlying
1001 * (slave) device queues -- before we let any writes go down, we need to
1002 * sync the dirty pages of the bitmap file to disk */
1003 void bitmap_unplug(struct bitmap *bitmap)
1004 {
1005 unsigned long i;
1006 int dirty, need_write;
1007 int writing = 0;
1008
1009 if (!bitmap || !bitmap->storage.filemap ||
1010 test_bit(BITMAP_STALE, &bitmap->flags))
1011 return;
1012
1013 /* look at each page to see if there are any set bits that need to be
1014 * flushed out to disk */
1015 for (i = 0; i < bitmap->storage.file_pages; i++) {
1016 if (!bitmap->storage.filemap)
1017 return;
1018 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1019 need_write = test_and_clear_page_attr(bitmap, i,
1020 BITMAP_PAGE_NEEDWRITE);
1021 if (dirty || need_write) {
1022 if (!writing) {
1023 bitmap_wait_writes(bitmap);
1024 if (bitmap->mddev->queue)
1025 blk_add_trace_msg(bitmap->mddev->queue,
1026 "md bitmap_unplug");
1027 }
1028 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1029 write_page(bitmap, bitmap->storage.filemap[i], 0);
1030 writing = 1;
1031 }
1032 }
1033 if (writing)
1034 bitmap_wait_writes(bitmap);
1035
1036 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1037 bitmap_file_kick(bitmap);
1038 }
1039 EXPORT_SYMBOL(bitmap_unplug);
1040
1041 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1042 /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
1043 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
1044 * memory mapping of the bitmap file
1045 * Special cases:
1046 * if there's no bitmap file, or if the bitmap file had been
1047 * previously kicked from the array, we mark all the bits as
1048 * 1's in order to cause a full resync.
1049 *
1050 * We ignore all bits for sectors that end earlier than 'start'.
1051 * This is used when reading an out-of-date bitmap...
1052 */
1053 static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1054 {
1055 unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
1056 struct page *page = NULL;
1057 unsigned long bit_cnt = 0;
1058 struct file *file;
1059 unsigned long offset;
1060 int outofdate;
1061 int ret = -ENOSPC;
1062 void *paddr;
1063 struct bitmap_storage *store = &bitmap->storage;
1064
1065 chunks = bitmap->counts.chunks;
1066 file = store->file;
1067
1068 if (!file && !bitmap->mddev->bitmap_info.offset) {
1069 /* No permanent bitmap - fill with '1s'. */
1070 store->filemap = NULL;
1071 store->file_pages = 0;
1072 for (i = 0; i < chunks ; i++) {
1073 /* if the disk bit is set, set the memory bit */
1074 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1075 >= start);
1076 bitmap_set_memory_bits(bitmap,
1077 (sector_t)i << bitmap->counts.chunkshift,
1078 needed);
1079 }
1080 return 0;
1081 }
1082
1083 outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1084 if (outofdate)
1085 pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
1086
1087 if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1088 pr_warn("%s: bitmap file too short %lu < %lu\n",
1089 bmname(bitmap),
1090 (unsigned long) i_size_read(file->f_mapping->host),
1091 store->bytes);
1092 goto err;
1093 }
1094
1095 oldindex = ~0L;
1096 offset = 0;
1097 if (!bitmap->mddev->bitmap_info.external)
1098 offset = sizeof(bitmap_super_t);
1099
1100 if (mddev_is_clustered(bitmap->mddev))
1101 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1102
1103 for (i = 0; i < chunks; i++) {
1104 int b;
1105 index = file_page_index(&bitmap->storage, i);
1106 bit = file_page_offset(&bitmap->storage, i);
1107 if (index != oldindex) { /* this is a new page, read it in */
1108 int count;
1109 /* unmap the old page, we're done with it */
1110 if (index == store->file_pages-1)
1111 count = store->bytes - index * PAGE_SIZE;
1112 else
1113 count = PAGE_SIZE;
1114 page = store->filemap[index];
1115 if (file)
1116 ret = read_page(file, index, bitmap,
1117 count, page);
1118 else
1119 ret = read_sb_page(
1120 bitmap->mddev,
1121 bitmap->mddev->bitmap_info.offset,
1122 page,
1123 index + node_offset, count);
1124
1125 if (ret)
1126 goto err;
1127
1128 oldindex = index;
1129
1130 if (outofdate) {
1131 /*
1132 * if bitmap is out of date, dirty the
1133 * whole page and write it out
1134 */
1135 paddr = kmap_atomic(page);
1136 memset(paddr + offset, 0xff,
1137 PAGE_SIZE - offset);
1138 kunmap_atomic(paddr);
1139 write_page(bitmap, page, 1);
1140
1141 ret = -EIO;
1142 if (test_bit(BITMAP_WRITE_ERROR,
1143 &bitmap->flags))
1144 goto err;
1145 }
1146 }
1147 paddr = kmap_atomic(page);
1148 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1149 b = test_bit(bit, paddr);
1150 else
1151 b = test_bit_le(bit, paddr);
1152 kunmap_atomic(paddr);
1153 if (b) {
1154 /* if the disk bit is set, set the memory bit */
1155 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1156 >= start);
1157 bitmap_set_memory_bits(bitmap,
1158 (sector_t)i << bitmap->counts.chunkshift,
1159 needed);
1160 bit_cnt++;
1161 }
1162 offset = 0;
1163 }
1164
1165 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1166 bmname(bitmap), store->file_pages,
1167 bit_cnt, chunks);
1168
1169 return 0;
1170
1171 err:
1172 pr_warn("%s: bitmap initialisation failed: %d\n",
1173 bmname(bitmap), ret);
1174 return ret;
1175 }
1176
1177 void bitmap_write_all(struct bitmap *bitmap)
1178 {
1179 /* We don't actually write all bitmap blocks here,
1180 * just flag them as needing to be written
1181 */
1182 int i;
1183
1184 if (!bitmap || !bitmap->storage.filemap)
1185 return;
1186 if (bitmap->storage.file)
1187 /* Only one copy, so nothing needed */
1188 return;
1189
1190 for (i = 0; i < bitmap->storage.file_pages; i++)
1191 set_page_attr(bitmap, i,
1192 BITMAP_PAGE_NEEDWRITE);
1193 bitmap->allclean = 0;
1194 }
1195
1196 static void bitmap_count_page(struct bitmap_counts *bitmap,
1197 sector_t offset, int inc)
1198 {
1199 sector_t chunk = offset >> bitmap->chunkshift;
1200 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1201 bitmap->bp[page].count += inc;
1202 bitmap_checkfree(bitmap, page);
1203 }
1204
1205 static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1206 {
1207 sector_t chunk = offset >> bitmap->chunkshift;
1208 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1209 struct bitmap_page *bp = &bitmap->bp[page];
1210
1211 if (!bp->pending)
1212 bp->pending = 1;
1213 }
1214
1215 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1216 sector_t offset, sector_t *blocks,
1217 int create);
1218
1219 /*
1220 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1221 * out to disk
1222 */
1223
1224 void bitmap_daemon_work(struct mddev *mddev)
1225 {
1226 struct bitmap *bitmap;
1227 unsigned long j;
1228 unsigned long nextpage;
1229 sector_t blocks;
1230 struct bitmap_counts *counts;
1231
1232 /* Use a mutex to guard daemon_work against
1233 * bitmap_destroy.
1234 */
1235 mutex_lock(&mddev->bitmap_info.mutex);
1236 bitmap = mddev->bitmap;
1237 if (bitmap == NULL) {
1238 mutex_unlock(&mddev->bitmap_info.mutex);
1239 return;
1240 }
1241 if (time_before(jiffies, bitmap->daemon_lastrun
1242 + mddev->bitmap_info.daemon_sleep))
1243 goto done;
1244
1245 bitmap->daemon_lastrun = jiffies;
1246 if (bitmap->allclean) {
1247 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1248 goto done;
1249 }
1250 bitmap->allclean = 1;
1251
1252 if (bitmap->mddev->queue)
1253 blk_add_trace_msg(bitmap->mddev->queue,
1254 "md bitmap_daemon_work");
1255
1256 /* Any file-page which is PENDING now needs to be written.
1257 * So set NEEDWRITE now, then after we make any last-minute changes
1258 * we will write it.
1259 */
1260 for (j = 0; j < bitmap->storage.file_pages; j++)
1261 if (test_and_clear_page_attr(bitmap, j,
1262 BITMAP_PAGE_PENDING))
1263 set_page_attr(bitmap, j,
1264 BITMAP_PAGE_NEEDWRITE);
1265
1266 if (bitmap->need_sync &&
1267 mddev->bitmap_info.external == 0) {
1268 /* Arrange for superblock update as well as
1269 * other changes */
1270 bitmap_super_t *sb;
1271 bitmap->need_sync = 0;
1272 if (bitmap->storage.filemap) {
1273 sb = kmap_atomic(bitmap->storage.sb_page);
1274 sb->events_cleared =
1275 cpu_to_le64(bitmap->events_cleared);
1276 kunmap_atomic(sb);
1277 set_page_attr(bitmap, 0,
1278 BITMAP_PAGE_NEEDWRITE);
1279 }
1280 }
1281 /* Now look at the bitmap counters and if any are '2' or '1',
1282 * decrement and handle accordingly.
1283 */
1284 counts = &bitmap->counts;
1285 spin_lock_irq(&counts->lock);
1286 nextpage = 0;
1287 for (j = 0; j < counts->chunks; j++) {
1288 bitmap_counter_t *bmc;
1289 sector_t block = (sector_t)j << counts->chunkshift;
1290
1291 if (j == nextpage) {
1292 nextpage += PAGE_COUNTER_RATIO;
1293 if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1294 j |= PAGE_COUNTER_MASK;
1295 continue;
1296 }
1297 counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1298 }
1299 bmc = bitmap_get_counter(counts,
1300 block,
1301 &blocks, 0);
1302
1303 if (!bmc) {
1304 j |= PAGE_COUNTER_MASK;
1305 continue;
1306 }
1307 if (*bmc == 1 && !bitmap->need_sync) {
1308 /* We can clear the bit */
1309 *bmc = 0;
1310 bitmap_count_page(counts, block, -1);
1311 bitmap_file_clear_bit(bitmap, block);
1312 } else if (*bmc && *bmc <= 2) {
1313 *bmc = 1;
1314 bitmap_set_pending(counts, block);
1315 bitmap->allclean = 0;
1316 }
1317 }
1318 spin_unlock_irq(&counts->lock);
1319
1320 bitmap_wait_writes(bitmap);
1321 /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1322 * DIRTY pages need to be written by bitmap_unplug so it can wait
1323 * for them.
1324 * If we find any DIRTY page we stop there and let bitmap_unplug
1325 * handle all the rest. This is important in the case where
1326 * the first blocking holds the superblock and it has been updated.
1327 * We mustn't write any other blocks before the superblock.
1328 */
1329 for (j = 0;
1330 j < bitmap->storage.file_pages
1331 && !test_bit(BITMAP_STALE, &bitmap->flags);
1332 j++) {
1333 if (test_page_attr(bitmap, j,
1334 BITMAP_PAGE_DIRTY))
1335 /* bitmap_unplug will handle the rest */
1336 break;
1337 if (test_and_clear_page_attr(bitmap, j,
1338 BITMAP_PAGE_NEEDWRITE)) {
1339 write_page(bitmap, bitmap->storage.filemap[j], 0);
1340 }
1341 }
1342
1343 done:
1344 if (bitmap->allclean == 0)
1345 mddev->thread->timeout =
1346 mddev->bitmap_info.daemon_sleep;
1347 mutex_unlock(&mddev->bitmap_info.mutex);
1348 }
1349
1350 static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1351 sector_t offset, sector_t *blocks,
1352 int create)
1353 __releases(bitmap->lock)
1354 __acquires(bitmap->lock)
1355 {
1356 /* If 'create', we might release the lock and reclaim it.
1357 * The lock must have been taken with interrupts enabled.
1358 * If !create, we don't release the lock.
1359 */
1360 sector_t chunk = offset >> bitmap->chunkshift;
1361 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1362 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1363 sector_t csize;
1364 int err;
1365
1366 err = bitmap_checkpage(bitmap, page, create, 0);
1367
1368 if (bitmap->bp[page].hijacked ||
1369 bitmap->bp[page].map == NULL)
1370 csize = ((sector_t)1) << (bitmap->chunkshift +
1371 PAGE_COUNTER_SHIFT - 1);
1372 else
1373 csize = ((sector_t)1) << bitmap->chunkshift;
1374 *blocks = csize - (offset & (csize - 1));
1375
1376 if (err < 0)
1377 return NULL;
1378
1379 /* now locked ... */
1380
1381 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1382 /* should we use the first or second counter field
1383 * of the hijacked pointer? */
1384 int hi = (pageoff > PAGE_COUNTER_MASK);
1385 return &((bitmap_counter_t *)
1386 &bitmap->bp[page].map)[hi];
1387 } else /* page is allocated */
1388 return (bitmap_counter_t *)
1389 &(bitmap->bp[page].map[pageoff]);
1390 }
1391
1392 int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1393 {
1394 if (!bitmap)
1395 return 0;
1396
1397 if (behind) {
1398 int bw;
1399 atomic_inc(&bitmap->behind_writes);
1400 bw = atomic_read(&bitmap->behind_writes);
1401 if (bw > bitmap->behind_writes_used)
1402 bitmap->behind_writes_used = bw;
1403
1404 pr_debug("inc write-behind count %d/%lu\n",
1405 bw, bitmap->mddev->bitmap_info.max_write_behind);
1406 }
1407
1408 while (sectors) {
1409 sector_t blocks;
1410 bitmap_counter_t *bmc;
1411
1412 spin_lock_irq(&bitmap->counts.lock);
1413 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1414 if (!bmc) {
1415 spin_unlock_irq(&bitmap->counts.lock);
1416 return 0;
1417 }
1418
1419 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1420 DEFINE_WAIT(__wait);
1421 /* note that it is safe to do the prepare_to_wait
1422 * after the test as long as we do it before dropping
1423 * the spinlock.
1424 */
1425 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1426 TASK_UNINTERRUPTIBLE);
1427 spin_unlock_irq(&bitmap->counts.lock);
1428 schedule();
1429 finish_wait(&bitmap->overflow_wait, &__wait);
1430 continue;
1431 }
1432
1433 switch (*bmc) {
1434 case 0:
1435 bitmap_file_set_bit(bitmap, offset);
1436 bitmap_count_page(&bitmap->counts, offset, 1);
1437 /* fall through */
1438 case 1:
1439 *bmc = 2;
1440 }
1441
1442 (*bmc)++;
1443
1444 spin_unlock_irq(&bitmap->counts.lock);
1445
1446 offset += blocks;
1447 if (sectors > blocks)
1448 sectors -= blocks;
1449 else
1450 sectors = 0;
1451 }
1452 return 0;
1453 }
1454 EXPORT_SYMBOL(bitmap_startwrite);
1455
1456 void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1457 int success, int behind)
1458 {
1459 if (!bitmap)
1460 return;
1461 if (behind) {
1462 if (atomic_dec_and_test(&bitmap->behind_writes))
1463 wake_up(&bitmap->behind_wait);
1464 pr_debug("dec write-behind count %d/%lu\n",
1465 atomic_read(&bitmap->behind_writes),
1466 bitmap->mddev->bitmap_info.max_write_behind);
1467 }
1468
1469 while (sectors) {
1470 sector_t blocks;
1471 unsigned long flags;
1472 bitmap_counter_t *bmc;
1473
1474 spin_lock_irqsave(&bitmap->counts.lock, flags);
1475 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1476 if (!bmc) {
1477 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1478 return;
1479 }
1480
1481 if (success && !bitmap->mddev->degraded &&
1482 bitmap->events_cleared < bitmap->mddev->events) {
1483 bitmap->events_cleared = bitmap->mddev->events;
1484 bitmap->need_sync = 1;
1485 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1486 }
1487
1488 if (!success && !NEEDED(*bmc))
1489 *bmc |= NEEDED_MASK;
1490
1491 if (COUNTER(*bmc) == COUNTER_MAX)
1492 wake_up(&bitmap->overflow_wait);
1493
1494 (*bmc)--;
1495 if (*bmc <= 2) {
1496 bitmap_set_pending(&bitmap->counts, offset);
1497 bitmap->allclean = 0;
1498 }
1499 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1500 offset += blocks;
1501 if (sectors > blocks)
1502 sectors -= blocks;
1503 else
1504 sectors = 0;
1505 }
1506 }
1507 EXPORT_SYMBOL(bitmap_endwrite);
1508
1509 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1510 int degraded)
1511 {
1512 bitmap_counter_t *bmc;
1513 int rv;
1514 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1515 *blocks = 1024;
1516 return 1; /* always resync if no bitmap */
1517 }
1518 spin_lock_irq(&bitmap->counts.lock);
1519 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1520 rv = 0;
1521 if (bmc) {
1522 /* locked */
1523 if (RESYNC(*bmc))
1524 rv = 1;
1525 else if (NEEDED(*bmc)) {
1526 rv = 1;
1527 if (!degraded) { /* don't set/clear bits if degraded */
1528 *bmc |= RESYNC_MASK;
1529 *bmc &= ~NEEDED_MASK;
1530 }
1531 }
1532 }
1533 spin_unlock_irq(&bitmap->counts.lock);
1534 return rv;
1535 }
1536
1537 int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1538 int degraded)
1539 {
1540 /* bitmap_start_sync must always report on multiples of whole
1541 * pages, otherwise resync (which is very PAGE_SIZE based) will
1542 * get confused.
1543 * So call __bitmap_start_sync repeatedly (if needed) until
1544 * At least PAGE_SIZE>>9 blocks are covered.
1545 * Return the 'or' of the result.
1546 */
1547 int rv = 0;
1548 sector_t blocks1;
1549
1550 *blocks = 0;
1551 while (*blocks < (PAGE_SIZE>>9)) {
1552 rv |= __bitmap_start_sync(bitmap, offset,
1553 &blocks1, degraded);
1554 offset += blocks1;
1555 *blocks += blocks1;
1556 }
1557 return rv;
1558 }
1559 EXPORT_SYMBOL(bitmap_start_sync);
1560
1561 void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1562 {
1563 bitmap_counter_t *bmc;
1564 unsigned long flags;
1565
1566 if (bitmap == NULL) {
1567 *blocks = 1024;
1568 return;
1569 }
1570 spin_lock_irqsave(&bitmap->counts.lock, flags);
1571 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1572 if (bmc == NULL)
1573 goto unlock;
1574 /* locked */
1575 if (RESYNC(*bmc)) {
1576 *bmc &= ~RESYNC_MASK;
1577
1578 if (!NEEDED(*bmc) && aborted)
1579 *bmc |= NEEDED_MASK;
1580 else {
1581 if (*bmc <= 2) {
1582 bitmap_set_pending(&bitmap->counts, offset);
1583 bitmap->allclean = 0;
1584 }
1585 }
1586 }
1587 unlock:
1588 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1589 }
1590 EXPORT_SYMBOL(bitmap_end_sync);
1591
1592 void bitmap_close_sync(struct bitmap *bitmap)
1593 {
1594 /* Sync has finished, and any bitmap chunks that weren't synced
1595 * properly have been aborted. It remains to us to clear the
1596 * RESYNC bit wherever it is still on
1597 */
1598 sector_t sector = 0;
1599 sector_t blocks;
1600 if (!bitmap)
1601 return;
1602 while (sector < bitmap->mddev->resync_max_sectors) {
1603 bitmap_end_sync(bitmap, sector, &blocks, 0);
1604 sector += blocks;
1605 }
1606 }
1607 EXPORT_SYMBOL(bitmap_close_sync);
1608
1609 void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1610 {
1611 sector_t s = 0;
1612 sector_t blocks;
1613
1614 if (!bitmap)
1615 return;
1616 if (sector == 0) {
1617 bitmap->last_end_sync = jiffies;
1618 return;
1619 }
1620 if (!force && time_before(jiffies, (bitmap->last_end_sync
1621 + bitmap->mddev->bitmap_info.daemon_sleep)))
1622 return;
1623 wait_event(bitmap->mddev->recovery_wait,
1624 atomic_read(&bitmap->mddev->recovery_active) == 0);
1625
1626 bitmap->mddev->curr_resync_completed = sector;
1627 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1628 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1629 s = 0;
1630 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1631 bitmap_end_sync(bitmap, s, &blocks, 0);
1632 s += blocks;
1633 }
1634 bitmap->last_end_sync = jiffies;
1635 sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1636 }
1637 EXPORT_SYMBOL(bitmap_cond_end_sync);
1638
1639 void bitmap_sync_with_cluster(struct mddev *mddev,
1640 sector_t old_lo, sector_t old_hi,
1641 sector_t new_lo, sector_t new_hi)
1642 {
1643 struct bitmap *bitmap = mddev->bitmap;
1644 sector_t sector, blocks = 0;
1645
1646 for (sector = old_lo; sector < new_lo; ) {
1647 bitmap_end_sync(bitmap, sector, &blocks, 0);
1648 sector += blocks;
1649 }
1650 WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
1651
1652 for (sector = old_hi; sector < new_hi; ) {
1653 bitmap_start_sync(bitmap, sector, &blocks, 0);
1654 sector += blocks;
1655 }
1656 WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
1657 }
1658 EXPORT_SYMBOL(bitmap_sync_with_cluster);
1659
1660 static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1661 {
1662 /* For each chunk covered by any of these sectors, set the
1663 * counter to 2 and possibly set resync_needed. They should all
1664 * be 0 at this point
1665 */
1666
1667 sector_t secs;
1668 bitmap_counter_t *bmc;
1669 spin_lock_irq(&bitmap->counts.lock);
1670 bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1671 if (!bmc) {
1672 spin_unlock_irq(&bitmap->counts.lock);
1673 return;
1674 }
1675 if (!*bmc) {
1676 *bmc = 2;
1677 bitmap_count_page(&bitmap->counts, offset, 1);
1678 bitmap_set_pending(&bitmap->counts, offset);
1679 bitmap->allclean = 0;
1680 }
1681 if (needed)
1682 *bmc |= NEEDED_MASK;
1683 spin_unlock_irq(&bitmap->counts.lock);
1684 }
1685
1686 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1687 void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1688 {
1689 unsigned long chunk;
1690
1691 for (chunk = s; chunk <= e; chunk++) {
1692 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1693 bitmap_set_memory_bits(bitmap, sec, 1);
1694 bitmap_file_set_bit(bitmap, sec);
1695 if (sec < bitmap->mddev->recovery_cp)
1696 /* We are asserting that the array is dirty,
1697 * so move the recovery_cp address back so
1698 * that it is obvious that it is dirty
1699 */
1700 bitmap->mddev->recovery_cp = sec;
1701 }
1702 }
1703
1704 /*
1705 * flush out any pending updates
1706 */
1707 void bitmap_flush(struct mddev *mddev)
1708 {
1709 struct bitmap *bitmap = mddev->bitmap;
1710 long sleep;
1711
1712 if (!bitmap) /* there was no bitmap */
1713 return;
1714
1715 /* run the daemon_work three time to ensure everything is flushed
1716 * that can be
1717 */
1718 sleep = mddev->bitmap_info.daemon_sleep * 2;
1719 bitmap->daemon_lastrun -= sleep;
1720 bitmap_daemon_work(mddev);
1721 bitmap->daemon_lastrun -= sleep;
1722 bitmap_daemon_work(mddev);
1723 bitmap->daemon_lastrun -= sleep;
1724 bitmap_daemon_work(mddev);
1725 bitmap_update_sb(bitmap);
1726 }
1727
1728 /*
1729 * free memory that was allocated
1730 */
1731 void bitmap_free(struct bitmap *bitmap)
1732 {
1733 unsigned long k, pages;
1734 struct bitmap_page *bp;
1735
1736 if (!bitmap) /* there was no bitmap */
1737 return;
1738
1739 if (bitmap->sysfs_can_clear)
1740 sysfs_put(bitmap->sysfs_can_clear);
1741
1742 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1743 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1744 md_cluster_stop(bitmap->mddev);
1745
1746 /* Shouldn't be needed - but just in case.... */
1747 wait_event(bitmap->write_wait,
1748 atomic_read(&bitmap->pending_writes) == 0);
1749
1750 /* release the bitmap file */
1751 bitmap_file_unmap(&bitmap->storage);
1752
1753 bp = bitmap->counts.bp;
1754 pages = bitmap->counts.pages;
1755
1756 /* free all allocated memory */
1757
1758 if (bp) /* deallocate the page memory */
1759 for (k = 0; k < pages; k++)
1760 if (bp[k].map && !bp[k].hijacked)
1761 kfree(bp[k].map);
1762 kfree(bp);
1763 kfree(bitmap);
1764 }
1765 EXPORT_SYMBOL(bitmap_free);
1766
1767 void bitmap_wait_behind_writes(struct mddev *mddev)
1768 {
1769 struct bitmap *bitmap = mddev->bitmap;
1770
1771 /* wait for behind writes to complete */
1772 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
1773 pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
1774 mdname(mddev));
1775 /* need to kick something here to make sure I/O goes? */
1776 wait_event(bitmap->behind_wait,
1777 atomic_read(&bitmap->behind_writes) == 0);
1778 }
1779 }
1780
1781 void bitmap_destroy(struct mddev *mddev)
1782 {
1783 struct bitmap *bitmap = mddev->bitmap;
1784
1785 if (!bitmap) /* there was no bitmap */
1786 return;
1787
1788 bitmap_wait_behind_writes(mddev);
1789
1790 mutex_lock(&mddev->bitmap_info.mutex);
1791 spin_lock(&mddev->lock);
1792 mddev->bitmap = NULL; /* disconnect from the md device */
1793 spin_unlock(&mddev->lock);
1794 mutex_unlock(&mddev->bitmap_info.mutex);
1795 if (mddev->thread)
1796 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1797
1798 bitmap_free(bitmap);
1799 }
1800
1801 /*
1802 * initialize the bitmap structure
1803 * if this returns an error, bitmap_destroy must be called to do clean up
1804 * once mddev->bitmap is set
1805 */
1806 struct bitmap *bitmap_create(struct mddev *mddev, int slot)
1807 {
1808 struct bitmap *bitmap;
1809 sector_t blocks = mddev->resync_max_sectors;
1810 struct file *file = mddev->bitmap_info.file;
1811 int err;
1812 struct kernfs_node *bm = NULL;
1813
1814 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1815
1816 BUG_ON(file && mddev->bitmap_info.offset);
1817
1818 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1819 if (!bitmap)
1820 return ERR_PTR(-ENOMEM);
1821
1822 spin_lock_init(&bitmap->counts.lock);
1823 atomic_set(&bitmap->pending_writes, 0);
1824 init_waitqueue_head(&bitmap->write_wait);
1825 init_waitqueue_head(&bitmap->overflow_wait);
1826 init_waitqueue_head(&bitmap->behind_wait);
1827
1828 bitmap->mddev = mddev;
1829 bitmap->cluster_slot = slot;
1830
1831 if (mddev->kobj.sd)
1832 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1833 if (bm) {
1834 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1835 sysfs_put(bm);
1836 } else
1837 bitmap->sysfs_can_clear = NULL;
1838
1839 bitmap->storage.file = file;
1840 if (file) {
1841 get_file(file);
1842 /* As future accesses to this file will use bmap,
1843 * and bypass the page cache, we must sync the file
1844 * first.
1845 */
1846 vfs_fsync(file, 1);
1847 }
1848 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1849 if (!mddev->bitmap_info.external) {
1850 /*
1851 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
1852 * instructing us to create a new on-disk bitmap instance.
1853 */
1854 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1855 err = bitmap_new_disk_sb(bitmap);
1856 else
1857 err = bitmap_read_sb(bitmap);
1858 } else {
1859 err = 0;
1860 if (mddev->bitmap_info.chunksize == 0 ||
1861 mddev->bitmap_info.daemon_sleep == 0)
1862 /* chunksize and time_base need to be
1863 * set first. */
1864 err = -EINVAL;
1865 }
1866 if (err)
1867 goto error;
1868
1869 bitmap->daemon_lastrun = jiffies;
1870 err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1871 if (err)
1872 goto error;
1873
1874 pr_debug("created bitmap (%lu pages) for device %s\n",
1875 bitmap->counts.pages, bmname(bitmap));
1876
1877 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1878 if (err)
1879 goto error;
1880
1881 return bitmap;
1882 error:
1883 bitmap_free(bitmap);
1884 return ERR_PTR(err);
1885 }
1886
1887 int bitmap_load(struct mddev *mddev)
1888 {
1889 int err = 0;
1890 sector_t start = 0;
1891 sector_t sector = 0;
1892 struct bitmap *bitmap = mddev->bitmap;
1893
1894 if (!bitmap)
1895 goto out;
1896
1897 if (mddev_is_clustered(mddev))
1898 md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
1899
1900 /* Clear out old bitmap info first: Either there is none, or we
1901 * are resuming after someone else has possibly changed things,
1902 * so we should forget old cached info.
1903 * All chunks should be clean, but some might need_sync.
1904 */
1905 while (sector < mddev->resync_max_sectors) {
1906 sector_t blocks;
1907 bitmap_start_sync(bitmap, sector, &blocks, 0);
1908 sector += blocks;
1909 }
1910 bitmap_close_sync(bitmap);
1911
1912 if (mddev->degraded == 0
1913 || bitmap->events_cleared == mddev->events)
1914 /* no need to keep dirty bits to optimise a
1915 * re-add of a missing device */
1916 start = mddev->recovery_cp;
1917
1918 mutex_lock(&mddev->bitmap_info.mutex);
1919 err = bitmap_init_from_disk(bitmap, start);
1920 mutex_unlock(&mddev->bitmap_info.mutex);
1921
1922 if (err)
1923 goto out;
1924 clear_bit(BITMAP_STALE, &bitmap->flags);
1925
1926 /* Kick recovery in case any bits were set */
1927 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1928
1929 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1930 md_wakeup_thread(mddev->thread);
1931
1932 bitmap_update_sb(bitmap);
1933
1934 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1935 err = -EIO;
1936 out:
1937 return err;
1938 }
1939 EXPORT_SYMBOL_GPL(bitmap_load);
1940
1941 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
1942 {
1943 int rv = 0;
1944 struct bitmap *bitmap;
1945
1946 bitmap = bitmap_create(mddev, slot);
1947 if (IS_ERR(bitmap)) {
1948 rv = PTR_ERR(bitmap);
1949 return ERR_PTR(rv);
1950 }
1951
1952 rv = bitmap_init_from_disk(bitmap, 0);
1953 if (rv) {
1954 bitmap_free(bitmap);
1955 return ERR_PTR(rv);
1956 }
1957
1958 return bitmap;
1959 }
1960 EXPORT_SYMBOL(get_bitmap_from_slot);
1961
1962 /* Loads the bitmap associated with slot and copies the resync information
1963 * to our bitmap
1964 */
1965 int bitmap_copy_from_slot(struct mddev *mddev, int slot,
1966 sector_t *low, sector_t *high, bool clear_bits)
1967 {
1968 int rv = 0, i, j;
1969 sector_t block, lo = 0, hi = 0;
1970 struct bitmap_counts *counts;
1971 struct bitmap *bitmap;
1972
1973 bitmap = get_bitmap_from_slot(mddev, slot);
1974 if (IS_ERR(bitmap)) {
1975 pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
1976 return -1;
1977 }
1978
1979 counts = &bitmap->counts;
1980 for (j = 0; j < counts->chunks; j++) {
1981 block = (sector_t)j << counts->chunkshift;
1982 if (bitmap_file_test_bit(bitmap, block)) {
1983 if (!lo)
1984 lo = block;
1985 hi = block;
1986 bitmap_file_clear_bit(bitmap, block);
1987 bitmap_set_memory_bits(mddev->bitmap, block, 1);
1988 bitmap_file_set_bit(mddev->bitmap, block);
1989 }
1990 }
1991
1992 if (clear_bits) {
1993 bitmap_update_sb(bitmap);
1994 /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
1995 * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
1996 for (i = 0; i < bitmap->storage.file_pages; i++)
1997 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
1998 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
1999 bitmap_unplug(bitmap);
2000 }
2001 bitmap_unplug(mddev->bitmap);
2002 *low = lo;
2003 *high = hi;
2004
2005 return rv;
2006 }
2007 EXPORT_SYMBOL_GPL(bitmap_copy_from_slot);
2008
2009
2010 void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
2011 {
2012 unsigned long chunk_kb;
2013 struct bitmap_counts *counts;
2014
2015 if (!bitmap)
2016 return;
2017
2018 counts = &bitmap->counts;
2019
2020 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
2021 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
2022 "%lu%s chunk",
2023 counts->pages - counts->missing_pages,
2024 counts->pages,
2025 (counts->pages - counts->missing_pages)
2026 << (PAGE_SHIFT - 10),
2027 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
2028 chunk_kb ? "KB" : "B");
2029 if (bitmap->storage.file) {
2030 seq_printf(seq, ", file: ");
2031 seq_file_path(seq, bitmap->storage.file, " \t\n");
2032 }
2033
2034 seq_printf(seq, "\n");
2035 }
2036
2037 int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2038 int chunksize, int init)
2039 {
2040 /* If chunk_size is 0, choose an appropriate chunk size.
2041 * Then possibly allocate new storage space.
2042 * Then quiesce, copy bits, replace bitmap, and re-start
2043 *
2044 * This function is called both to set up the initial bitmap
2045 * and to resize the bitmap while the array is active.
2046 * If this happens as a result of the array being resized,
2047 * chunksize will be zero, and we need to choose a suitable
2048 * chunksize, otherwise we use what we are given.
2049 */
2050 struct bitmap_storage store;
2051 struct bitmap_counts old_counts;
2052 unsigned long chunks;
2053 sector_t block;
2054 sector_t old_blocks, new_blocks;
2055 int chunkshift;
2056 int ret = 0;
2057 long pages;
2058 struct bitmap_page *new_bp;
2059
2060 if (chunksize == 0) {
2061 /* If there is enough space, leave the chunk size unchanged,
2062 * else increase by factor of two until there is enough space.
2063 */
2064 long bytes;
2065 long space = bitmap->mddev->bitmap_info.space;
2066
2067 if (space == 0) {
2068 /* We don't know how much space there is, so limit
2069 * to current size - in sectors.
2070 */
2071 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2072 if (!bitmap->mddev->bitmap_info.external)
2073 bytes += sizeof(bitmap_super_t);
2074 space = DIV_ROUND_UP(bytes, 512);
2075 bitmap->mddev->bitmap_info.space = space;
2076 }
2077 chunkshift = bitmap->counts.chunkshift;
2078 chunkshift--;
2079 do {
2080 /* 'chunkshift' is shift from block size to chunk size */
2081 chunkshift++;
2082 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2083 bytes = DIV_ROUND_UP(chunks, 8);
2084 if (!bitmap->mddev->bitmap_info.external)
2085 bytes += sizeof(bitmap_super_t);
2086 } while (bytes > (space << 9));
2087 } else
2088 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
2089
2090 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
2091 memset(&store, 0, sizeof(store));
2092 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2093 ret = bitmap_storage_alloc(&store, chunks,
2094 !bitmap->mddev->bitmap_info.external,
2095 mddev_is_clustered(bitmap->mddev)
2096 ? bitmap->cluster_slot : 0);
2097 if (ret) {
2098 bitmap_file_unmap(&store);
2099 goto err;
2100 }
2101
2102 pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
2103
2104 new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
2105 ret = -ENOMEM;
2106 if (!new_bp) {
2107 bitmap_file_unmap(&store);
2108 goto err;
2109 }
2110
2111 if (!init)
2112 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2113
2114 store.file = bitmap->storage.file;
2115 bitmap->storage.file = NULL;
2116
2117 if (store.sb_page && bitmap->storage.sb_page)
2118 memcpy(page_address(store.sb_page),
2119 page_address(bitmap->storage.sb_page),
2120 sizeof(bitmap_super_t));
2121 bitmap_file_unmap(&bitmap->storage);
2122 bitmap->storage = store;
2123
2124 old_counts = bitmap->counts;
2125 bitmap->counts.bp = new_bp;
2126 bitmap->counts.pages = pages;
2127 bitmap->counts.missing_pages = pages;
2128 bitmap->counts.chunkshift = chunkshift;
2129 bitmap->counts.chunks = chunks;
2130 bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
2131 BITMAP_BLOCK_SHIFT);
2132
2133 blocks = min(old_counts.chunks << old_counts.chunkshift,
2134 chunks << chunkshift);
2135
2136 spin_lock_irq(&bitmap->counts.lock);
2137 /* For cluster raid, need to pre-allocate bitmap */
2138 if (mddev_is_clustered(bitmap->mddev)) {
2139 unsigned long page;
2140 for (page = 0; page < pages; page++) {
2141 ret = bitmap_checkpage(&bitmap->counts, page, 1, 1);
2142 if (ret) {
2143 unsigned long k;
2144
2145 /* deallocate the page memory */
2146 for (k = 0; k < page; k++) {
2147 kfree(new_bp[k].map);
2148 }
2149
2150 /* restore some fields from old_counts */
2151 bitmap->counts.bp = old_counts.bp;
2152 bitmap->counts.pages = old_counts.pages;
2153 bitmap->counts.missing_pages = old_counts.pages;
2154 bitmap->counts.chunkshift = old_counts.chunkshift;
2155 bitmap->counts.chunks = old_counts.chunks;
2156 bitmap->mddev->bitmap_info.chunksize = 1 << (old_counts.chunkshift +
2157 BITMAP_BLOCK_SHIFT);
2158 blocks = old_counts.chunks << old_counts.chunkshift;
2159 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2160 break;
2161 } else
2162 bitmap->counts.bp[page].count += 1;
2163 }
2164 }
2165
2166 for (block = 0; block < blocks; ) {
2167 bitmap_counter_t *bmc_old, *bmc_new;
2168 int set;
2169
2170 bmc_old = bitmap_get_counter(&old_counts, block,
2171 &old_blocks, 0);
2172 set = bmc_old && NEEDED(*bmc_old);
2173
2174 if (set) {
2175 bmc_new = bitmap_get_counter(&bitmap->counts, block,
2176 &new_blocks, 1);
2177 if (*bmc_new == 0) {
2178 /* need to set on-disk bits too. */
2179 sector_t end = block + new_blocks;
2180 sector_t start = block >> chunkshift;
2181 start <<= chunkshift;
2182 while (start < end) {
2183 bitmap_file_set_bit(bitmap, block);
2184 start += 1 << chunkshift;
2185 }
2186 *bmc_new = 2;
2187 bitmap_count_page(&bitmap->counts,
2188 block, 1);
2189 bitmap_set_pending(&bitmap->counts,
2190 block);
2191 }
2192 *bmc_new |= NEEDED_MASK;
2193 if (new_blocks < old_blocks)
2194 old_blocks = new_blocks;
2195 }
2196 block += old_blocks;
2197 }
2198
2199 if (!init) {
2200 int i;
2201 while (block < (chunks << chunkshift)) {
2202 bitmap_counter_t *bmc;
2203 bmc = bitmap_get_counter(&bitmap->counts, block,
2204 &new_blocks, 1);
2205 if (bmc) {
2206 /* new space. It needs to be resynced, so
2207 * we set NEEDED_MASK.
2208 */
2209 if (*bmc == 0) {
2210 *bmc = NEEDED_MASK | 2;
2211 bitmap_count_page(&bitmap->counts,
2212 block, 1);
2213 bitmap_set_pending(&bitmap->counts,
2214 block);
2215 }
2216 }
2217 block += new_blocks;
2218 }
2219 for (i = 0; i < bitmap->storage.file_pages; i++)
2220 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2221 }
2222 spin_unlock_irq(&bitmap->counts.lock);
2223
2224 if (!init) {
2225 bitmap_unplug(bitmap);
2226 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2227 }
2228 ret = 0;
2229 err:
2230 return ret;
2231 }
2232 EXPORT_SYMBOL_GPL(bitmap_resize);
2233
2234 static ssize_t
2235 location_show(struct mddev *mddev, char *page)
2236 {
2237 ssize_t len;
2238 if (mddev->bitmap_info.file)
2239 len = sprintf(page, "file");
2240 else if (mddev->bitmap_info.offset)
2241 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
2242 else
2243 len = sprintf(page, "none");
2244 len += sprintf(page+len, "\n");
2245 return len;
2246 }
2247
2248 static ssize_t
2249 location_store(struct mddev *mddev, const char *buf, size_t len)
2250 {
2251 int rv;
2252
2253 rv = mddev_lock(mddev);
2254 if (rv)
2255 return rv;
2256 if (mddev->pers) {
2257 if (!mddev->pers->quiesce) {
2258 rv = -EBUSY;
2259 goto out;
2260 }
2261 if (mddev->recovery || mddev->sync_thread) {
2262 rv = -EBUSY;
2263 goto out;
2264 }
2265 }
2266
2267 if (mddev->bitmap || mddev->bitmap_info.file ||
2268 mddev->bitmap_info.offset) {
2269 /* bitmap already configured. Only option is to clear it */
2270 if (strncmp(buf, "none", 4) != 0) {
2271 rv = -EBUSY;
2272 goto out;
2273 }
2274 if (mddev->pers) {
2275 mddev->pers->quiesce(mddev, 1);
2276 bitmap_destroy(mddev);
2277 mddev->pers->quiesce(mddev, 0);
2278 }
2279 mddev->bitmap_info.offset = 0;
2280 if (mddev->bitmap_info.file) {
2281 struct file *f = mddev->bitmap_info.file;
2282 mddev->bitmap_info.file = NULL;
2283 fput(f);
2284 }
2285 } else {
2286 /* No bitmap, OK to set a location */
2287 long long offset;
2288 if (strncmp(buf, "none", 4) == 0)
2289 /* nothing to be done */;
2290 else if (strncmp(buf, "file:", 5) == 0) {
2291 /* Not supported yet */
2292 rv = -EINVAL;
2293 goto out;
2294 } else {
2295 if (buf[0] == '+')
2296 rv = kstrtoll(buf+1, 10, &offset);
2297 else
2298 rv = kstrtoll(buf, 10, &offset);
2299 if (rv)
2300 goto out;
2301 if (offset == 0) {
2302 rv = -EINVAL;
2303 goto out;
2304 }
2305 if (mddev->bitmap_info.external == 0 &&
2306 mddev->major_version == 0 &&
2307 offset != mddev->bitmap_info.default_offset) {
2308 rv = -EINVAL;
2309 goto out;
2310 }
2311 mddev->bitmap_info.offset = offset;
2312 if (mddev->pers) {
2313 struct bitmap *bitmap;
2314 mddev->pers->quiesce(mddev, 1);
2315 bitmap = bitmap_create(mddev, -1);
2316 if (IS_ERR(bitmap))
2317 rv = PTR_ERR(bitmap);
2318 else {
2319 mddev->bitmap = bitmap;
2320 rv = bitmap_load(mddev);
2321 if (rv)
2322 mddev->bitmap_info.offset = 0;
2323 }
2324 mddev->pers->quiesce(mddev, 0);
2325 if (rv) {
2326 bitmap_destroy(mddev);
2327 goto out;
2328 }
2329 }
2330 }
2331 }
2332 if (!mddev->external) {
2333 /* Ensure new bitmap info is stored in
2334 * metadata promptly.
2335 */
2336 set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
2337 md_wakeup_thread(mddev->thread);
2338 }
2339 rv = 0;
2340 out:
2341 mddev_unlock(mddev);
2342 if (rv)
2343 return rv;
2344 return len;
2345 }
2346
2347 static struct md_sysfs_entry bitmap_location =
2348 __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2349
2350 /* 'bitmap/space' is the space available at 'location' for the
2351 * bitmap. This allows the kernel to know when it is safe to
2352 * resize the bitmap to match a resized array.
2353 */
2354 static ssize_t
2355 space_show(struct mddev *mddev, char *page)
2356 {
2357 return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2358 }
2359
2360 static ssize_t
2361 space_store(struct mddev *mddev, const char *buf, size_t len)
2362 {
2363 unsigned long sectors;
2364 int rv;
2365
2366 rv = kstrtoul(buf, 10, &sectors);
2367 if (rv)
2368 return rv;
2369
2370 if (sectors == 0)
2371 return -EINVAL;
2372
2373 if (mddev->bitmap &&
2374 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2375 return -EFBIG; /* Bitmap is too big for this small space */
2376
2377 /* could make sure it isn't too big, but that isn't really
2378 * needed - user-space should be careful.
2379 */
2380 mddev->bitmap_info.space = sectors;
2381 return len;
2382 }
2383
2384 static struct md_sysfs_entry bitmap_space =
2385 __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2386
2387 static ssize_t
2388 timeout_show(struct mddev *mddev, char *page)
2389 {
2390 ssize_t len;
2391 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2392 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2393
2394 len = sprintf(page, "%lu", secs);
2395 if (jifs)
2396 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2397 len += sprintf(page+len, "\n");
2398 return len;
2399 }
2400
2401 static ssize_t
2402 timeout_store(struct mddev *mddev, const char *buf, size_t len)
2403 {
2404 /* timeout can be set at any time */
2405 unsigned long timeout;
2406 int rv = strict_strtoul_scaled(buf, &timeout, 4);
2407 if (rv)
2408 return rv;
2409
2410 /* just to make sure we don't overflow... */
2411 if (timeout >= LONG_MAX / HZ)
2412 return -EINVAL;
2413
2414 timeout = timeout * HZ / 10000;
2415
2416 if (timeout >= MAX_SCHEDULE_TIMEOUT)
2417 timeout = MAX_SCHEDULE_TIMEOUT-1;
2418 if (timeout < 1)
2419 timeout = 1;
2420 mddev->bitmap_info.daemon_sleep = timeout;
2421 if (mddev->thread) {
2422 /* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
2423 * the bitmap is all clean and we don't need to
2424 * adjust the timeout right now
2425 */
2426 if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2427 mddev->thread->timeout = timeout;
2428 md_wakeup_thread(mddev->thread);
2429 }
2430 }
2431 return len;
2432 }
2433
2434 static struct md_sysfs_entry bitmap_timeout =
2435 __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2436
2437 static ssize_t
2438 backlog_show(struct mddev *mddev, char *page)
2439 {
2440 return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2441 }
2442
2443 static ssize_t
2444 backlog_store(struct mddev *mddev, const char *buf, size_t len)
2445 {
2446 unsigned long backlog;
2447 int rv = kstrtoul(buf, 10, &backlog);
2448 if (rv)
2449 return rv;
2450 if (backlog > COUNTER_MAX)
2451 return -EINVAL;
2452 mddev->bitmap_info.max_write_behind = backlog;
2453 return len;
2454 }
2455
2456 static struct md_sysfs_entry bitmap_backlog =
2457 __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2458
2459 static ssize_t
2460 chunksize_show(struct mddev *mddev, char *page)
2461 {
2462 return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2463 }
2464
2465 static ssize_t
2466 chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2467 {
2468 /* Can only be changed when no bitmap is active */
2469 int rv;
2470 unsigned long csize;
2471 if (mddev->bitmap)
2472 return -EBUSY;
2473 rv = kstrtoul(buf, 10, &csize);
2474 if (rv)
2475 return rv;
2476 if (csize < 512 ||
2477 !is_power_of_2(csize))
2478 return -EINVAL;
2479 mddev->bitmap_info.chunksize = csize;
2480 return len;
2481 }
2482
2483 static struct md_sysfs_entry bitmap_chunksize =
2484 __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2485
2486 static ssize_t metadata_show(struct mddev *mddev, char *page)
2487 {
2488 if (mddev_is_clustered(mddev))
2489 return sprintf(page, "clustered\n");
2490 return sprintf(page, "%s\n", (mddev->bitmap_info.external
2491 ? "external" : "internal"));
2492 }
2493
2494 static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2495 {
2496 if (mddev->bitmap ||
2497 mddev->bitmap_info.file ||
2498 mddev->bitmap_info.offset)
2499 return -EBUSY;
2500 if (strncmp(buf, "external", 8) == 0)
2501 mddev->bitmap_info.external = 1;
2502 else if ((strncmp(buf, "internal", 8) == 0) ||
2503 (strncmp(buf, "clustered", 9) == 0))
2504 mddev->bitmap_info.external = 0;
2505 else
2506 return -EINVAL;
2507 return len;
2508 }
2509
2510 static struct md_sysfs_entry bitmap_metadata =
2511 __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2512
2513 static ssize_t can_clear_show(struct mddev *mddev, char *page)
2514 {
2515 int len;
2516 spin_lock(&mddev->lock);
2517 if (mddev->bitmap)
2518 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2519 "false" : "true"));
2520 else
2521 len = sprintf(page, "\n");
2522 spin_unlock(&mddev->lock);
2523 return len;
2524 }
2525
2526 static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2527 {
2528 if (mddev->bitmap == NULL)
2529 return -ENOENT;
2530 if (strncmp(buf, "false", 5) == 0)
2531 mddev->bitmap->need_sync = 1;
2532 else if (strncmp(buf, "true", 4) == 0) {
2533 if (mddev->degraded)
2534 return -EBUSY;
2535 mddev->bitmap->need_sync = 0;
2536 } else
2537 return -EINVAL;
2538 return len;
2539 }
2540
2541 static struct md_sysfs_entry bitmap_can_clear =
2542 __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2543
2544 static ssize_t
2545 behind_writes_used_show(struct mddev *mddev, char *page)
2546 {
2547 ssize_t ret;
2548 spin_lock(&mddev->lock);
2549 if (mddev->bitmap == NULL)
2550 ret = sprintf(page, "0\n");
2551 else
2552 ret = sprintf(page, "%lu\n",
2553 mddev->bitmap->behind_writes_used);
2554 spin_unlock(&mddev->lock);
2555 return ret;
2556 }
2557
2558 static ssize_t
2559 behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2560 {
2561 if (mddev->bitmap)
2562 mddev->bitmap->behind_writes_used = 0;
2563 return len;
2564 }
2565
2566 static struct md_sysfs_entry max_backlog_used =
2567 __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2568 behind_writes_used_show, behind_writes_used_reset);
2569
2570 static struct attribute *md_bitmap_attrs[] = {
2571 &bitmap_location.attr,
2572 &bitmap_space.attr,
2573 &bitmap_timeout.attr,
2574 &bitmap_backlog.attr,
2575 &bitmap_chunksize.attr,
2576 &bitmap_metadata.attr,
2577 &bitmap_can_clear.attr,
2578 &max_backlog_used.attr,
2579 NULL
2580 };
2581 struct attribute_group md_bitmap_group = {
2582 .name = "bitmap",
2583 .attrs = md_bitmap_attrs,
2584 };
2585
Linux with added FPC-III support