2 * Copyright (C) 2011 Red Hat, Inc.
4 * This file is released under the GPL.
7 #include "dm-space-map-common.h"
8 #include "dm-transaction-manager.h"
10 #include <linux/bitops.h>
11 #include <linux/device-mapper.h>
13 #define DM_MSG_PREFIX "space map common"
15 /*----------------------------------------------------------------*/
20 #define INDEX_CSUM_XOR 160478
22 static void index_prepare_for_write(struct dm_block_validator
*v
,
26 struct disk_metadata_index
*mi_le
= dm_block_data(b
);
28 mi_le
->blocknr
= cpu_to_le64(dm_block_location(b
));
29 mi_le
->csum
= cpu_to_le32(dm_bm_checksum(&mi_le
->padding
,
30 block_size
- sizeof(__le32
),
34 static int index_check(struct dm_block_validator
*v
,
38 struct disk_metadata_index
*mi_le
= dm_block_data(b
);
41 if (dm_block_location(b
) != le64_to_cpu(mi_le
->blocknr
)) {
42 DMERR_LIMIT("index_check failed: blocknr %llu != wanted %llu",
43 le64_to_cpu(mi_le
->blocknr
), dm_block_location(b
));
47 csum_disk
= cpu_to_le32(dm_bm_checksum(&mi_le
->padding
,
48 block_size
- sizeof(__le32
),
50 if (csum_disk
!= mi_le
->csum
) {
51 DMERR_LIMIT("index_check failed: csum %u != wanted %u",
52 le32_to_cpu(csum_disk
), le32_to_cpu(mi_le
->csum
));
59 static struct dm_block_validator index_validator
= {
61 .prepare_for_write
= index_prepare_for_write
,
65 /*----------------------------------------------------------------*/
70 #define BITMAP_CSUM_XOR 240779
72 static void bitmap_prepare_for_write(struct dm_block_validator
*v
,
76 struct disk_bitmap_header
*disk_header
= dm_block_data(b
);
78 disk_header
->blocknr
= cpu_to_le64(dm_block_location(b
));
79 disk_header
->csum
= cpu_to_le32(dm_bm_checksum(&disk_header
->not_used
,
80 block_size
- sizeof(__le32
),
84 static int bitmap_check(struct dm_block_validator
*v
,
88 struct disk_bitmap_header
*disk_header
= dm_block_data(b
);
91 if (dm_block_location(b
) != le64_to_cpu(disk_header
->blocknr
)) {
92 DMERR_LIMIT("bitmap check failed: blocknr %llu != wanted %llu",
93 le64_to_cpu(disk_header
->blocknr
), dm_block_location(b
));
97 csum_disk
= cpu_to_le32(dm_bm_checksum(&disk_header
->not_used
,
98 block_size
- sizeof(__le32
),
100 if (csum_disk
!= disk_header
->csum
) {
101 DMERR_LIMIT("bitmap check failed: csum %u != wanted %u",
102 le32_to_cpu(csum_disk
), le32_to_cpu(disk_header
->csum
));
109 static struct dm_block_validator dm_sm_bitmap_validator
= {
111 .prepare_for_write
= bitmap_prepare_for_write
,
112 .check
= bitmap_check
115 /*----------------------------------------------------------------*/
117 #define ENTRIES_PER_WORD 32
118 #define ENTRIES_SHIFT 5
120 static void *dm_bitmap_data(struct dm_block
*b
)
122 return dm_block_data(b
) + sizeof(struct disk_bitmap_header
);
125 #define WORD_MASK_HIGH 0xAAAAAAAAAAAAAAAAULL
127 static unsigned bitmap_word_used(void *addr
, unsigned b
)
129 __le64
*words_le
= addr
;
130 __le64
*w_le
= words_le
+ (b
>> ENTRIES_SHIFT
);
132 uint64_t bits
= le64_to_cpu(*w_le
);
133 uint64_t mask
= (bits
+ WORD_MASK_HIGH
+ 1) & WORD_MASK_HIGH
;
135 return !(~bits
& mask
);
138 static unsigned sm_lookup_bitmap(void *addr
, unsigned b
)
140 __le64
*words_le
= addr
;
141 __le64
*w_le
= words_le
+ (b
>> ENTRIES_SHIFT
);
144 b
= (b
& (ENTRIES_PER_WORD
- 1)) << 1;
145 hi
= !!test_bit_le(b
, (void *) w_le
);
146 lo
= !!test_bit_le(b
+ 1, (void *) w_le
);
147 return (hi
<< 1) | lo
;
150 static void sm_set_bitmap(void *addr
, unsigned b
, unsigned val
)
152 __le64
*words_le
= addr
;
153 __le64
*w_le
= words_le
+ (b
>> ENTRIES_SHIFT
);
155 b
= (b
& (ENTRIES_PER_WORD
- 1)) << 1;
158 __set_bit_le(b
, (void *) w_le
);
160 __clear_bit_le(b
, (void *) w_le
);
163 __set_bit_le(b
+ 1, (void *) w_le
);
165 __clear_bit_le(b
+ 1, (void *) w_le
);
168 static int sm_find_free(void *addr
, unsigned begin
, unsigned end
,
171 while (begin
< end
) {
172 if (!(begin
& (ENTRIES_PER_WORD
- 1)) &&
173 bitmap_word_used(addr
, begin
)) {
174 begin
+= ENTRIES_PER_WORD
;
178 if (!sm_lookup_bitmap(addr
, begin
)) {
189 /*----------------------------------------------------------------*/
191 static int sm_ll_init(struct ll_disk
*ll
, struct dm_transaction_manager
*tm
)
195 ll
->bitmap_info
.tm
= tm
;
196 ll
->bitmap_info
.levels
= 1;
199 * Because the new bitmap blocks are created via a shadow
200 * operation, the old entry has already had its reference count
201 * decremented and we don't need the btree to do any bookkeeping.
203 ll
->bitmap_info
.value_type
.size
= sizeof(struct disk_index_entry
);
204 ll
->bitmap_info
.value_type
.inc
= NULL
;
205 ll
->bitmap_info
.value_type
.dec
= NULL
;
206 ll
->bitmap_info
.value_type
.equal
= NULL
;
208 ll
->ref_count_info
.tm
= tm
;
209 ll
->ref_count_info
.levels
= 1;
210 ll
->ref_count_info
.value_type
.size
= sizeof(uint32_t);
211 ll
->ref_count_info
.value_type
.inc
= NULL
;
212 ll
->ref_count_info
.value_type
.dec
= NULL
;
213 ll
->ref_count_info
.value_type
.equal
= NULL
;
215 ll
->block_size
= dm_bm_block_size(dm_tm_get_bm(tm
));
217 if (ll
->block_size
> (1 << 30)) {
218 DMERR("block size too big to hold bitmaps");
222 ll
->entries_per_block
= (ll
->block_size
- sizeof(struct disk_bitmap_header
)) *
226 ll
->ref_count_root
= 0;
227 ll
->bitmap_index_changed
= false;
232 int sm_ll_extend(struct ll_disk
*ll
, dm_block_t extra_blocks
)
235 dm_block_t i
, nr_blocks
, nr_indexes
;
236 unsigned old_blocks
, blocks
;
238 nr_blocks
= ll
->nr_blocks
+ extra_blocks
;
239 old_blocks
= dm_sector_div_up(ll
->nr_blocks
, ll
->entries_per_block
);
240 blocks
= dm_sector_div_up(nr_blocks
, ll
->entries_per_block
);
242 nr_indexes
= dm_sector_div_up(nr_blocks
, ll
->entries_per_block
);
243 if (nr_indexes
> ll
->max_entries(ll
)) {
244 DMERR("space map too large");
249 * We need to set this before the dm_tm_new_block() call below.
251 ll
->nr_blocks
= nr_blocks
;
252 for (i
= old_blocks
; i
< blocks
; i
++) {
254 struct disk_index_entry idx
;
256 r
= dm_tm_new_block(ll
->tm
, &dm_sm_bitmap_validator
, &b
);
260 idx
.blocknr
= cpu_to_le64(dm_block_location(b
));
262 dm_tm_unlock(ll
->tm
, b
);
264 idx
.nr_free
= cpu_to_le32(ll
->entries_per_block
);
265 idx
.none_free_before
= 0;
267 r
= ll
->save_ie(ll
, i
, &idx
);
275 int sm_ll_lookup_bitmap(struct ll_disk
*ll
, dm_block_t b
, uint32_t *result
)
278 dm_block_t index
= b
;
279 struct disk_index_entry ie_disk
;
280 struct dm_block
*blk
;
282 b
= do_div(index
, ll
->entries_per_block
);
283 r
= ll
->load_ie(ll
, index
, &ie_disk
);
287 r
= dm_tm_read_lock(ll
->tm
, le64_to_cpu(ie_disk
.blocknr
),
288 &dm_sm_bitmap_validator
, &blk
);
292 *result
= sm_lookup_bitmap(dm_bitmap_data(blk
), b
);
294 dm_tm_unlock(ll
->tm
, blk
);
299 static int sm_ll_lookup_big_ref_count(struct ll_disk
*ll
, dm_block_t b
,
305 r
= dm_btree_lookup(&ll
->ref_count_info
, ll
->ref_count_root
, &b
, &le_rc
);
309 *result
= le32_to_cpu(le_rc
);
314 int sm_ll_lookup(struct ll_disk
*ll
, dm_block_t b
, uint32_t *result
)
316 int r
= sm_ll_lookup_bitmap(ll
, b
, result
);
324 return sm_ll_lookup_big_ref_count(ll
, b
, result
);
327 int sm_ll_find_free_block(struct ll_disk
*ll
, dm_block_t begin
,
328 dm_block_t end
, dm_block_t
*result
)
331 struct disk_index_entry ie_disk
;
332 dm_block_t i
, index_begin
= begin
;
333 dm_block_t index_end
= dm_sector_div_up(end
, ll
->entries_per_block
);
338 begin
= do_div(index_begin
, ll
->entries_per_block
);
339 end
= do_div(end
, ll
->entries_per_block
);
341 for (i
= index_begin
; i
< index_end
; i
++, begin
= 0) {
342 struct dm_block
*blk
;
346 r
= ll
->load_ie(ll
, i
, &ie_disk
);
350 if (le32_to_cpu(ie_disk
.nr_free
) == 0)
353 r
= dm_tm_read_lock(ll
->tm
, le64_to_cpu(ie_disk
.blocknr
),
354 &dm_sm_bitmap_validator
, &blk
);
358 bit_end
= (i
== index_end
- 1) ? end
: ll
->entries_per_block
;
360 r
= sm_find_free(dm_bitmap_data(blk
),
361 max_t(unsigned, begin
, le32_to_cpu(ie_disk
.none_free_before
)),
365 * This might happen because we started searching
366 * part way through the bitmap.
368 dm_tm_unlock(ll
->tm
, blk
);
372 dm_tm_unlock(ll
->tm
, blk
);
376 dm_tm_unlock(ll
->tm
, blk
);
378 *result
= i
* ll
->entries_per_block
+ (dm_block_t
) position
;
385 static int sm_ll_mutate(struct ll_disk
*ll
, dm_block_t b
,
386 int (*mutator
)(void *context
, uint32_t old
, uint32_t *new),
387 void *context
, enum allocation_event
*ev
)
390 uint32_t bit
, old
, ref_count
;
392 dm_block_t index
= b
;
393 struct disk_index_entry ie_disk
;
397 bit
= do_div(index
, ll
->entries_per_block
);
398 r
= ll
->load_ie(ll
, index
, &ie_disk
);
402 r
= dm_tm_shadow_block(ll
->tm
, le64_to_cpu(ie_disk
.blocknr
),
403 &dm_sm_bitmap_validator
, &nb
, &inc
);
405 DMERR("dm_tm_shadow_block() failed");
408 ie_disk
.blocknr
= cpu_to_le64(dm_block_location(nb
));
410 bm_le
= dm_bitmap_data(nb
);
411 old
= sm_lookup_bitmap(bm_le
, bit
);
414 r
= sm_ll_lookup_big_ref_count(ll
, b
, &old
);
416 dm_tm_unlock(ll
->tm
, nb
);
421 r
= mutator(context
, old
, &ref_count
);
423 dm_tm_unlock(ll
->tm
, nb
);
427 if (ref_count
<= 2) {
428 sm_set_bitmap(bm_le
, bit
, ref_count
);
430 dm_tm_unlock(ll
->tm
, nb
);
433 r
= dm_btree_remove(&ll
->ref_count_info
,
435 &b
, &ll
->ref_count_root
);
441 __le32 le_rc
= cpu_to_le32(ref_count
);
443 sm_set_bitmap(bm_le
, bit
, 3);
444 dm_tm_unlock(ll
->tm
, nb
);
446 __dm_bless_for_disk(&le_rc
);
447 r
= dm_btree_insert(&ll
->ref_count_info
, ll
->ref_count_root
,
448 &b
, &le_rc
, &ll
->ref_count_root
);
450 DMERR("ref count insert failed");
455 if (ref_count
&& !old
) {
458 le32_add_cpu(&ie_disk
.nr_free
, -1);
459 if (le32_to_cpu(ie_disk
.none_free_before
) == bit
)
460 ie_disk
.none_free_before
= cpu_to_le32(bit
+ 1);
462 } else if (old
&& !ref_count
) {
465 le32_add_cpu(&ie_disk
.nr_free
, 1);
466 ie_disk
.none_free_before
= cpu_to_le32(min(le32_to_cpu(ie_disk
.none_free_before
), bit
));
469 return ll
->save_ie(ll
, index
, &ie_disk
);
472 static int set_ref_count(void *context
, uint32_t old
, uint32_t *new)
474 *new = *((uint32_t *) context
);
478 int sm_ll_insert(struct ll_disk
*ll
, dm_block_t b
,
479 uint32_t ref_count
, enum allocation_event
*ev
)
481 return sm_ll_mutate(ll
, b
, set_ref_count
, &ref_count
, ev
);
484 static int inc_ref_count(void *context
, uint32_t old
, uint32_t *new)
490 int sm_ll_inc(struct ll_disk
*ll
, dm_block_t b
, enum allocation_event
*ev
)
492 return sm_ll_mutate(ll
, b
, inc_ref_count
, NULL
, ev
);
495 static int dec_ref_count(void *context
, uint32_t old
, uint32_t *new)
498 DMERR_LIMIT("unable to decrement a reference count below 0");
506 int sm_ll_dec(struct ll_disk
*ll
, dm_block_t b
, enum allocation_event
*ev
)
508 return sm_ll_mutate(ll
, b
, dec_ref_count
, NULL
, ev
);
511 int sm_ll_commit(struct ll_disk
*ll
)
515 if (ll
->bitmap_index_changed
) {
518 ll
->bitmap_index_changed
= false;
524 /*----------------------------------------------------------------*/
526 static int metadata_ll_load_ie(struct ll_disk
*ll
, dm_block_t index
,
527 struct disk_index_entry
*ie
)
529 memcpy(ie
, ll
->mi_le
.index
+ index
, sizeof(*ie
));
533 static int metadata_ll_save_ie(struct ll_disk
*ll
, dm_block_t index
,
534 struct disk_index_entry
*ie
)
536 ll
->bitmap_index_changed
= true;
537 memcpy(ll
->mi_le
.index
+ index
, ie
, sizeof(*ie
));
541 static int metadata_ll_init_index(struct ll_disk
*ll
)
546 r
= dm_tm_new_block(ll
->tm
, &index_validator
, &b
);
550 memcpy(dm_block_data(b
), &ll
->mi_le
, sizeof(ll
->mi_le
));
551 ll
->bitmap_root
= dm_block_location(b
);
553 dm_tm_unlock(ll
->tm
, b
);
558 static int metadata_ll_open(struct ll_disk
*ll
)
561 struct dm_block
*block
;
563 r
= dm_tm_read_lock(ll
->tm
, ll
->bitmap_root
,
564 &index_validator
, &block
);
568 memcpy(&ll
->mi_le
, dm_block_data(block
), sizeof(ll
->mi_le
));
569 dm_tm_unlock(ll
->tm
, block
);
574 static dm_block_t
metadata_ll_max_entries(struct ll_disk
*ll
)
576 return MAX_METADATA_BITMAPS
;
579 static int metadata_ll_commit(struct ll_disk
*ll
)
584 r
= dm_tm_shadow_block(ll
->tm
, ll
->bitmap_root
, &index_validator
, &b
, &inc
);
588 memcpy(dm_block_data(b
), &ll
->mi_le
, sizeof(ll
->mi_le
));
589 ll
->bitmap_root
= dm_block_location(b
);
591 dm_tm_unlock(ll
->tm
, b
);
596 int sm_ll_new_metadata(struct ll_disk
*ll
, struct dm_transaction_manager
*tm
)
600 r
= sm_ll_init(ll
, tm
);
604 ll
->load_ie
= metadata_ll_load_ie
;
605 ll
->save_ie
= metadata_ll_save_ie
;
606 ll
->init_index
= metadata_ll_init_index
;
607 ll
->open_index
= metadata_ll_open
;
608 ll
->max_entries
= metadata_ll_max_entries
;
609 ll
->commit
= metadata_ll_commit
;
612 ll
->nr_allocated
= 0;
614 r
= ll
->init_index(ll
);
618 r
= dm_btree_empty(&ll
->ref_count_info
, &ll
->ref_count_root
);
625 int sm_ll_open_metadata(struct ll_disk
*ll
, struct dm_transaction_manager
*tm
,
626 void *root_le
, size_t len
)
629 struct disk_sm_root
*smr
= root_le
;
631 if (len
< sizeof(struct disk_sm_root
)) {
632 DMERR("sm_metadata root too small");
636 r
= sm_ll_init(ll
, tm
);
640 ll
->load_ie
= metadata_ll_load_ie
;
641 ll
->save_ie
= metadata_ll_save_ie
;
642 ll
->init_index
= metadata_ll_init_index
;
643 ll
->open_index
= metadata_ll_open
;
644 ll
->max_entries
= metadata_ll_max_entries
;
645 ll
->commit
= metadata_ll_commit
;
647 ll
->nr_blocks
= le64_to_cpu(smr
->nr_blocks
);
648 ll
->nr_allocated
= le64_to_cpu(smr
->nr_allocated
);
649 ll
->bitmap_root
= le64_to_cpu(smr
->bitmap_root
);
650 ll
->ref_count_root
= le64_to_cpu(smr
->ref_count_root
);
652 return ll
->open_index(ll
);
655 /*----------------------------------------------------------------*/
657 static int disk_ll_load_ie(struct ll_disk
*ll
, dm_block_t index
,
658 struct disk_index_entry
*ie
)
660 return dm_btree_lookup(&ll
->bitmap_info
, ll
->bitmap_root
, &index
, ie
);
663 static int disk_ll_save_ie(struct ll_disk
*ll
, dm_block_t index
,
664 struct disk_index_entry
*ie
)
666 __dm_bless_for_disk(ie
);
667 return dm_btree_insert(&ll
->bitmap_info
, ll
->bitmap_root
,
668 &index
, ie
, &ll
->bitmap_root
);
671 static int disk_ll_init_index(struct ll_disk
*ll
)
673 return dm_btree_empty(&ll
->bitmap_info
, &ll
->bitmap_root
);
676 static int disk_ll_open(struct ll_disk
*ll
)
682 static dm_block_t
disk_ll_max_entries(struct ll_disk
*ll
)
687 static int disk_ll_commit(struct ll_disk
*ll
)
692 int sm_ll_new_disk(struct ll_disk
*ll
, struct dm_transaction_manager
*tm
)
696 r
= sm_ll_init(ll
, tm
);
700 ll
->load_ie
= disk_ll_load_ie
;
701 ll
->save_ie
= disk_ll_save_ie
;
702 ll
->init_index
= disk_ll_init_index
;
703 ll
->open_index
= disk_ll_open
;
704 ll
->max_entries
= disk_ll_max_entries
;
705 ll
->commit
= disk_ll_commit
;
708 ll
->nr_allocated
= 0;
710 r
= ll
->init_index(ll
);
714 r
= dm_btree_empty(&ll
->ref_count_info
, &ll
->ref_count_root
);
721 int sm_ll_open_disk(struct ll_disk
*ll
, struct dm_transaction_manager
*tm
,
722 void *root_le
, size_t len
)
725 struct disk_sm_root
*smr
= root_le
;
727 if (len
< sizeof(struct disk_sm_root
)) {
728 DMERR("sm_metadata root too small");
732 r
= sm_ll_init(ll
, tm
);
736 ll
->load_ie
= disk_ll_load_ie
;
737 ll
->save_ie
= disk_ll_save_ie
;
738 ll
->init_index
= disk_ll_init_index
;
739 ll
->open_index
= disk_ll_open
;
740 ll
->max_entries
= disk_ll_max_entries
;
741 ll
->commit
= disk_ll_commit
;
743 ll
->nr_blocks
= le64_to_cpu(smr
->nr_blocks
);
744 ll
->nr_allocated
= le64_to_cpu(smr
->nr_allocated
);
745 ll
->bitmap_root
= le64_to_cpu(smr
->bitmap_root
);
746 ll
->ref_count_root
= le64_to_cpu(smr
->ref_count_root
);
748 return ll
->open_index(ll
);
751 /*----------------------------------------------------------------*/