drivers/md/dm-cache-metadata.c

   1 /*
   2  * Copyright (C) 2012 Red Hat, Inc.
   3  *
   4  * This file is released under the GPL.
   5  */
   6
   7 #include "dm-cache-metadata.h"
   8
   9 #include "persistent-data/dm-array.h"
  10 #include "persistent-data/dm-bitset.h"
  11 #include "persistent-data/dm-space-map.h"
  12 #include "persistent-data/dm-space-map-disk.h"
  13 #include "persistent-data/dm-transaction-manager.h"
  14
  15 #include <linux/device-mapper.h>
  16
  17 /*----------------------------------------------------------------*/
  18
  19 #define DM_MSG_PREFIX   "cache metadata"
  20
  21 #define CACHE_SUPERBLOCK_MAGIC 06142003
  22 #define CACHE_SUPERBLOCK_LOCATION 0
  23 #define CACHE_VERSION 1
  24 #define CACHE_METADATA_CACHE_SIZE 64
  25
  26 /*
  27  *  3 for btree insert +
  28  *  2 for btree lookup used within space map
  29  */
  30 #define CACHE_MAX_CONCURRENT_LOCKS 5
  31 #define SPACE_MAP_ROOT_SIZE 128
  32
  33 enum superblock_flag_bits {
  34         /* for spotting crashes that would invalidate the dirty bitset */
  35         CLEAN_SHUTDOWN,
  36 };
  37
  38 /*
  39  * Each mapping from cache block -> origin block carries a set of flags.
  40  */
  41 enum mapping_bits {
  42         /*
  43          * A valid mapping.  Because we're using an array we clear this
  44          * flag for an non existant mapping.
  45          */
  46         M_VALID = 1,
  47
  48         /*
  49          * The data on the cache is different from that on the origin.
  50          */
  51         M_DIRTY = 2
  52 };
  53
  54 struct cache_disk_superblock {
  55         __le32 csum;
  56         __le32 flags;
  57         __le64 blocknr;
  58
  59         __u8 uuid[16];
  60         __le64 magic;
  61         __le32 version;
  62
  63         __u8 policy_name[CACHE_POLICY_NAME_SIZE];
  64         __le32 policy_hint_size;
  65
  66         __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  67         __le64 mapping_root;
  68         __le64 hint_root;
  69
  70         __le64 discard_root;
  71         __le64 discard_block_size;
  72         __le64 discard_nr_blocks;
  73
  74         __le32 data_block_size;
  75         __le32 metadata_block_size;
  76         __le32 cache_blocks;
  77
  78         __le32 compat_flags;
  79         __le32 compat_ro_flags;
  80         __le32 incompat_flags;
  81
  82         __le32 read_hits;
  83         __le32 read_misses;
  84         __le32 write_hits;
  85         __le32 write_misses;
  86
  87         __le32 policy_version[CACHE_POLICY_VERSION_SIZE];
  88 } __packed;
  89
  90 struct dm_cache_metadata {
  91         atomic_t ref_count;
  92         struct list_head list;
  93
  94         struct block_device *bdev;
  95         struct dm_block_manager *bm;
  96         struct dm_space_map *metadata_sm;
  97         struct dm_transaction_manager *tm;
  98
  99         struct dm_array_info info;
 100         struct dm_array_info hint_info;
 101         struct dm_disk_bitset discard_info;
 102
 103         struct rw_semaphore root_lock;
 104         dm_block_t root;
 105         dm_block_t hint_root;
 106         dm_block_t discard_root;
 107
 108         sector_t discard_block_size;
 109         dm_dblock_t discard_nr_blocks;
 110
 111         sector_t data_block_size;
 112         dm_cblock_t cache_blocks;
 113         bool changed:1;
 114         bool clean_when_opened:1;
 115
 116         char policy_name[CACHE_POLICY_NAME_SIZE];
 117         unsigned policy_version[CACHE_POLICY_VERSION_SIZE];
 118         size_t policy_hint_size;
 119         struct dm_cache_statistics stats;
 120 };
 121
 122 /*-------------------------------------------------------------------
 123  * superblock validator
 124  *-----------------------------------------------------------------*/
 125
 126 #define SUPERBLOCK_CSUM_XOR 9031977
 127
 128 static void sb_prepare_for_write(struct dm_block_validator *v,
 129                                  struct dm_block *b,
 130                                  size_t sb_block_size)
 131 {
 132         struct cache_disk_superblock *disk_super = dm_block_data(b);
 133
 134         disk_super->blocknr = cpu_to_le64(dm_block_location(b));
 135         disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 136                                                       sb_block_size - sizeof(__le32),
 137                                                       SUPERBLOCK_CSUM_XOR));
 138 }
 139
 140 static int sb_check(struct dm_block_validator *v,
 141                     struct dm_block *b,
 142                     size_t sb_block_size)
 143 {
 144         struct cache_disk_superblock *disk_super = dm_block_data(b);
 145         __le32 csum_le;
 146
 147         if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
 148                 DMERR("sb_check failed: blocknr %llu: wanted %llu",
 149                       le64_to_cpu(disk_super->blocknr),
 150                       (unsigned long long)dm_block_location(b));
 151                 return -ENOTBLK;
 152         }
 153
 154         if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
 155                 DMERR("sb_check failed: magic %llu: wanted %llu",
 156                       le64_to_cpu(disk_super->magic),
 157                       (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
 158                 return -EILSEQ;
 159         }
 160
 161         csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 162                                              sb_block_size - sizeof(__le32),
 163                                              SUPERBLOCK_CSUM_XOR));
 164         if (csum_le != disk_super->csum) {
 165                 DMERR("sb_check failed: csum %u: wanted %u",
 166                       le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
 167                 return -EILSEQ;
 168         }
 169
 170         return 0;
 171 }
 172
 173 static struct dm_block_validator sb_validator = {
 174         .name = "superblock",
 175         .prepare_for_write = sb_prepare_for_write,
 176         .check = sb_check
 177 };
 178
 179 /*----------------------------------------------------------------*/
 180
 181 static int superblock_read_lock(struct dm_cache_metadata *cmd,
 182                                 struct dm_block **sblock)
 183 {
 184         return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 185                                &sb_validator, sblock);
 186 }
 187
 188 static int superblock_lock_zero(struct dm_cache_metadata *cmd,
 189                                 struct dm_block **sblock)
 190 {
 191         return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 192                                      &sb_validator, sblock);
 193 }
 194
 195 static int superblock_lock(struct dm_cache_metadata *cmd,
 196                            struct dm_block **sblock)
 197 {
 198         return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 199                                 &sb_validator, sblock);
 200 }
 201
 202 /*----------------------------------------------------------------*/
 203
 204 static int __superblock_all_zeroes(struct dm_block_manager *bm, int *result)
 205 {
 206         int r;
 207         unsigned i;
 208         struct dm_block *b;
 209         __le64 *data_le, zero = cpu_to_le64(0);
 210         unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
 211
 212         /*
 213          * We can't use a validator here - it may be all zeroes.
 214          */
 215         r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
 216         if (r)
 217                 return r;
 218
 219         data_le = dm_block_data(b);
 220         *result = 1;
 221         for (i = 0; i < sb_block_size; i++) {
 222                 if (data_le[i] != zero) {
 223                         *result = 0;
 224                         break;
 225                 }
 226         }
 227
 228         return dm_bm_unlock(b);
 229 }
 230
 231 static void __setup_mapping_info(struct dm_cache_metadata *cmd)
 232 {
 233         struct dm_btree_value_type vt;
 234
 235         vt.context = NULL;
 236         vt.size = sizeof(__le64);
 237         vt.inc = NULL;
 238         vt.dec = NULL;
 239         vt.equal = NULL;
 240         dm_array_info_init(&cmd->info, cmd->tm, &vt);
 241
 242         if (cmd->policy_hint_size) {
 243                 vt.size = sizeof(__le32);
 244                 dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
 245         }
 246 }
 247
 248 static int __write_initial_superblock(struct dm_cache_metadata *cmd)
 249 {
 250         int r;
 251         struct dm_block *sblock;
 252         size_t metadata_len;
 253         struct cache_disk_superblock *disk_super;
 254         sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT;
 255
 256         /* FIXME: see if we can lose the max sectors limit */
 257         if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
 258                 bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
 259
 260         r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
 261         if (r < 0)
 262                 return r;
 263
 264         r = dm_tm_pre_commit(cmd->tm);
 265         if (r < 0)
 266                 return r;
 267
 268         r = superblock_lock_zero(cmd, &sblock);
 269         if (r)
 270                 return r;
 271
 272         disk_super = dm_block_data(sblock);
 273         disk_super->flags = 0;
 274         memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
 275         disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
 276         disk_super->version = cpu_to_le32(CACHE_VERSION);
 277         memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
 278         memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));
 279         disk_super->policy_hint_size = 0;
 280
 281         r = dm_sm_copy_root(cmd->metadata_sm, &disk_super->metadata_space_map_root,
 282                             metadata_len);
 283         if (r < 0)
 284                 goto bad_locked;
 285
 286         disk_super->mapping_root = cpu_to_le64(cmd->root);
 287         disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 288         disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 289         disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 290         disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 291         disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE >> SECTOR_SHIFT);
 292         disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
 293         disk_super->cache_blocks = cpu_to_le32(0);
 294
 295         disk_super->read_hits = cpu_to_le32(0);
 296         disk_super->read_misses = cpu_to_le32(0);
 297         disk_super->write_hits = cpu_to_le32(0);
 298         disk_super->write_misses = cpu_to_le32(0);
 299
 300         return dm_tm_commit(cmd->tm, sblock);
 301
 302 bad_locked:
 303         dm_bm_unlock(sblock);
 304         return r;
 305 }
 306
 307 static int __format_metadata(struct dm_cache_metadata *cmd)
 308 {
 309         int r;
 310
 311         r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 312                                  &cmd->tm, &cmd->metadata_sm);
 313         if (r < 0) {
 314                 DMERR("tm_create_with_sm failed");
 315                 return r;
 316         }
 317
 318         __setup_mapping_info(cmd);
 319
 320         r = dm_array_empty(&cmd->info, &cmd->root);
 321         if (r < 0)
 322                 goto bad;
 323
 324         dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 325
 326         r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
 327         if (r < 0)
 328                 goto bad;
 329
 330         cmd->discard_block_size = 0;
 331         cmd->discard_nr_blocks = 0;
 332
 333         r = __write_initial_superblock(cmd);
 334         if (r)
 335                 goto bad;
 336
 337         cmd->clean_when_opened = true;
 338         return 0;
 339
 340 bad:
 341         dm_tm_destroy(cmd->tm);
 342         dm_sm_destroy(cmd->metadata_sm);
 343
 344         return r;
 345 }
 346
 347 static int __check_incompat_features(struct cache_disk_superblock *disk_super,
 348                                      struct dm_cache_metadata *cmd)
 349 {
 350         uint32_t features;
 351
 352         features = le32_to_cpu(disk_super->incompat_flags) & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
 353         if (features) {
 354                 DMERR("could not access metadata due to unsupported optional features (%lx).",
 355                       (unsigned long)features);
 356                 return -EINVAL;
 357         }
 358
 359         /*
 360          * Check for read-only metadata to skip the following RDWR checks.
 361          */
 362         if (get_disk_ro(cmd->bdev->bd_disk))
 363                 return 0;
 364
 365         features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
 366         if (features) {
 367                 DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
 368                       (unsigned long)features);
 369                 return -EINVAL;
 370         }
 371
 372         return 0;
 373 }
 374
 375 static int __open_metadata(struct dm_cache_metadata *cmd)
 376 {
 377         int r;
 378         struct dm_block *sblock;
 379         struct cache_disk_superblock *disk_super;
 380         unsigned long sb_flags;
 381
 382         r = superblock_read_lock(cmd, &sblock);
 383         if (r < 0) {
 384                 DMERR("couldn't read lock superblock");
 385                 return r;
 386         }
 387
 388         disk_super = dm_block_data(sblock);
 389
 390         /* Verify the data block size hasn't changed */
 391         if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {
 392                 DMERR("changing the data block size (from %u to %llu) is not supported",
 393                       le32_to_cpu(disk_super->data_block_size),
 394                       (unsigned long long)cmd->data_block_size);
 395                 r = -EINVAL;
 396                 goto bad;
 397         }
 398
 399         r = __check_incompat_features(disk_super, cmd);
 400         if (r < 0)
 401                 goto bad;
 402
 403         r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 404                                disk_super->metadata_space_map_root,
 405                                sizeof(disk_super->metadata_space_map_root),
 406                                &cmd->tm, &cmd->metadata_sm);
 407         if (r < 0) {
 408                 DMERR("tm_open_with_sm failed");
 409                 goto bad;
 410         }
 411
 412         __setup_mapping_info(cmd);
 413         dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 414         sb_flags = le32_to_cpu(disk_super->flags);
 415         cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
 416         return dm_bm_unlock(sblock);
 417
 418 bad:
 419         dm_bm_unlock(sblock);
 420         return r;
 421 }
 422
 423 static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
 424                                      bool format_device)
 425 {
 426         int r, unformatted;
 427
 428         r = __superblock_all_zeroes(cmd->bm, &unformatted);
 429         if (r)
 430                 return r;
 431
 432         if (unformatted)
 433                 return format_device ? __format_metadata(cmd) : -EPERM;
 434
 435         return __open_metadata(cmd);
 436 }
 437
 438 static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
 439                                             bool may_format_device)
 440 {
 441         int r;
 442         cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE,
 443                                           CACHE_METADATA_CACHE_SIZE,
 444                                           CACHE_MAX_CONCURRENT_LOCKS);
 445         if (IS_ERR(cmd->bm)) {
 446                 DMERR("could not create block manager");
 447                 return PTR_ERR(cmd->bm);
 448         }
 449
 450         r = __open_or_format_metadata(cmd, may_format_device);
 451         if (r)
 452                 dm_block_manager_destroy(cmd->bm);
 453
 454         return r;
 455 }
 456
 457 static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd)
 458 {
 459         dm_sm_destroy(cmd->metadata_sm);
 460         dm_tm_destroy(cmd->tm);
 461         dm_block_manager_destroy(cmd->bm);
 462 }
 463
 464 typedef unsigned long (*flags_mutator)(unsigned long);
 465
 466 static void update_flags(struct cache_disk_superblock *disk_super,
 467                          flags_mutator mutator)
 468 {
 469         uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
 470         disk_super->flags = cpu_to_le32(sb_flags);
 471 }
 472
 473 static unsigned long set_clean_shutdown(unsigned long flags)
 474 {
 475         set_bit(CLEAN_SHUTDOWN, &flags);
 476         return flags;
 477 }
 478
 479 static unsigned long clear_clean_shutdown(unsigned long flags)
 480 {
 481         clear_bit(CLEAN_SHUTDOWN, &flags);
 482         return flags;
 483 }
 484
 485 static void read_superblock_fields(struct dm_cache_metadata *cmd,
 486                                    struct cache_disk_superblock *disk_super)
 487 {
 488         cmd->root = le64_to_cpu(disk_super->mapping_root);
 489         cmd->hint_root = le64_to_cpu(disk_super->hint_root);
 490         cmd->discard_root = le64_to_cpu(disk_super->discard_root);
 491         cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
 492         cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
 493         cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
 494         cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
 495         strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
 496         cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);
 497         cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);
 498         cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);
 499         cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
 500
 501         cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
 502         cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
 503         cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
 504         cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
 505
 506         cmd->changed = false;
 507 }
 508
 509 /*
 510  * The mutator updates the superblock flags.
 511  */
 512 static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
 513                                      flags_mutator mutator)
 514 {
 515         int r;
 516         struct cache_disk_superblock *disk_super;
 517         struct dm_block *sblock;
 518
 519         r = superblock_lock(cmd, &sblock);
 520         if (r)
 521                 return r;
 522
 523         disk_super = dm_block_data(sblock);
 524         update_flags(disk_super, mutator);
 525         read_superblock_fields(cmd, disk_super);
 526         dm_bm_unlock(sblock);
 527
 528         return dm_bm_flush(cmd->bm);
 529 }
 530
 531 static int __begin_transaction(struct dm_cache_metadata *cmd)
 532 {
 533         int r;
 534         struct cache_disk_superblock *disk_super;
 535         struct dm_block *sblock;
 536
 537         /*
 538          * We re-read the superblock every time.  Shouldn't need to do this
 539          * really.
 540          */
 541         r = superblock_read_lock(cmd, &sblock);
 542         if (r)
 543                 return r;
 544
 545         disk_super = dm_block_data(sblock);
 546         read_superblock_fields(cmd, disk_super);
 547         dm_bm_unlock(sblock);
 548
 549         return 0;
 550 }
 551
 552 static int __commit_transaction(struct dm_cache_metadata *cmd,
 553                                 flags_mutator mutator)
 554 {
 555         int r;
 556         size_t metadata_len;
 557         struct cache_disk_superblock *disk_super;
 558         struct dm_block *sblock;
 559
 560         /*
 561          * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
 562          */
 563         BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
 564
 565         r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
 566                             &cmd->discard_root);
 567         if (r)
 568                 return r;
 569
 570         r = dm_tm_pre_commit(cmd->tm);
 571         if (r < 0)
 572                 return r;
 573
 574         r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
 575         if (r < 0)
 576                 return r;
 577
 578         r = superblock_lock(cmd, &sblock);
 579         if (r)
 580                 return r;
 581
 582         disk_super = dm_block_data(sblock);
 583
 584         if (mutator)
 585                 update_flags(disk_super, mutator);
 586
 587         disk_super->mapping_root = cpu_to_le64(cmd->root);
 588         disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 589         disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 590         disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 591         disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 592         disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
 593         strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
 594         disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
 595         disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);
 596         disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);
 597
 598         disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
 599         disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
 600         disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
 601         disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
 602
 603         r = dm_sm_copy_root(cmd->metadata_sm, &disk_super->metadata_space_map_root,
 604                             metadata_len);
 605         if (r < 0) {
 606                 dm_bm_unlock(sblock);
 607                 return r;
 608         }
 609
 610         return dm_tm_commit(cmd->tm, sblock);
 611 }
 612
 613 /*----------------------------------------------------------------*/
 614
 615 /*
 616  * The mappings are held in a dm-array that has 64-bit values stored in
 617  * little-endian format.  The index is the cblock, the high 48bits of the
 618  * value are the oblock and the low 16 bit the flags.
 619  */
 620 #define FLAGS_MASK ((1 << 16) - 1)
 621
 622 static __le64 pack_value(dm_oblock_t block, unsigned flags)
 623 {
 624         uint64_t value = from_oblock(block);
 625         value <<= 16;
 626         value = value | (flags & FLAGS_MASK);
 627         return cpu_to_le64(value);
 628 }
 629
 630 static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags)
 631 {
 632         uint64_t value = le64_to_cpu(value_le);
 633         uint64_t b = value >> 16;
 634         *block = to_oblock(b);
 635         *flags = value & FLAGS_MASK;
 636 }
 637
 638 /*----------------------------------------------------------------*/
 639
 640 static struct dm_cache_metadata *metadata_open(struct block_device *bdev,
 641                                                sector_t data_block_size,
 642                                                bool may_format_device,
 643                                                size_t policy_hint_size)
 644 {
 645         int r;
 646         struct dm_cache_metadata *cmd;
 647
 648         cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 649         if (!cmd) {
 650                 DMERR("could not allocate metadata struct");
 651                 return ERR_PTR(-ENOMEM);
 652         }
 653
 654         atomic_set(&cmd->ref_count, 1);
 655         init_rwsem(&cmd->root_lock);
 656         cmd->bdev = bdev;
 657         cmd->data_block_size = data_block_size;
 658         cmd->cache_blocks = 0;
 659         cmd->policy_hint_size = policy_hint_size;
 660         cmd->changed = true;
 661
 662         r = __create_persistent_data_objects(cmd, may_format_device);
 663         if (r) {
 664                 kfree(cmd);
 665                 return ERR_PTR(r);
 666         }
 667
 668         r = __begin_transaction_flags(cmd, clear_clean_shutdown);
 669         if (r < 0) {
 670                 dm_cache_metadata_close(cmd);
 671                 return ERR_PTR(r);
 672         }
 673
 674         return cmd;
 675 }
 676
 677 /*
 678  * We keep a little list of ref counted metadata objects to prevent two
 679  * different target instances creating separate bufio instances.  This is
 680  * an issue if a table is reloaded before the suspend.
 681  */
 682 static DEFINE_MUTEX(table_lock);
 683 static LIST_HEAD(table);
 684
 685 static struct dm_cache_metadata *lookup(struct block_device *bdev)
 686 {
 687         struct dm_cache_metadata *cmd;
 688
 689         list_for_each_entry(cmd, &table, list)
 690                 if (cmd->bdev == bdev) {
 691                         atomic_inc(&cmd->ref_count);
 692                         return cmd;
 693                 }
 694
 695         return NULL;
 696 }
 697
 698 static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
 699                                                 sector_t data_block_size,
 700                                                 bool may_format_device,
 701                                                 size_t policy_hint_size)
 702 {
 703         struct dm_cache_metadata *cmd, *cmd2;
 704
 705         mutex_lock(&table_lock);
 706         cmd = lookup(bdev);
 707         mutex_unlock(&table_lock);
 708
 709         if (cmd)
 710                 return cmd;
 711
 712         cmd = metadata_open(bdev, data_block_size, may_format_device, policy_hint_size);
 713         if (!IS_ERR(cmd)) {
 714                 mutex_lock(&table_lock);
 715                 cmd2 = lookup(bdev);
 716                 if (cmd2) {
 717                         mutex_unlock(&table_lock);
 718                         __destroy_persistent_data_objects(cmd);
 719                         kfree(cmd);
 720                         return cmd2;
 721                 }
 722                 list_add(&cmd->list, &table);
 723                 mutex_unlock(&table_lock);
 724         }
 725
 726         return cmd;
 727 }
 728
 729 static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
 730 {
 731         if (cmd->data_block_size != data_block_size) {
 732                 DMERR("data_block_size (%llu) different from that in metadata (%llu)\n",
 733                       (unsigned long long) data_block_size,
 734                       (unsigned long long) cmd->data_block_size);
 735                 return false;
 736         }
 737
 738         return true;
 739 }
 740
 741 struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
 742                                                  sector_t data_block_size,
 743                                                  bool may_format_device,
 744                                                  size_t policy_hint_size)
 745 {
 746         struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size,
 747                                                        may_format_device, policy_hint_size);
 748
 749         if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {
 750                 dm_cache_metadata_close(cmd);
 751                 return ERR_PTR(-EINVAL);
 752         }
 753
 754         return cmd;
 755 }
 756
 757 void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
 758 {
 759         if (atomic_dec_and_test(&cmd->ref_count)) {
 760                 mutex_lock(&table_lock);
 761                 list_del(&cmd->list);
 762                 mutex_unlock(&table_lock);
 763
 764                 __destroy_persistent_data_objects(cmd);
 765                 kfree(cmd);
 766         }
 767 }
 768
 769 int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
 770 {
 771         int r;
 772         __le64 null_mapping = pack_value(0, 0);
 773
 774         down_write(&cmd->root_lock);
 775         __dm_bless_for_disk(&null_mapping);
 776         r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
 777                             from_cblock(new_cache_size),
 778                             &null_mapping, &cmd->root);
 779         if (!r)
 780                 cmd->cache_blocks = new_cache_size;
 781         cmd->changed = true;
 782         up_write(&cmd->root_lock);
 783
 784         return r;
 785 }
 786
 787 int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
 788                                    sector_t discard_block_size,
 789                                    dm_dblock_t new_nr_entries)
 790 {
 791         int r;
 792
 793         down_write(&cmd->root_lock);
 794         r = dm_bitset_resize(&cmd->discard_info,
 795                              cmd->discard_root,
 796                              from_dblock(cmd->discard_nr_blocks),
 797                              from_dblock(new_nr_entries),
 798                              false, &cmd->discard_root);
 799         if (!r) {
 800                 cmd->discard_block_size = discard_block_size;
 801                 cmd->discard_nr_blocks = new_nr_entries;
 802         }
 803
 804         cmd->changed = true;
 805         up_write(&cmd->root_lock);
 806
 807         return r;
 808 }
 809
 810 static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
 811 {
 812         return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
 813                                  from_dblock(b), &cmd->discard_root);
 814 }
 815
 816 static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
 817 {
 818         return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
 819                                    from_dblock(b), &cmd->discard_root);
 820 }
 821
 822 static int __is_discarded(struct dm_cache_metadata *cmd, dm_dblock_t b,
 823                           bool *is_discarded)
 824 {
 825         return dm_bitset_test_bit(&cmd->discard_info, cmd->discard_root,
 826                                   from_dblock(b), &cmd->discard_root,
 827                                   is_discarded);
 828 }
 829
 830 static int __discard(struct dm_cache_metadata *cmd,
 831                      dm_dblock_t dblock, bool discard)
 832 {
 833         int r;
 834
 835         r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
 836         if (r)
 837                 return r;
 838
 839         cmd->changed = true;
 840         return 0;
 841 }
 842
 843 int dm_cache_set_discard(struct dm_cache_metadata *cmd,
 844                          dm_dblock_t dblock, bool discard)
 845 {
 846         int r;
 847
 848         down_write(&cmd->root_lock);
 849         r = __discard(cmd, dblock, discard);
 850         up_write(&cmd->root_lock);
 851
 852         return r;
 853 }
 854
 855 static int __load_discards(struct dm_cache_metadata *cmd,
 856                            load_discard_fn fn, void *context)
 857 {
 858         int r = 0;
 859         dm_block_t b;
 860         bool discard;
 861
 862         for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
 863                 dm_dblock_t dblock = to_dblock(b);
 864
 865                 if (cmd->clean_when_opened) {
 866                         r = __is_discarded(cmd, dblock, &discard);
 867                         if (r)
 868                                 return r;
 869                 } else
 870                         discard = false;
 871
 872                 r = fn(context, cmd->discard_block_size, dblock, discard);
 873                 if (r)
 874                         break;
 875         }
 876
 877         return r;
 878 }
 879
 880 int dm_cache_load_discards(struct dm_cache_metadata *cmd,
 881                            load_discard_fn fn, void *context)
 882 {
 883         int r;
 884
 885         down_read(&cmd->root_lock);
 886         r = __load_discards(cmd, fn, context);
 887         up_read(&cmd->root_lock);
 888
 889         return r;
 890 }
 891
 892 dm_cblock_t dm_cache_size(struct dm_cache_metadata *cmd)
 893 {
 894         dm_cblock_t r;
 895
 896         down_read(&cmd->root_lock);
 897         r = cmd->cache_blocks;
 898         up_read(&cmd->root_lock);
 899
 900         return r;
 901 }
 902
 903 static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
 904 {
 905         int r;
 906         __le64 value = pack_value(0, 0);
 907
 908         __dm_bless_for_disk(&value);
 909         r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
 910                                &value, &cmd->root);
 911         if (r)
 912                 return r;
 913
 914         cmd->changed = true;
 915         return 0;
 916 }
 917
 918 int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
 919 {
 920         int r;
 921
 922         down_write(&cmd->root_lock);
 923         r = __remove(cmd, cblock);
 924         up_write(&cmd->root_lock);
 925
 926         return r;
 927 }
 928
 929 static int __insert(struct dm_cache_metadata *cmd,
 930                     dm_cblock_t cblock, dm_oblock_t oblock)
 931 {
 932         int r;
 933         __le64 value = pack_value(oblock, M_VALID);
 934         __dm_bless_for_disk(&value);
 935
 936         r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
 937                                &value, &cmd->root);
 938         if (r)
 939                 return r;
 940
 941         cmd->changed = true;
 942         return 0;
 943 }
 944
 945 int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
 946                             dm_cblock_t cblock, dm_oblock_t oblock)
 947 {
 948         int r;
 949
 950         down_write(&cmd->root_lock);
 951         r = __insert(cmd, cblock, oblock);
 952         up_write(&cmd->root_lock);
 953
 954         return r;
 955 }
 956
 957 struct thunk {
 958         load_mapping_fn fn;
 959         void *context;
 960
 961         struct dm_cache_metadata *cmd;
 962         bool respect_dirty_flags;
 963         bool hints_valid;
 964 };
 965
 966 static bool policy_unchanged(struct dm_cache_metadata *cmd,
 967                              struct dm_cache_policy *policy)
 968 {
 969         const char *policy_name = dm_cache_policy_get_name(policy);
 970         const unsigned *policy_version = dm_cache_policy_get_version(policy);
 971         size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);
 972
 973         /*
 974          * Ensure policy names match.
 975          */
 976         if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))
 977                 return false;
 978
 979         /*
 980          * Ensure policy major versions match.
 981          */
 982         if (cmd->policy_version[0] != policy_version[0])
 983                 return false;
 984
 985         /*
 986          * Ensure policy hint sizes match.
 987          */
 988         if (cmd->policy_hint_size != policy_hint_size)
 989                 return false;
 990
 991         return true;
 992 }
 993
 994 static bool hints_array_initialized(struct dm_cache_metadata *cmd)
 995 {
 996         return cmd->hint_root && cmd->policy_hint_size;
 997 }
 998
 999 static bool hints_array_available(struct dm_cache_metadata *cmd,
1000                                   struct dm_cache_policy *policy)
1001 {
1002         return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&
1003                 hints_array_initialized(cmd);
1004 }
1005
1006 static int __load_mapping(void *context, uint64_t cblock, void *leaf)
1007 {
1008         int r = 0;
1009         bool dirty;
1010         __le64 value;
1011         __le32 hint_value = 0;
1012         dm_oblock_t oblock;
1013         unsigned flags;
1014         struct thunk *thunk = context;
1015         struct dm_cache_metadata *cmd = thunk->cmd;
1016
1017         memcpy(&value, leaf, sizeof(value));
1018         unpack_value(value, &oblock, &flags);
1019
1020         if (flags & M_VALID) {
1021                 if (thunk->hints_valid) {
1022                         r = dm_array_get_value(&cmd->hint_info, cmd->hint_root,
1023                                                cblock, &hint_value);
1024                         if (r && r != -ENODATA)
1025                                 return r;
1026                 }
1027
1028                 dirty = thunk->respect_dirty_flags ? (flags & M_DIRTY) : true;
1029                 r = thunk->fn(thunk->context, oblock, to_cblock(cblock),
1030                               dirty, le32_to_cpu(hint_value), thunk->hints_valid);
1031         }
1032
1033         return r;
1034 }
1035
1036 static int __load_mappings(struct dm_cache_metadata *cmd,
1037                            struct dm_cache_policy *policy,
1038                            load_mapping_fn fn, void *context)
1039 {
1040         struct thunk thunk;
1041
1042         thunk.fn = fn;
1043         thunk.context = context;
1044
1045         thunk.cmd = cmd;
1046         thunk.respect_dirty_flags = cmd->clean_when_opened;
1047         thunk.hints_valid = hints_array_available(cmd, policy);
1048
1049         return dm_array_walk(&cmd->info, cmd->root, __load_mapping, &thunk);
1050 }
1051
1052 int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
1053                            struct dm_cache_policy *policy,
1054                            load_mapping_fn fn, void *context)
1055 {
1056         int r;
1057
1058         down_read(&cmd->root_lock);
1059         r = __load_mappings(cmd, policy, fn, context);
1060         up_read(&cmd->root_lock);
1061
1062         return r;
1063 }
1064
1065 static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
1066 {
1067         int r = 0;
1068         __le64 value;
1069         dm_oblock_t oblock;
1070         unsigned flags;
1071
1072         memcpy(&value, leaf, sizeof(value));
1073         unpack_value(value, &oblock, &flags);
1074
1075         return r;
1076 }
1077
1078 static int __dump_mappings(struct dm_cache_metadata *cmd)
1079 {
1080         return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
1081 }
1082
1083 void dm_cache_dump(struct dm_cache_metadata *cmd)
1084 {
1085         down_read(&cmd->root_lock);
1086         __dump_mappings(cmd);
1087         up_read(&cmd->root_lock);
1088 }
1089
1090 int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
1091 {
1092         int r;
1093
1094         down_read(&cmd->root_lock);
1095         r = cmd->changed;
1096         up_read(&cmd->root_lock);
1097
1098         return r;
1099 }
1100
1101 static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
1102 {
1103         int r;
1104         unsigned flags;
1105         dm_oblock_t oblock;
1106         __le64 value;
1107
1108         r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
1109         if (r)
1110                 return r;
1111
1112         unpack_value(value, &oblock, &flags);
1113
1114         if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
1115                 /* nothing to be done */
1116                 return 0;
1117
1118         value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
1119         __dm_bless_for_disk(&value);
1120
1121         r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1122                                &value, &cmd->root);
1123         if (r)
1124                 return r;
1125
1126         cmd->changed = true;
1127         return 0;
1128
1129 }
1130
1131 int dm_cache_set_dirty(struct dm_cache_metadata *cmd,
1132                        dm_cblock_t cblock, bool dirty)
1133 {
1134         int r;
1135
1136         down_write(&cmd->root_lock);
1137         r = __dirty(cmd, cblock, dirty);
1138         up_write(&cmd->root_lock);
1139
1140         return r;
1141 }
1142
1143 void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
1144                                  struct dm_cache_statistics *stats)
1145 {
1146         down_read(&cmd->root_lock);
1147         *stats = cmd->stats;
1148         up_read(&cmd->root_lock);
1149 }
1150
1151 void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
1152                                  struct dm_cache_statistics *stats)
1153 {
1154         down_write(&cmd->root_lock);
1155         cmd->stats = *stats;
1156         up_write(&cmd->root_lock);
1157 }
1158
1159 int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
1160 {
1161         int r;
1162         flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
1163                                  clear_clean_shutdown);
1164
1165         down_write(&cmd->root_lock);
1166         r = __commit_transaction(cmd, mutator);
1167         if (r)
1168                 goto out;
1169
1170         r = __begin_transaction(cmd);
1171
1172 out:
1173         up_write(&cmd->root_lock);
1174         return r;
1175 }
1176
1177 int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
1178                                            dm_block_t *result)
1179 {
1180         int r = -EINVAL;
1181
1182         down_read(&cmd->root_lock);
1183         r = dm_sm_get_nr_free(cmd->metadata_sm, result);
1184         up_read(&cmd->root_lock);
1185
1186         return r;
1187 }
1188
1189 int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
1190                                    dm_block_t *result)
1191 {
1192         int r = -EINVAL;
1193
1194         down_read(&cmd->root_lock);
1195         r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
1196         up_read(&cmd->root_lock);
1197
1198         return r;
1199 }
1200
1201 /*----------------------------------------------------------------*/
1202
1203 static int begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1204 {
1205         int r;
1206         __le32 value;
1207         size_t hint_size;
1208         const char *policy_name = dm_cache_policy_get_name(policy);
1209         const unsigned *policy_version = dm_cache_policy_get_version(policy);
1210
1211         if (!policy_name[0] ||
1212             (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
1213                 return -EINVAL;
1214
1215         if (!policy_unchanged(cmd, policy)) {
1216                 strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
1217                 memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
1218
1219                 hint_size = dm_cache_policy_get_hint_size(policy);
1220                 if (!hint_size)
1221                         return 0; /* short-circuit hints initialization */
1222                 cmd->policy_hint_size = hint_size;
1223
1224                 if (cmd->hint_root) {
1225                         r = dm_array_del(&cmd->hint_info, cmd->hint_root);
1226                         if (r)
1227                                 return r;
1228                 }
1229
1230                 r = dm_array_empty(&cmd->hint_info, &cmd->hint_root);
1231                 if (r)
1232                         return r;
1233
1234                 value = cpu_to_le32(0);
1235                 __dm_bless_for_disk(&value);
1236                 r = dm_array_resize(&cmd->hint_info, cmd->hint_root, 0,
1237                                     from_cblock(cmd->cache_blocks),
1238                                     &value, &cmd->hint_root);
1239                 if (r)
1240                         return r;
1241         }
1242
1243         return 0;
1244 }
1245
1246 int dm_cache_begin_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1247 {
1248         int r;
1249
1250         down_write(&cmd->root_lock);
1251         r = begin_hints(cmd, policy);
1252         up_write(&cmd->root_lock);
1253
1254         return r;
1255 }
1256
1257 static int save_hint(struct dm_cache_metadata *cmd, dm_cblock_t cblock,
1258                      uint32_t hint)
1259 {
1260         int r;
1261         __le32 value = cpu_to_le32(hint);
1262         __dm_bless_for_disk(&value);
1263
1264         r = dm_array_set_value(&cmd->hint_info, cmd->hint_root,
1265                                from_cblock(cblock), &value, &cmd->hint_root);
1266         cmd->changed = true;
1267
1268         return r;
1269 }
1270
1271 int dm_cache_save_hint(struct dm_cache_metadata *cmd, dm_cblock_t cblock,
1272                        uint32_t hint)
1273 {
1274         int r;
1275
1276         if (!hints_array_initialized(cmd))
1277                 return 0;
1278
1279         down_write(&cmd->root_lock);
1280         r = save_hint(cmd, cblock, hint);
1281         up_write(&cmd->root_lock);
1282
1283         return r;
1284 }