include/sys/dbuf.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  24  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  25  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  26  */
  27
  28 #ifndef _SYS_DBUF_H
  29 #define _SYS_DBUF_H
  30
  31 #include <sys/dmu.h>
  32 #include <sys/spa.h>
  33 #include <sys/txg.h>
  34 #include <sys/zio.h>
  35 #include <sys/arc.h>
  36 #include <sys/zfs_context.h>
  37 #include <sys/zfs_refcount.h>
  38 #include <sys/zrlock.h>
  39 #include <sys/multilist.h>
  40
  41 #ifdef  __cplusplus
  42 extern "C" {
  43 #endif
  44
  45 #define IN_DMU_SYNC 2
  46
  47 /*
  48  * define flags for dbuf_read
  49  */
  50
  51 #define DB_RF_MUST_SUCCEED      (1 << 0)
  52 #define DB_RF_CANFAIL           (1 << 1)
  53 #define DB_RF_HAVESTRUCT        (1 << 2)
  54 #define DB_RF_NOPREFETCH        (1 << 3)
  55 #define DB_RF_NEVERWAIT         (1 << 4)
  56 #define DB_RF_CACHED            (1 << 5)
  57 #define DB_RF_NO_DECRYPT        (1 << 6)
  58 #define DB_RF_PARTIAL_FIRST     (1 << 7)
  59 #define DB_RF_PARTIAL_MORE      (1 << 8)
  60
  61 /*
  62  * The simplified state transition diagram for dbufs looks like:
  63  *
  64  *                  +-------> READ ------+
  65  *                  |                    |
  66  *                  |                    V
  67  *  (alloc)-->UNCACHED                  CACHED-->EVICTING-->(free)
  68  *             ^    |                    ^          ^
  69  *             |    |                    |          |
  70  *             |    +-------> FILL ------+          |
  71  *             |    |                    |          |
  72  *             |    |                    |          |
  73  *             |    +------> NOFILL -----+-----> UNCACHED
  74  *             |               |               (Direct I/O)
  75  *             +---------------+
  76  *
  77  * DB_SEARCH is an invalid state for a dbuf. It is used by dbuf_free_range
  78  * to find all dbufs in a range of a dnode and must be less than any other
  79  * dbuf_states_t (see comment on dn_dbufs in dnode.h).
  80  */
  81 typedef enum dbuf_states {
  82         DB_MARKER = -2,
  83         DB_SEARCH = -1,
  84         DB_UNCACHED,
  85         DB_FILL,
  86         DB_NOFILL,
  87         DB_READ,
  88         DB_CACHED,
  89         DB_EVICTING
  90 } dbuf_states_t;
  91
  92 typedef enum dbuf_cached_state {
  93         DB_NO_CACHE = -1,
  94         DB_DBUF_CACHE,
  95         DB_DBUF_METADATA_CACHE,
  96         DB_CACHE_MAX
  97 } dbuf_cached_state_t;
  98
  99 struct dnode;
 100 struct dmu_tx;
 101
 102 /*
 103  * level = 0 means the user data
 104  * level = 1 means the single indirect block
 105  * etc.
 106  */
 107
 108 struct dmu_buf_impl;
 109
 110 typedef enum override_states {
 111         DR_NOT_OVERRIDDEN,
 112         DR_IN_DMU_SYNC,
 113         DR_OVERRIDDEN
 114 } override_states_t;
 115
 116 typedef enum db_lock_type {
 117         DLT_NONE,
 118         DLT_PARENT,
 119         DLT_OBJSET
 120 } db_lock_type_t;
 121
 122 typedef struct dbuf_dirty_record {
 123         /* link on our parents dirty list */
 124         list_node_t dr_dirty_node;
 125
 126         /* transaction group this data will sync in */
 127         uint64_t dr_txg;
 128
 129         /* zio of outstanding write IO */
 130         zio_t *dr_zio;
 131
 132         /* pointer back to our dbuf */
 133         struct dmu_buf_impl *dr_dbuf;
 134
 135         /* list link for dbuf dirty records */
 136         list_node_t dr_dbuf_node;
 137
 138         /*
 139          * The dnode we are part of.  Note that the dnode can not be moved or
 140          * evicted due to the hold that's added by dnode_setdirty() or
 141          * dmu_objset_sync_dnodes(), and released by dnode_rele_task() or
 142          * userquota_updates_task().  This hold is necessary for
 143          * dirty_lightweight_leaf-type dirty records, which don't have a hold
 144          * on a dbuf.
 145          */
 146         dnode_t *dr_dnode;
 147
 148         /* pointer to parent dirty record */
 149         struct dbuf_dirty_record *dr_parent;
 150
 151         /* How much space was changed to dsl_pool_dirty_space() for this? */
 152         unsigned int dr_accounted;
 153
 154         /* A copy of the bp that points to us */
 155         blkptr_t dr_bp_copy;
 156
 157         union dirty_types {
 158                 struct dirty_indirect {
 159
 160                         /* protect access to list */
 161                         kmutex_t dr_mtx;
 162
 163                         /* Our list of dirty children */
 164                         list_t dr_children;
 165                 } di;
 166                 struct dirty_leaf {
 167
 168                         /*
 169                          * dr_data is set when we dirty the buffer
 170                          * so that we can retain the pointer even if it
 171                          * gets COW'd in a subsequent transaction group.
 172                          */
 173                         arc_buf_t *dr_data;
 174                         override_states_t dr_override_state;
 175                         uint8_t dr_copies;
 176                         boolean_t dr_nopwrite;
 177                         boolean_t dr_brtwrite;
 178                         boolean_t dr_diowrite;
 179                         boolean_t dr_has_raw_params;
 180
 181                         /* Override and raw params are mutually exclusive. */
 182                         union {
 183                                 blkptr_t dr_overridden_by;
 184                                 struct {
 185                                         /*
 186                                          * If dr_has_raw_params is set, the
 187                                          * following crypt params will be set
 188                                          * on the BP that's written.
 189                                          */
 190                                         boolean_t dr_byteorder;
 191                                         uint8_t dr_salt[ZIO_DATA_SALT_LEN];
 192                                         uint8_t dr_iv[ZIO_DATA_IV_LEN];
 193                                         uint8_t dr_mac[ZIO_DATA_MAC_LEN];
 194                                 };
 195                         };
 196                 } dl;
 197                 struct dirty_lightweight_leaf {
 198                         /*
 199                          * This dirty record refers to a leaf (level=0)
 200                          * block, whose dbuf has not been instantiated for
 201                          * performance reasons.
 202                          */
 203                         uint64_t dr_blkid;
 204                         abd_t *dr_abd;
 205                         zio_prop_t dr_props;
 206                         zio_flag_t dr_flags;
 207                 } dll;
 208         } dt;
 209 } dbuf_dirty_record_t;
 210
 211 typedef struct dmu_buf_impl {
 212         /*
 213          * The following members are immutable, with the exception of
 214          * db.db_data, which is protected by db_mtx.
 215          */
 216
 217         /* the publicly visible structure */
 218         dmu_buf_t db;
 219
 220         /* the objset we belong to */
 221         struct objset *db_objset;
 222
 223         /*
 224          * Handle to safely access the dnode we belong to (NULL when evicted)
 225          * if dnode_move() is used on the platform, or just dnode otherwise.
 226          */
 227 #if !defined(__linux__) && !defined(__FreeBSD__)
 228 #define USE_DNODE_HANDLE        1
 229         struct dnode_handle *db_dnode_handle;
 230 #else
 231         struct dnode *db_dnode;
 232 #endif
 233
 234         /*
 235          * our parent buffer; if the dnode points to us directly,
 236          * db_parent == db_dnode_handle->dnh_dnode->dn_dbuf
 237          * only accessed by sync thread ???
 238          * (NULL when evicted)
 239          * May change from NULL to non-NULL under the protection of db_mtx
 240          * (see dbuf_check_blkptr())
 241          */
 242         struct dmu_buf_impl *db_parent;
 243
 244         /*
 245          * link for hash table of all dmu_buf_impl_t's
 246          */
 247         struct dmu_buf_impl *db_hash_next;
 248
 249         /*
 250          * Our link on the owner dnodes's dn_dbufs list.
 251          * Protected by its dn_dbufs_mtx.  Should be on the same cache line
 252          * as db_level and db_blkid for the best avl_add() performance.
 253          */
 254         avl_node_t db_link;
 255
 256         /* our block number */
 257         uint64_t db_blkid;
 258
 259         /*
 260          * Pointer to the blkptr_t which points to us. May be NULL if we
 261          * don't have one yet. (NULL when evicted)
 262          */
 263         blkptr_t *db_blkptr;
 264
 265         /*
 266          * Our indirection level.  Data buffers have db_level==0.
 267          * Indirect buffers which point to data buffers have
 268          * db_level==1. etc.  Buffers which contain dnodes have
 269          * db_level==0, since the dnodes are stored in a file.
 270          */
 271         uint8_t db_level;
 272
 273         /* This block was freed while a read or write was active. */
 274         uint8_t db_freed_in_flight;
 275
 276         /*
 277          * Evict user data as soon as the dirty and reference counts are equal.
 278          */
 279         uint8_t db_user_immediate_evict;
 280
 281         /*
 282          * dnode_evict_dbufs() or dnode_evict_bonus() tried to evict this dbuf,
 283          * but couldn't due to outstanding references.  Evict once the refcount
 284          * drops to 0.
 285          */
 286         uint8_t db_pending_evict;
 287
 288         /* Number of TXGs in which this buffer is dirty. */
 289         uint8_t db_dirtycnt;
 290
 291         /* The buffer was partially read.  More reads may follow. */
 292         uint8_t db_partial_read;
 293
 294         /*
 295          * Protects db_buf's contents if they contain an indirect block or data
 296          * block of the meta-dnode. We use this lock to protect the structure of
 297          * the block tree. This means that when modifying this dbuf's data, we
 298          * grab its rwlock. When modifying its parent's data (including the
 299          * blkptr to this dbuf), we grab the parent's rwlock. The lock ordering
 300          * for this lock is:
 301          * 1) dn_struct_rwlock
 302          * 2) db_rwlock
 303          * We don't currently grab multiple dbufs' db_rwlocks at once.
 304          */
 305         krwlock_t db_rwlock;
 306
 307         /* buffer holding our data */
 308         arc_buf_t *db_buf;
 309
 310         /* db_mtx protects the members below */
 311         kmutex_t db_mtx;
 312
 313         /*
 314          * Current state of the buffer
 315          */
 316         dbuf_states_t db_state;
 317
 318         /* In which dbuf cache this dbuf is, if any. */
 319         dbuf_cached_state_t db_caching_status;
 320
 321         /*
 322          * Refcount accessed by dmu_buf_{hold,rele}.
 323          * If nonzero, the buffer can't be destroyed.
 324          * Protected by db_mtx.
 325          */
 326         zfs_refcount_t db_holds;
 327
 328         kcondvar_t db_changed;
 329         dbuf_dirty_record_t *db_data_pending;
 330
 331         /* List of dirty records for the buffer sorted newest to oldest. */
 332         list_t db_dirty_records;
 333
 334         /* Link in dbuf_cache or dbuf_metadata_cache */
 335         multilist_node_t db_cache_link;
 336
 337         uint64_t db_hash;
 338
 339         /* User callback information. */
 340         dmu_buf_user_t *db_user;
 341 } dmu_buf_impl_t;
 342
 343 #define DBUF_HASH_MUTEX(h, idx) \
 344         (&(h)->hash_mutexes[(idx) & ((h)->hash_mutex_mask)])
 345
 346 typedef struct dbuf_hash_table {
 347         uint64_t hash_table_mask;
 348         uint64_t hash_mutex_mask;
 349         dmu_buf_impl_t **hash_table;
 350         kmutex_t *hash_mutexes;
 351 } dbuf_hash_table_t;
 352
 353 typedef void (*dbuf_prefetch_fn)(void *, uint64_t, uint64_t, boolean_t);
 354
 355 extern kmem_cache_t *dbuf_dirty_kmem_cache;
 356
 357 uint64_t dbuf_whichblock(const struct dnode *di, const int64_t level,
 358     const uint64_t offset);
 359
 360 void dbuf_create_bonus(struct dnode *dn);
 361 int dbuf_spill_set_blksz(dmu_buf_t *db, uint64_t blksz, dmu_tx_t *tx);
 362
 363 void dbuf_rm_spill(struct dnode *dn, dmu_tx_t *tx);
 364
 365 dmu_buf_impl_t *dbuf_hold(struct dnode *dn, uint64_t blkid, const void *tag);
 366 dmu_buf_impl_t *dbuf_hold_level(struct dnode *dn, int level, uint64_t blkid,
 367     const void *tag);
 368 int dbuf_hold_impl(struct dnode *dn, uint8_t level, uint64_t blkid,
 369     boolean_t fail_sparse, boolean_t fail_uncached,
 370     const void *tag, dmu_buf_impl_t **dbp);
 371
 372 int dbuf_prefetch_impl(struct dnode *dn, int64_t level, uint64_t blkid,
 373     zio_priority_t prio, arc_flags_t aflags, dbuf_prefetch_fn cb,
 374     void *arg);
 375 int dbuf_prefetch(struct dnode *dn, int64_t level, uint64_t blkid,
 376     zio_priority_t prio, arc_flags_t aflags);
 377
 378 void dbuf_add_ref(dmu_buf_impl_t *db, const void *tag);
 379 boolean_t dbuf_try_add_ref(dmu_buf_t *db, objset_t *os, uint64_t obj,
 380     uint64_t blkid, const void *tag);
 381 uint64_t dbuf_refcount(dmu_buf_impl_t *db);
 382
 383 void dbuf_rele(dmu_buf_impl_t *db, const void *tag);
 384 void dbuf_rele_and_unlock(dmu_buf_impl_t *db, const void *tag,
 385     boolean_t evicting);
 386
 387 dmu_buf_impl_t *dbuf_find(struct objset *os, uint64_t object, uint8_t level,
 388     uint64_t blkid, uint64_t *hash_out);
 389
 390 int dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags);
 391 void dmu_buf_will_clone_or_dio(dmu_buf_t *db, dmu_tx_t *tx);
 392 void dmu_buf_will_not_fill(dmu_buf_t *db, dmu_tx_t *tx);
 393 void dmu_buf_will_fill(dmu_buf_t *db, dmu_tx_t *tx, boolean_t canfail);
 394 boolean_t dmu_buf_fill_done(dmu_buf_t *db, dmu_tx_t *tx, boolean_t failed);
 395 void dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx);
 396 dbuf_dirty_record_t *dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
 397 dbuf_dirty_record_t *dbuf_dirty_lightweight(dnode_t *dn, uint64_t blkid,
 398     dmu_tx_t *tx);
 399 boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
 400 int dmu_buf_get_bp_from_dbuf(dmu_buf_impl_t *db, blkptr_t **bp);
 401 int dmu_buf_untransform_direct(dmu_buf_impl_t *db, spa_t *spa);
 402 arc_buf_t *dbuf_loan_arcbuf(dmu_buf_impl_t *db);
 403 void dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data,
 404     bp_embedded_type_t etype, enum zio_compress comp,
 405     int uncompressed_size, int compressed_size, int byteorder, dmu_tx_t *tx);
 406
 407 int dmu_lightweight_write_by_dnode(dnode_t *dn, uint64_t offset, abd_t *abd,
 408     const struct zio_prop *zp, zio_flag_t flags, dmu_tx_t *tx);
 409
 410 void dmu_buf_redact(dmu_buf_t *dbuf, dmu_tx_t *tx);
 411 void dbuf_destroy(dmu_buf_impl_t *db);
 412
 413 void dbuf_unoverride(dbuf_dirty_record_t *dr);
 414 void dbuf_sync_list(list_t *list, int level, dmu_tx_t *tx);
 415 void dbuf_release_bp(dmu_buf_impl_t *db);
 416 db_lock_type_t dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw,
 417     const void *tag);
 418 void dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type,
 419     const void *tag);
 420
 421 void dbuf_free_range(struct dnode *dn, uint64_t start, uint64_t end,
 422     struct dmu_tx *);
 423
 424 void dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx);
 425
 426 void dbuf_stats_init(dbuf_hash_table_t *hash);
 427 void dbuf_stats_destroy(void);
 428
 429 int dbuf_dnode_findbp(dnode_t *dn, uint64_t level, uint64_t blkid,
 430     blkptr_t *bp, uint16_t *datablkszsec, uint8_t *indblkshift);
 431
 432 #ifdef USE_DNODE_HANDLE
 433 #define DB_DNODE(_db)           ((_db)->db_dnode_handle->dnh_dnode)
 434 #define DB_DNODE_LOCK(_db)      ((_db)->db_dnode_handle->dnh_zrlock)
 435 #define DB_DNODE_ENTER(_db)     (zrl_add(&DB_DNODE_LOCK(_db)))
 436 #define DB_DNODE_EXIT(_db)      (zrl_remove(&DB_DNODE_LOCK(_db)))
 437 #define DB_DNODE_HELD(_db)      (!zrl_is_zero(&DB_DNODE_LOCK(_db)))
 438 #else
 439 #define DB_DNODE(_db)           ((_db)->db_dnode)
 440 #define DB_DNODE_LOCK(_db)
 441 #define DB_DNODE_ENTER(_db)
 442 #define DB_DNODE_EXIT(_db)
 443 #define DB_DNODE_HELD(_db)      (B_TRUE)
 444 #endif
 445
 446 void dbuf_init(void);
 447 void dbuf_fini(void);
 448
 449 boolean_t dbuf_is_metadata(dmu_buf_impl_t *db);
 450
 451 static inline dbuf_dirty_record_t *
 452 dbuf_find_dirty_lte(dmu_buf_impl_t *db, uint64_t txg)
 453 {
 454         dbuf_dirty_record_t *dr;
 455
 456         for (dr = list_head(&db->db_dirty_records);
 457             dr != NULL && dr->dr_txg > txg;
 458             dr = list_next(&db->db_dirty_records, dr))
 459                 continue;
 460         return (dr);
 461 }
 462
 463 static inline dbuf_dirty_record_t *
 464 dbuf_find_dirty_eq(dmu_buf_impl_t *db, uint64_t txg)
 465 {
 466         dbuf_dirty_record_t *dr;
 467
 468         dr = dbuf_find_dirty_lte(db, txg);
 469         if (dr && dr->dr_txg == txg)
 470                 return (dr);
 471         return (NULL);
 472 }
 473
 474 #define DBUF_GET_BUFC_TYPE(_db) \
 475         (dbuf_is_metadata(_db) ? ARC_BUFC_METADATA : ARC_BUFC_DATA)
 476
 477 #define DBUF_IS_CACHEABLE(_db)                                          \
 478         ((_db)->db_objset->os_primary_cache == ZFS_CACHE_ALL ||         \
 479         (dbuf_is_metadata(_db) &&                                       \
 480         ((_db)->db_objset->os_primary_cache == ZFS_CACHE_METADATA)))
 481
 482 boolean_t dbuf_is_l2cacheable(dmu_buf_impl_t *db, blkptr_t *db_bp);
 483
 484 #ifdef ZFS_DEBUG
 485
 486 /*
 487  * There should be a ## between the string literal and fmt, to make it
 488  * clear that we're joining two strings together, but gcc does not
 489  * support that preprocessor token.
 490  */
 491 #define dprintf_dbuf(dbuf, fmt, ...) do { \
 492         if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
 493         char __db_buf[32]; \
 494         uint64_t __db_obj = (dbuf)->db.db_object; \
 495         if (__db_obj == DMU_META_DNODE_OBJECT) \
 496                 (void) strlcpy(__db_buf, "mdn", sizeof (__db_buf));     \
 497         else \
 498                 (void) snprintf(__db_buf, sizeof (__db_buf), "%lld", \
 499                     (u_longlong_t)__db_obj); \
 500         dprintf_ds((dbuf)->db_objset->os_dsl_dataset, \
 501             "obj=%s lvl=%u blkid=%lld " fmt, \
 502             __db_buf, (dbuf)->db_level, \
 503             (u_longlong_t)(dbuf)->db_blkid, __VA_ARGS__); \
 504         } \
 505 } while (0)
 506
 507 #define dprintf_dbuf_bp(db, bp, fmt, ...) do {                  \
 508         if (zfs_flags & ZFS_DEBUG_DPRINTF) {                    \
 509         char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP);  \
 510         snprintf_blkptr(__blkbuf, BP_SPRINTF_LEN, bp);          \
 511         dprintf_dbuf(db, fmt " %s\n", __VA_ARGS__, __blkbuf);   \
 512         kmem_free(__blkbuf, BP_SPRINTF_LEN);                    \
 513         }                                                       \
 514 } while (0)
 515
 516 #define DBUF_VERIFY(db) dbuf_verify(db)
 517
 518 #else
 519
 520 #define dprintf_dbuf(db, fmt, ...)
 521 #define dprintf_dbuf_bp(db, bp, fmt, ...)
 522 #define DBUF_VERIFY(db)
 523
 524 #endif
 525
 526
 527 #ifdef  __cplusplus
 528 }
 529 #endif
 530
 531 #endif /* _SYS_DBUF_H */