include/sys/zio.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 /*
  23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  25  * Copyright (c) 2012, 2024 by Delphix. All rights reserved.
  26  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  27  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  28  * Copyright 2016 Toomas Soome <tsoome@me.com>
  29  * Copyright (c) 2019, Allan Jude
  30  * Copyright (c) 2019, 2023, 2024, Klara Inc.
  31  * Copyright (c) 2019-2020, Michael Niewöhner
  32  * Copyright (c) 2024 by George Melikov. All rights reserved.
  33  */
  34
  35 #ifndef _ZIO_H
  36 #define _ZIO_H
  37
  38 #include <sys/zio_priority.h>
  39 #include <sys/zfs_context.h>
  40 #include <sys/spa.h>
  41 #include <sys/txg.h>
  42 #include <sys/avl.h>
  43 #include <sys/fs/zfs.h>
  44 #include <sys/zio_impl.h>
  45
  46 #ifdef  __cplusplus
  47 extern "C" {
  48 #endif
  49
  50 /*
  51  * Embedded checksum
  52  */
  53 #define ZEC_MAGIC       0x210da7ab10c7a11ULL
  54
  55 typedef struct zio_eck {
  56         uint64_t        zec_magic;      /* for validation, endianness   */
  57         zio_cksum_t     zec_cksum;      /* 256-bit checksum             */
  58 } zio_eck_t;
  59
  60 /*
  61  * Gang block headers are self-checksumming and contain an array
  62  * of block pointers.
  63  */
  64 #define SPA_GANGBLOCKSIZE       SPA_MINBLOCKSIZE
  65 #define SPA_GBH_NBLKPTRS        ((SPA_GANGBLOCKSIZE - \
  66         sizeof (zio_eck_t)) / sizeof (blkptr_t))
  67 #define SPA_GBH_FILLER          ((SPA_GANGBLOCKSIZE - \
  68         sizeof (zio_eck_t) - \
  69         (SPA_GBH_NBLKPTRS * sizeof (blkptr_t))) /\
  70         sizeof (uint64_t))
  71
  72 typedef struct zio_gbh {
  73         blkptr_t                zg_blkptr[SPA_GBH_NBLKPTRS];
  74         uint64_t                zg_filler[SPA_GBH_FILLER];
  75         zio_eck_t               zg_tail;
  76 } zio_gbh_phys_t;
  77
  78 enum zio_checksum {
  79         ZIO_CHECKSUM_INHERIT = 0,
  80         ZIO_CHECKSUM_ON,
  81         ZIO_CHECKSUM_OFF,
  82         ZIO_CHECKSUM_LABEL,
  83         ZIO_CHECKSUM_GANG_HEADER,
  84         ZIO_CHECKSUM_ZILOG,
  85         ZIO_CHECKSUM_FLETCHER_2,
  86         ZIO_CHECKSUM_FLETCHER_4,
  87         ZIO_CHECKSUM_SHA256,
  88         ZIO_CHECKSUM_ZILOG2,
  89         ZIO_CHECKSUM_NOPARITY,
  90         ZIO_CHECKSUM_SHA512,
  91         ZIO_CHECKSUM_SKEIN,
  92         ZIO_CHECKSUM_EDONR,
  93         ZIO_CHECKSUM_BLAKE3,
  94         ZIO_CHECKSUM_FUNCTIONS
  95 };
  96
  97 /*
  98  * The number of "legacy" compression functions which can be set on individual
  99  * objects.
 100  */
 101 #define ZIO_CHECKSUM_LEGACY_FUNCTIONS ZIO_CHECKSUM_ZILOG2
 102
 103 #define ZIO_CHECKSUM_ON_VALUE   ZIO_CHECKSUM_FLETCHER_4
 104 #define ZIO_CHECKSUM_DEFAULT    ZIO_CHECKSUM_ON
 105
 106 #define ZIO_CHECKSUM_MASK       0xffULL
 107 #define ZIO_CHECKSUM_VERIFY     (1U << 8)
 108
 109 #define ZIO_DEDUPCHECKSUM       ZIO_CHECKSUM_SHA256
 110
 111 /* macros defining encryption lengths */
 112 #define ZIO_OBJSET_MAC_LEN              32
 113 #define ZIO_DATA_IV_LEN                 12
 114 #define ZIO_DATA_SALT_LEN               8
 115 #define ZIO_DATA_MAC_LEN                16
 116
 117 /*
 118  * The number of "legacy" compression functions which can be set on individual
 119  * objects.
 120  */
 121 #define ZIO_COMPRESS_LEGACY_FUNCTIONS ZIO_COMPRESS_LZ4
 122
 123 /*
 124  * The meaning of "compress = on" selected by the compression features enabled
 125  * on a given pool.
 126  */
 127 #define ZIO_COMPRESS_LEGACY_ON_VALUE    ZIO_COMPRESS_LZJB
 128 #define ZIO_COMPRESS_LZ4_ON_VALUE       ZIO_COMPRESS_LZ4
 129
 130 #define ZIO_COMPRESS_DEFAULT            ZIO_COMPRESS_ON
 131
 132 #define BOOTFS_COMPRESS_VALID(compress)                 \
 133         ((compress) == ZIO_COMPRESS_LZJB ||             \
 134         (compress) == ZIO_COMPRESS_LZ4 ||               \
 135         (compress) == ZIO_COMPRESS_GZIP_1 ||            \
 136         (compress) == ZIO_COMPRESS_GZIP_2 ||            \
 137         (compress) == ZIO_COMPRESS_GZIP_3 ||            \
 138         (compress) == ZIO_COMPRESS_GZIP_4 ||            \
 139         (compress) == ZIO_COMPRESS_GZIP_5 ||            \
 140         (compress) == ZIO_COMPRESS_GZIP_6 ||            \
 141         (compress) == ZIO_COMPRESS_GZIP_7 ||            \
 142         (compress) == ZIO_COMPRESS_GZIP_8 ||            \
 143         (compress) == ZIO_COMPRESS_GZIP_9 ||            \
 144         (compress) == ZIO_COMPRESS_ZLE ||               \
 145         (compress) == ZIO_COMPRESS_ZSTD ||              \
 146         (compress) == ZIO_COMPRESS_ON ||                \
 147         (compress) == ZIO_COMPRESS_OFF)
 148
 149
 150 #define ZIO_COMPRESS_ALGO(x)    (x & SPA_COMPRESSMASK)
 151 #define ZIO_COMPRESS_LEVEL(x)   ((x & ~SPA_COMPRESSMASK) >> SPA_COMPRESSBITS)
 152 #define ZIO_COMPRESS_RAW(type, level)   (type | ((level) << SPA_COMPRESSBITS))
 153
 154 #define ZIO_COMPLEVEL_ZSTD(level)       \
 155         ZIO_COMPRESS_RAW(ZIO_COMPRESS_ZSTD, level)
 156
 157 #define ZIO_FAILURE_MODE_WAIT           0
 158 #define ZIO_FAILURE_MODE_CONTINUE       1
 159 #define ZIO_FAILURE_MODE_PANIC          2
 160
 161 typedef enum zio_suspend_reason {
 162         ZIO_SUSPEND_NONE = 0,
 163         ZIO_SUSPEND_IOERR,
 164         ZIO_SUSPEND_MMP,
 165 } zio_suspend_reason_t;
 166
 167 /*
 168  * This was originally an enum type. However, those are 32-bit and there is no
 169  * way to make a 64-bit enum type. Since we ran out of bits for flags, we were
 170  * forced to upgrade it to a uint64_t.
 171  *
 172  * NOTE: PLEASE UPDATE THE BITFIELD STRINGS IN zfs_valstr.c IF YOU ADD ANOTHER
 173  * FLAG.
 174  */
 175 typedef uint64_t zio_flag_t;
 176         /*
 177          * Flags inherited by gang, ddt, and vdev children,
 178          * and that must be equal for two zios to aggregate
 179          */
 180 #define ZIO_FLAG_DONT_AGGREGATE (1ULL << 0)
 181 #define ZIO_FLAG_IO_REPAIR      (1ULL << 1)
 182 #define ZIO_FLAG_SELF_HEAL      (1ULL << 2)
 183 #define ZIO_FLAG_RESILVER       (1ULL << 3)
 184 #define ZIO_FLAG_SCRUB          (1ULL << 4)
 185 #define ZIO_FLAG_SCAN_THREAD    (1ULL << 5)
 186 #define ZIO_FLAG_PHYSICAL       (1ULL << 6)
 187
 188 #define ZIO_FLAG_AGG_INHERIT    (ZIO_FLAG_CANFAIL - 1)
 189
 190         /*
 191          * Flags inherited by ddt, gang, and vdev children.
 192          */
 193 #define ZIO_FLAG_CANFAIL        (1ULL << 7)     /* must be first for INHERIT */
 194 #define ZIO_FLAG_SPECULATIVE    (1ULL << 8)
 195 #define ZIO_FLAG_CONFIG_WRITER  (1ULL << 9)
 196 #define ZIO_FLAG_DONT_RETRY     (1ULL << 10)
 197 #define ZIO_FLAG_NODATA         (1ULL << 12)
 198 #define ZIO_FLAG_INDUCE_DAMAGE  (1ULL << 13)
 199 #define ZIO_FLAG_IO_ALLOCATING  (1ULL << 14)
 200
 201 #define ZIO_FLAG_DDT_INHERIT    (ZIO_FLAG_IO_RETRY - 1)
 202 #define ZIO_FLAG_GANG_INHERIT   (ZIO_FLAG_IO_RETRY - 1)
 203
 204         /*
 205          * Flags inherited by vdev children.
 206          */
 207 #define ZIO_FLAG_IO_RETRY       (1ULL << 15)    /* must be first for INHERIT */
 208 #define ZIO_FLAG_PROBE          (1ULL << 16)
 209 #define ZIO_FLAG_TRYHARD        (1ULL << 17)
 210 #define ZIO_FLAG_OPTIONAL       (1ULL << 18)
 211 #define ZIO_FLAG_DIO_READ       (1ULL << 19)
 212 #define ZIO_FLAG_VDEV_INHERIT   (ZIO_FLAG_DONT_QUEUE - 1)
 213
 214         /*
 215          * Flags not inherited by any children.
 216          */
 217 #define ZIO_FLAG_DONT_QUEUE     (1ULL << 20)    /* must be first for INHERIT */
 218 #define ZIO_FLAG_DONT_PROPAGATE (1ULL << 21)
 219 #define ZIO_FLAG_IO_BYPASS      (1ULL << 22)
 220 #define ZIO_FLAG_IO_REWRITE     (1ULL << 23)
 221 #define ZIO_FLAG_RAW_COMPRESS   (1ULL << 24)
 222 #define ZIO_FLAG_RAW_ENCRYPT    (1ULL << 25)
 223 #define ZIO_FLAG_GANG_CHILD     (1ULL << 26)
 224 #define ZIO_FLAG_DDT_CHILD      (1ULL << 27)
 225 #define ZIO_FLAG_GODFATHER      (1ULL << 28)
 226 #define ZIO_FLAG_NOPWRITE       (1ULL << 29)
 227 #define ZIO_FLAG_REEXECUTED     (1ULL << 30)
 228 #define ZIO_FLAG_DELEGATED      (1ULL << 31)
 229 #define ZIO_FLAG_DIO_CHKSUM_ERR (1ULL << 32)
 230
 231 #define ZIO_ALLOCATOR_NONE      (-1)
 232 #define ZIO_HAS_ALLOCATOR(zio)  ((zio)->io_allocator != ZIO_ALLOCATOR_NONE)
 233
 234 #define ZIO_FLAG_MUSTSUCCEED            0
 235 #define ZIO_FLAG_RAW    (ZIO_FLAG_RAW_COMPRESS | ZIO_FLAG_RAW_ENCRYPT)
 236
 237 #define ZIO_DDT_CHILD_FLAGS(zio)                                \
 238         (((zio)->io_flags & ZIO_FLAG_DDT_INHERIT) |             \
 239         ZIO_FLAG_DDT_CHILD | ZIO_FLAG_CANFAIL)
 240
 241 #define ZIO_GANG_CHILD_FLAGS(zio)                               \
 242         (((zio)->io_flags & ZIO_FLAG_GANG_INHERIT) |            \
 243         ZIO_FLAG_GANG_CHILD | ZIO_FLAG_CANFAIL)
 244
 245 #define ZIO_VDEV_CHILD_FLAGS(zio)                               \
 246         (((zio)->io_flags & ZIO_FLAG_VDEV_INHERIT) |            \
 247         ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_CANFAIL)
 248
 249 #define ZIO_CHILD_BIT(x)                (1U << (x))
 250 #define ZIO_CHILD_BIT_IS_SET(val, x)    ((val) & (1U << (x)))
 251
 252 enum zio_child {
 253         ZIO_CHILD_VDEV = 0,
 254         ZIO_CHILD_GANG,
 255         ZIO_CHILD_DDT,
 256         ZIO_CHILD_LOGICAL,
 257         ZIO_CHILD_TYPES
 258 };
 259
 260 #define ZIO_CHILD_VDEV_BIT              ZIO_CHILD_BIT(ZIO_CHILD_VDEV)
 261 #define ZIO_CHILD_GANG_BIT              ZIO_CHILD_BIT(ZIO_CHILD_GANG)
 262 #define ZIO_CHILD_DDT_BIT               ZIO_CHILD_BIT(ZIO_CHILD_DDT)
 263 #define ZIO_CHILD_LOGICAL_BIT           ZIO_CHILD_BIT(ZIO_CHILD_LOGICAL)
 264 #define ZIO_CHILD_ALL_BITS                                      \
 265         (ZIO_CHILD_VDEV_BIT | ZIO_CHILD_GANG_BIT |              \
 266         ZIO_CHILD_DDT_BIT | ZIO_CHILD_LOGICAL_BIT)
 267
 268 enum zio_wait_type {
 269         ZIO_WAIT_READY = 0,
 270         ZIO_WAIT_DONE,
 271         ZIO_WAIT_TYPES
 272 };
 273
 274 typedef void zio_done_func_t(zio_t *zio);
 275
 276 extern int zio_exclude_metadata;
 277 extern int zio_dva_throttle_enabled;
 278 extern const char *const zio_type_name[ZIO_TYPES];
 279
 280 /*
 281  * A bookmark is a four-tuple <objset, object, level, blkid> that uniquely
 282  * identifies any block in the pool.  By convention, the meta-objset (MOS)
 283  * is objset 0, and the meta-dnode is object 0.  This covers all blocks
 284  * except root blocks and ZIL blocks, which are defined as follows:
 285  *
 286  * Root blocks (objset_phys_t) are object 0, level -1:  <objset, 0, -1, 0>.
 287  * ZIL blocks are bookmarked <objset, 0, -2, blkid == ZIL sequence number>.
 288  * dmu_sync()ed ZIL data blocks are bookmarked <objset, object, -2, blkid>.
 289  * dnode visit bookmarks are <objset, object id of dnode, -3, 0>.
 290  *
 291  * Note: this structure is called a bookmark because its original purpose
 292  * was to remember where to resume a pool-wide traverse.
 293  *
 294  * Note: this structure is passed between userland and the kernel, and is
 295  * stored on disk (by virtue of being incorporated into other on-disk
 296  * structures, e.g. dsl_scan_phys_t).
 297  *
 298  * If the head_errlog feature is enabled a different on-disk format for error
 299  * logs is used. This introduces the use of an error bookmark, a four-tuple
 300  * <object, level, blkid, birth> that uniquely identifies any error block
 301  * in the pool. The birth transaction group is used to track whether the block
 302  * has been overwritten by newer data or added to a snapshot since its marking
 303  * as an error.
 304  */
 305 struct zbookmark_phys {
 306         uint64_t        zb_objset;
 307         uint64_t        zb_object;
 308         int64_t         zb_level;
 309         uint64_t        zb_blkid;
 310 };
 311
 312 struct zbookmark_err_phys {
 313         uint64_t        zb_object;
 314         int64_t         zb_level;
 315         uint64_t        zb_blkid;
 316         uint64_t        zb_birth;
 317 };
 318
 319 #define SET_BOOKMARK(zb, objset, object, level, blkid)  \
 320 {                                                       \
 321         (zb)->zb_objset = objset;                       \
 322         (zb)->zb_object = object;                       \
 323         (zb)->zb_level = level;                         \
 324         (zb)->zb_blkid = blkid;                         \
 325 }
 326
 327 #define ZB_DESTROYED_OBJSET     (-1ULL)
 328
 329 #define ZB_ROOT_OBJECT          (0ULL)
 330 #define ZB_ROOT_LEVEL           (-1LL)
 331 #define ZB_ROOT_BLKID           (0ULL)
 332
 333 #define ZB_ZIL_OBJECT           (0ULL)
 334 #define ZB_ZIL_LEVEL            (-2LL)
 335
 336 #define ZB_DNODE_LEVEL          (-3LL)
 337 #define ZB_DNODE_BLKID          (0ULL)
 338
 339 #define ZB_IS_ZERO(zb)                                          \
 340         ((zb)->zb_objset == 0 && (zb)->zb_object == 0 &&        \
 341         (zb)->zb_level == 0 && (zb)->zb_blkid == 0)
 342 #define ZB_IS_ROOT(zb)                          \
 343         ((zb)->zb_object == ZB_ROOT_OBJECT &&   \
 344         (zb)->zb_level == ZB_ROOT_LEVEL &&      \
 345         (zb)->zb_blkid == ZB_ROOT_BLKID)
 346
 347 typedef struct zio_prop {
 348         enum zio_checksum       zp_checksum;
 349         enum zio_compress       zp_compress;
 350         uint8_t                 zp_complevel;
 351         uint8_t                 zp_level;
 352         uint8_t                 zp_copies;
 353         dmu_object_type_t       zp_type;
 354         boolean_t               zp_dedup;
 355         boolean_t               zp_dedup_verify;
 356         boolean_t               zp_nopwrite;
 357         boolean_t               zp_brtwrite;
 358         boolean_t               zp_encrypt;
 359         boolean_t               zp_byteorder;
 360         boolean_t               zp_direct_write;
 361         uint8_t                 zp_salt[ZIO_DATA_SALT_LEN];
 362         uint8_t                 zp_iv[ZIO_DATA_IV_LEN];
 363         uint8_t                 zp_mac[ZIO_DATA_MAC_LEN];
 364         uint32_t                zp_zpl_smallblk;
 365         dmu_object_type_t       zp_storage_type;
 366 } zio_prop_t;
 367
 368 typedef struct zio_cksum_report zio_cksum_report_t;
 369
 370 typedef void zio_cksum_finish_f(zio_cksum_report_t *rep,
 371     const abd_t *good_data);
 372 typedef void zio_cksum_free_f(void *cbdata, size_t size);
 373
 374 struct zio_bad_cksum;                           /* defined in zio_checksum.h */
 375 struct dnode_phys;
 376 struct abd;
 377
 378 struct zio_cksum_report {
 379         struct zio_cksum_report *zcr_next;
 380         nvlist_t                *zcr_ereport;
 381         nvlist_t                *zcr_detector;
 382         void                    *zcr_cbdata;
 383         size_t                  zcr_cbinfo;     /* passed to zcr_free() */
 384         uint64_t                zcr_sector;
 385         uint64_t                zcr_align;
 386         uint64_t                zcr_length;
 387         zio_cksum_finish_f      *zcr_finish;
 388         zio_cksum_free_f        *zcr_free;
 389
 390         /* internal use only */
 391         struct zio_bad_cksum    *zcr_ckinfo;    /* information from failure */
 392 };
 393
 394 typedef struct zio_vsd_ops {
 395         zio_done_func_t         *vsd_free;
 396 } zio_vsd_ops_t;
 397
 398 typedef struct zio_gang_node {
 399         zio_gbh_phys_t          *gn_gbh;
 400         struct zio_gang_node    *gn_child[SPA_GBH_NBLKPTRS];
 401 } zio_gang_node_t;
 402
 403 typedef zio_t *zio_gang_issue_func_t(zio_t *zio, blkptr_t *bp,
 404     zio_gang_node_t *gn, struct abd *data, uint64_t offset);
 405
 406 typedef void zio_transform_func_t(zio_t *zio, struct abd *data, uint64_t size);
 407
 408 typedef struct zio_transform {
 409         struct abd              *zt_orig_abd;
 410         uint64_t                zt_orig_size;
 411         uint64_t                zt_bufsize;
 412         zio_transform_func_t    *zt_transform;
 413         struct zio_transform    *zt_next;
 414 } zio_transform_t;
 415
 416 typedef zio_t *zio_pipe_stage_t(zio_t *zio);
 417
 418 /*
 419  * The io_reexecute flags are distinct from io_flags because the child must
 420  * be able to propagate them to the parent.  The normal io_flags are local
 421  * to the zio, not protected by any lock, and not modifiable by children;
 422  * the reexecute flags are protected by io_lock, modifiable by children,
 423  * and always propagated -- even when ZIO_FLAG_DONT_PROPAGATE is set.
 424  */
 425 #define ZIO_REEXECUTE_NOW       0x01
 426 #define ZIO_REEXECUTE_SUSPEND   0x02
 427
 428 /*
 429  * The io_trim flags are used to specify the type of TRIM to perform.  They
 430  * only apply to ZIO_TYPE_TRIM zios are distinct from io_flags.
 431  */
 432 enum trim_flag {
 433         ZIO_TRIM_SECURE         = 1U << 0,
 434 };
 435
 436 typedef struct zio_alloc_list {
 437         list_t  zal_list;
 438         uint64_t zal_size;
 439 } zio_alloc_list_t;
 440
 441 typedef struct zio_link {
 442         zio_t           *zl_parent;
 443         zio_t           *zl_child;
 444         list_node_t     zl_parent_node;
 445         list_node_t     zl_child_node;
 446 } zio_link_t;
 447
 448 enum zio_qstate {
 449         ZIO_QS_NONE = 0,
 450         ZIO_QS_QUEUED,
 451         ZIO_QS_ACTIVE,
 452 };
 453
 454 struct zio {
 455         /* Core information about this I/O */
 456         zbookmark_phys_t        io_bookmark;
 457         zio_prop_t      io_prop;
 458         zio_type_t      io_type;
 459         enum zio_child  io_child_type;
 460         enum trim_flag  io_trim_flags;
 461         zio_priority_t  io_priority;
 462         uint8_t         io_reexecute;
 463         uint8_t         io_state[ZIO_WAIT_TYPES];
 464         uint64_t        io_txg;
 465         spa_t           *io_spa;
 466         blkptr_t        *io_bp;
 467         blkptr_t        *io_bp_override;
 468         blkptr_t        io_bp_copy;
 469         list_t          io_parent_list;
 470         list_t          io_child_list;
 471         zio_t           *io_logical;
 472         zio_transform_t *io_transform_stack;
 473
 474         /* Callback info */
 475         zio_done_func_t *io_ready;
 476         zio_done_func_t *io_children_ready;
 477         zio_done_func_t *io_done;
 478         void            *io_private;
 479         int64_t         io_prev_space_delta;    /* DMU private */
 480         blkptr_t        io_bp_orig;
 481         /* io_lsize != io_orig_size iff this is a raw write */
 482         uint64_t        io_lsize;
 483
 484         /* Data represented by this I/O */
 485         struct abd      *io_abd;
 486         struct abd      *io_orig_abd;
 487         uint64_t        io_size;
 488         uint64_t        io_orig_size;
 489
 490         /* Stuff for the vdev stack */
 491         vdev_t          *io_vd;
 492         void            *io_vsd;
 493         const zio_vsd_ops_t *io_vsd_ops;
 494         metaslab_class_t *io_metaslab_class;    /* dva throttle class */
 495
 496         enum zio_qstate io_queue_state; /* vdev queue state */
 497         union {
 498                 list_node_t l;
 499                 avl_node_t a;
 500         } io_queue_node ____cacheline_aligned;  /* allocator and vdev queues */
 501         avl_node_t      io_offset_node; /* vdev offset queues */
 502         uint64_t        io_offset;
 503         hrtime_t        io_timestamp;   /* submitted at */
 504         hrtime_t        io_queued_timestamp;
 505         hrtime_t        io_target_timestamp;
 506         hrtime_t        io_delta;       /* vdev queue service delta */
 507         hrtime_t        io_delay;       /* Device access time (disk or */
 508                                         /* file). */
 509         zio_alloc_list_t        io_alloc_list;
 510
 511         /* Internal pipeline state */
 512         zio_flag_t      io_flags;
 513         enum zio_stage  io_stage;
 514         enum zio_stage  io_pipeline;
 515         zio_flag_t      io_orig_flags;
 516         enum zio_stage  io_orig_stage;
 517         enum zio_stage  io_orig_pipeline;
 518         enum zio_stage  io_pipeline_trace;
 519         int             io_error;
 520         int             io_child_error[ZIO_CHILD_TYPES];
 521         uint64_t        io_children[ZIO_CHILD_TYPES][ZIO_WAIT_TYPES];
 522         uint64_t        *io_stall;
 523         zio_t           *io_gang_leader;
 524         zio_gang_node_t *io_gang_tree;
 525         void            *io_executor;
 526         void            *io_waiter;
 527         void            *io_bio;
 528         kmutex_t        io_lock;
 529         kcondvar_t      io_cv;
 530         int             io_allocator;
 531
 532         /* FMA state */
 533         zio_cksum_report_t *io_cksum_report;
 534         uint64_t        io_ena;
 535
 536         /* Taskq dispatching state */
 537         taskq_ent_t     io_tqent;
 538 };
 539
 540 enum blk_verify_flag {
 541         BLK_VERIFY_ONLY,
 542         BLK_VERIFY_LOG,
 543         BLK_VERIFY_HALT
 544 };
 545
 546 enum blk_config_flag {
 547         BLK_CONFIG_HELD,   // SCL_VDEV held for writer
 548         BLK_CONFIG_NEEDED, // SCL_VDEV should be obtained for reader
 549         BLK_CONFIG_SKIP,   // skip checks which require SCL_VDEV
 550 };
 551
 552 extern int zio_bookmark_compare(const void *, const void *);
 553
 554 extern zio_t *zio_null(zio_t *pio, spa_t *spa, vdev_t *vd,
 555     zio_done_func_t *done, void *priv, zio_flag_t flags);
 556
 557 extern zio_t *zio_root(spa_t *spa,
 558     zio_done_func_t *done, void *priv, zio_flag_t flags);
 559
 560 extern void zio_destroy(zio_t *zio);
 561
 562 extern zio_t *zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
 563     struct abd *data, uint64_t lsize, zio_done_func_t *done, void *priv,
 564     zio_priority_t priority, zio_flag_t flags, const zbookmark_phys_t *zb);
 565
 566 extern zio_t *zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
 567     struct abd *data, uint64_t size, uint64_t psize, const zio_prop_t *zp,
 568     zio_done_func_t *ready, zio_done_func_t *children_ready,
 569     zio_done_func_t *done, void *priv, zio_priority_t priority,
 570     zio_flag_t flags, const zbookmark_phys_t *zb);
 571
 572 extern zio_t *zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
 573     struct abd *data, uint64_t size, zio_done_func_t *done, void *priv,
 574     zio_priority_t priority, zio_flag_t flags, zbookmark_phys_t *zb);
 575
 576 extern void zio_write_override(zio_t *zio, blkptr_t *bp, int copies,
 577     boolean_t nopwrite, boolean_t brtwrite);
 578
 579 extern void zio_free(spa_t *spa, uint64_t txg, const blkptr_t *bp);
 580
 581 extern zio_t *zio_claim(zio_t *pio, spa_t *spa, uint64_t txg,
 582     const blkptr_t *bp,
 583     zio_done_func_t *done, void *priv, zio_flag_t flags);
 584
 585 extern zio_t *zio_trim(zio_t *pio, vdev_t *vd, uint64_t offset, uint64_t size,
 586     zio_done_func_t *done, void *priv, zio_priority_t priority,
 587     zio_flag_t flags, enum trim_flag trim_flags);
 588
 589 extern zio_t *zio_read_phys(zio_t *pio, vdev_t *vd, uint64_t offset,
 590     uint64_t size, struct abd *data, int checksum,
 591     zio_done_func_t *done, void *priv, zio_priority_t priority,
 592     zio_flag_t flags, boolean_t labels);
 593
 594 extern zio_t *zio_write_phys(zio_t *pio, vdev_t *vd, uint64_t offset,
 595     uint64_t size, struct abd *data, int checksum,
 596     zio_done_func_t *done, void *priv, zio_priority_t priority,
 597     zio_flag_t flags, boolean_t labels);
 598
 599 extern zio_t *zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg,
 600     const blkptr_t *bp, zio_flag_t flags);
 601
 602 extern int zio_alloc_zil(spa_t *spa, objset_t *os, uint64_t txg,
 603     blkptr_t *new_bp, uint64_t size, boolean_t *slog);
 604 extern void zio_flush(zio_t *zio, vdev_t *vd);
 605 extern void zio_shrink(zio_t *zio, uint64_t size);
 606
 607 extern size_t zio_get_compression_max_size(enum zio_compress compress,
 608     uint64_t gcd_alloc, uint64_t min_alloc, size_t s_len);
 609 extern int zio_wait(zio_t *zio);
 610 extern void zio_nowait(zio_t *zio);
 611 extern void zio_execute(void *zio);
 612 extern void zio_interrupt(void *zio);
 613 extern void zio_delay_init(zio_t *zio);
 614 extern void zio_delay_interrupt(zio_t *zio);
 615 extern void zio_deadman(zio_t *zio, const char *tag);
 616
 617 extern zio_t *zio_walk_parents(zio_t *cio, zio_link_t **);
 618 extern zio_t *zio_walk_children(zio_t *pio, zio_link_t **);
 619 extern zio_t *zio_unique_parent(zio_t *cio);
 620 extern void zio_add_child(zio_t *pio, zio_t *cio);
 621 extern void zio_add_child_first(zio_t *pio, zio_t *cio);
 622
 623 extern void *zio_buf_alloc(size_t size);
 624 extern void zio_buf_free(void *buf, size_t size);
 625 extern void *zio_data_buf_alloc(size_t size);
 626 extern void zio_data_buf_free(void *buf, size_t size);
 627
 628 extern void zio_push_transform(zio_t *zio, struct abd *abd, uint64_t size,
 629     uint64_t bufsize, zio_transform_func_t *transform);
 630 extern void zio_pop_transforms(zio_t *zio);
 631
 632 extern void zio_resubmit_stage_async(void *);
 633
 634 extern zio_t *zio_vdev_child_io(zio_t *zio, blkptr_t *bp, vdev_t *vd,
 635     uint64_t offset, struct abd *data, uint64_t size, int type,
 636     zio_priority_t priority, zio_flag_t flags,
 637     zio_done_func_t *done, void *priv);
 638
 639 extern zio_t *zio_vdev_delegated_io(vdev_t *vd, uint64_t offset,
 640     struct abd *data, uint64_t size, zio_type_t type, zio_priority_t priority,
 641     zio_flag_t flags, zio_done_func_t *done, void *priv);
 642
 643 extern void zio_vdev_io_bypass(zio_t *zio);
 644 extern void zio_vdev_io_reissue(zio_t *zio);
 645 extern void zio_vdev_io_redone(zio_t *zio);
 646
 647 extern void zio_change_priority(zio_t *pio, zio_priority_t priority);
 648
 649 extern void zio_checksum_verified(zio_t *zio);
 650 extern void zio_dio_chksum_verify_error_report(zio_t *zio);
 651 extern int zio_worst_error(int e1, int e2);
 652
 653 extern enum zio_checksum zio_checksum_select(enum zio_checksum child,
 654     enum zio_checksum parent);
 655 extern enum zio_checksum zio_checksum_dedup_select(spa_t *spa,
 656     enum zio_checksum child, enum zio_checksum parent);
 657 extern enum zio_compress zio_compress_select(spa_t *spa,
 658     enum zio_compress child, enum zio_compress parent);
 659 extern uint8_t zio_complevel_select(spa_t *spa, enum zio_compress compress,
 660     uint8_t child, uint8_t parent);
 661
 662 extern void zio_suspend(spa_t *spa, zio_t *zio, zio_suspend_reason_t);
 663 extern int zio_resume(spa_t *spa);
 664 extern void zio_resume_wait(spa_t *spa);
 665
 666 extern boolean_t zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp,
 667     enum blk_config_flag blk_config, enum blk_verify_flag blk_verify);
 668
 669 /*
 670  * Initial setup and teardown.
 671  */
 672 extern void zio_init(void);
 673 extern void zio_fini(void);
 674
 675 /*
 676  * Fault injection
 677  */
 678 struct zinject_record;
 679 extern uint32_t zio_injection_enabled;
 680 extern int zio_inject_fault(char *name, int flags, int *id,
 681     struct zinject_record *record);
 682 extern int zio_inject_list_next(int *id, char *name, size_t buflen,
 683     struct zinject_record *record);
 684 extern int zio_clear_fault(int id);
 685 extern void zio_handle_panic_injection(spa_t *spa, const char *tag,
 686     uint64_t type);
 687 extern int zio_handle_decrypt_injection(spa_t *spa, const zbookmark_phys_t *zb,
 688     uint64_t type, int error);
 689 extern int zio_handle_fault_injection(zio_t *zio, int error);
 690 extern int zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error);
 691 extern int zio_handle_device_injections(vdev_t *vd, zio_t *zio, int err1,
 692     int err2);
 693 extern int zio_handle_label_injection(zio_t *zio, int error);
 694 extern void zio_handle_ignored_writes(zio_t *zio);
 695 extern hrtime_t zio_handle_io_delay(zio_t *zio);
 696 extern void zio_handle_import_delay(spa_t *spa, hrtime_t elapsed);
 697 extern void zio_handle_export_delay(spa_t *spa, hrtime_t elapsed);
 698
 699 /*
 700  * Checksum ereport functions
 701  */
 702 extern int zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd,
 703     const zbookmark_phys_t *zb, struct zio *zio, uint64_t offset,
 704     uint64_t length, struct zio_bad_cksum *info);
 705 extern void zfs_ereport_finish_checksum(zio_cksum_report_t *report,
 706     const abd_t *good_data, const abd_t *bad_data, boolean_t drop_if_identical);
 707
 708 extern void zfs_ereport_free_checksum(zio_cksum_report_t *report);
 709
 710 /* If we have the good data in hand, this function can be used */
 711 extern int zfs_ereport_post_checksum(spa_t *spa, vdev_t *vd,
 712     const zbookmark_phys_t *zb, struct zio *zio, uint64_t offset,
 713     uint64_t length, const abd_t *good_data, const abd_t *bad_data,
 714     struct zio_bad_cksum *info);
 715
 716 void zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr);
 717 extern void zfs_ereport_snapshot_post(const char *subclass, spa_t *spa,
 718     const char *name);
 719
 720 /* Called from spa_sync(), but primarily an injection handler */
 721 extern void spa_handle_ignored_writes(spa_t *spa);
 722
 723 /* zbookmark_phys functions */
 724 boolean_t zbookmark_subtree_completed(const struct dnode_phys *dnp,
 725     const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block);
 726 boolean_t zbookmark_subtree_tbd(const struct dnode_phys *dnp,
 727     const zbookmark_phys_t *subtree_root, const zbookmark_phys_t *last_block);
 728 int zbookmark_compare(uint16_t dbss1, uint8_t ibs1, uint16_t dbss2,
 729     uint8_t ibs2, const zbookmark_phys_t *zb1, const zbookmark_phys_t *zb2);
 730
 731 #ifdef  __cplusplus
 732 }
 733 #endif
 734
 735 #endif  /* _ZIO_H */