include/sys/abd.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
  23  * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
  24  */
  25
  26 #ifndef _ABD_H
  27 #define _ABD_H
  28
  29 #include <sys/isa_defs.h>
  30 #include <sys/debug.h>
  31 #include <sys/zfs_refcount.h>
  32 #include <sys/uio.h>
  33
  34 #ifdef __cplusplus
  35 extern "C" {
  36 #endif
  37
  38 typedef enum abd_flags {
  39         ABD_FLAG_LINEAR         = 1 << 0, /* is buffer linear (or scattered)? */
  40         ABD_FLAG_OWNER          = 1 << 1, /* does it own its data buffers? */
  41         ABD_FLAG_META           = 1 << 2, /* does this represent FS metadata? */
  42         ABD_FLAG_MULTI_ZONE     = 1 << 3, /* pages split over memory zones */
  43         ABD_FLAG_MULTI_CHUNK    = 1 << 4, /* pages split over multiple chunks */
  44         ABD_FLAG_LINEAR_PAGE    = 1 << 5, /* linear but allocd from page */
  45         ABD_FLAG_GANG           = 1 << 6, /* mult ABDs chained together */
  46         ABD_FLAG_GANG_FREE      = 1 << 7, /* gang ABD is responsible for mem */
  47         ABD_FLAG_ZEROS          = 1 << 8, /* ABD for zero-filled buffer */
  48         ABD_FLAG_ALLOCD         = 1 << 9, /* we allocated the abd_t */
  49 } abd_flags_t;
  50
  51 typedef struct abd {
  52         abd_flags_t     abd_flags;
  53         uint_t          abd_size;       /* excludes scattered abd_offset */
  54         list_node_t     abd_gang_link;
  55 #ifdef ZFS_DEBUG
  56         struct abd      *abd_parent;
  57         zfs_refcount_t  abd_children;
  58 #endif
  59         kmutex_t        abd_mtx;
  60         union {
  61                 struct abd_scatter {
  62                         uint_t          abd_offset;
  63 #if defined(__FreeBSD__) && defined(_KERNEL)
  64                         void    *abd_chunks[1]; /* actually variable-length */
  65 #else
  66                         uint_t          abd_nents;
  67                         struct scatterlist *abd_sgl;
  68 #endif
  69                 } abd_scatter;
  70                 struct abd_linear {
  71                         void            *abd_buf;
  72                         struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
  73                 } abd_linear;
  74                 struct abd_gang {
  75                         list_t abd_gang_chain;
  76                 } abd_gang;
  77         } abd_u;
  78 } abd_t;
  79
  80 typedef int abd_iter_func_t(void *buf, size_t len, void *priv);
  81 typedef int abd_iter_func2_t(void *bufa, void *bufb, size_t len, void *priv);
  82
  83 extern int zfs_abd_scatter_enabled;
  84
  85 /*
  86  * Allocations and deallocations
  87  */
  88
  89 __attribute__((malloc))
  90 abd_t *abd_alloc(size_t, boolean_t);
  91 __attribute__((malloc))
  92 abd_t *abd_alloc_linear(size_t, boolean_t);
  93 __attribute__((malloc))
  94 abd_t *abd_alloc_gang(void);
  95 __attribute__((malloc))
  96 abd_t *abd_alloc_for_io(size_t, boolean_t);
  97 __attribute__((malloc))
  98 abd_t *abd_alloc_sametype(abd_t *, size_t);
  99 boolean_t abd_size_alloc_linear(size_t);
 100 void abd_gang_add(abd_t *, abd_t *, boolean_t);
 101 void abd_free(abd_t *);
 102 abd_t *abd_get_offset(abd_t *, size_t);
 103 abd_t *abd_get_offset_size(abd_t *, size_t, size_t);
 104 abd_t *abd_get_offset_struct(abd_t *, abd_t *, size_t, size_t);
 105 abd_t *abd_get_zeros(size_t);
 106 abd_t *abd_get_from_buf(void *, size_t);
 107 void abd_cache_reap_now(void);
 108
 109 /*
 110  * Conversion to and from a normal buffer
 111  */
 112
 113 void *abd_to_buf(abd_t *);
 114 void *abd_borrow_buf(abd_t *, size_t);
 115 void *abd_borrow_buf_copy(abd_t *, size_t);
 116 void abd_return_buf(abd_t *, void *, size_t);
 117 void abd_return_buf_copy(abd_t *, void *, size_t);
 118 void abd_take_ownership_of_buf(abd_t *, boolean_t);
 119 void abd_release_ownership_of_buf(abd_t *);
 120
 121 /*
 122  * ABD operations
 123  */
 124
 125 int abd_iterate_func(abd_t *, size_t, size_t, abd_iter_func_t *, void *);
 126 int abd_iterate_func2(abd_t *, abd_t *, size_t, size_t, size_t,
 127     abd_iter_func2_t *, void *);
 128 void abd_copy_off(abd_t *, abd_t *, size_t, size_t, size_t);
 129 void abd_copy_from_buf_off(abd_t *, const void *, size_t, size_t);
 130 void abd_copy_to_buf_off(void *, abd_t *, size_t, size_t);
 131 int abd_cmp(abd_t *, abd_t *);
 132 int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
 133 void abd_zero_off(abd_t *, size_t, size_t);
 134 void abd_verify(abd_t *);
 135
 136 void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
 137         ssize_t csize, ssize_t dsize, const unsigned parity,
 138         void (*func_raidz_gen)(void **, const void *, size_t, size_t));
 139 void abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
 140         ssize_t tsize, const unsigned parity,
 141         void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
 142         const unsigned *mul),
 143         const unsigned *mul);
 144
 145 /*
 146  * Wrappers for calls with offsets of 0
 147  */
 148
 149 static inline void
 150 abd_copy(abd_t *dabd, abd_t *sabd, size_t size)
 151 {
 152         abd_copy_off(dabd, sabd, 0, 0, size);
 153 }
 154
 155 static inline void
 156 abd_copy_from_buf(abd_t *abd, const void *buf, size_t size)
 157 {
 158         abd_copy_from_buf_off(abd, buf, 0, size);
 159 }
 160
 161 static inline void
 162 abd_copy_to_buf(void* buf, abd_t *abd, size_t size)
 163 {
 164         abd_copy_to_buf_off(buf, abd, 0, size);
 165 }
 166
 167 static inline int
 168 abd_cmp_buf(abd_t *abd, const void *buf, size_t size)
 169 {
 170         return (abd_cmp_buf_off(abd, buf, 0, size));
 171 }
 172
 173 static inline void
 174 abd_zero(abd_t *abd, size_t size)
 175 {
 176         abd_zero_off(abd, 0, size);
 177 }
 178
 179 /*
 180  * ABD type check functions
 181  */
 182 static inline boolean_t
 183 abd_is_linear(abd_t *abd)
 184 {
 185         return ((abd->abd_flags & ABD_FLAG_LINEAR) ? B_TRUE : B_FALSE);
 186 }
 187
 188 static inline boolean_t
 189 abd_is_linear_page(abd_t *abd)
 190 {
 191         return ((abd->abd_flags & ABD_FLAG_LINEAR_PAGE) ? B_TRUE : B_FALSE);
 192 }
 193
 194 static inline boolean_t
 195 abd_is_gang(abd_t *abd)
 196 {
 197         return ((abd->abd_flags & ABD_FLAG_GANG) ? B_TRUE : B_FALSE);
 198 }
 199
 200 static inline uint_t
 201 abd_get_size(abd_t *abd)
 202 {
 203         return (abd->abd_size);
 204 }
 205
 206 /*
 207  * Module lifecycle
 208  * Defined in each specific OS's abd_os.c
 209  */
 210
 211 void abd_init(void);
 212 void abd_fini(void);
 213
 214 /*
 215  * Linux ABD bio functions
 216  */
 217 #if defined(__linux__) && defined(_KERNEL)
 218 unsigned int abd_bio_map_off(struct bio *, abd_t *, unsigned int, size_t);
 219 unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
 220 #endif
 221
 222 #ifdef __cplusplus
 223 }
 224 #endif
 225
 226 #endif  /* _ABD_H */