include/sys/abd_impl.h

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
  23  * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
  24  */
  25
  26 #ifndef _ABD_IMPL_H
  27 #define _ABD_IMPL_H
  28
  29 #include <sys/abd.h>
  30
  31 #ifdef __cplusplus
  32 extern "C" {
  33 #endif
  34
  35 typedef enum abd_flags {
  36         ABD_FLAG_LINEAR         = 1 << 0, /* is buffer linear (or scattered)? */
  37         ABD_FLAG_OWNER          = 1 << 1, /* does it own its data buffers? */
  38         ABD_FLAG_META           = 1 << 2, /* does this represent FS metadata? */
  39         ABD_FLAG_MULTI_ZONE     = 1 << 3, /* pages split over memory zones */
  40         ABD_FLAG_MULTI_CHUNK    = 1 << 4, /* pages split over multiple chunks */
  41         ABD_FLAG_LINEAR_PAGE    = 1 << 5, /* linear but allocd from page */
  42         ABD_FLAG_GANG           = 1 << 6, /* mult ABDs chained together */
  43         ABD_FLAG_GANG_FREE      = 1 << 7, /* gang ABD is responsible for mem */
  44         ABD_FLAG_ZEROS          = 1 << 8, /* ABD for zero-filled buffer */
  45 } abd_flags_t;
  46
  47 typedef enum abd_stats_op {
  48         ABDSTAT_INCR, /* Increase abdstat values */
  49         ABDSTAT_DECR  /* Decrease abdstat values */
  50 } abd_stats_op_t;
  51
  52 struct abd {
  53         abd_flags_t     abd_flags;
  54         uint_t          abd_size;       /* excludes scattered abd_offset */
  55         list_node_t     abd_gang_link;
  56         struct abd      *abd_parent;
  57         zfs_refcount_t  abd_children;
  58         kmutex_t        abd_mtx;
  59         union {
  60                 struct abd_scatter {
  61                         uint_t          abd_offset;
  62 #if defined(__FreeBSD__) && defined(_KERNEL)
  63                         uint_t  abd_chunk_size;
  64                         void    *abd_chunks[];
  65 #else
  66                         uint_t          abd_nents;
  67                         struct scatterlist *abd_sgl;
  68 #endif
  69                 } abd_scatter;
  70                 struct abd_linear {
  71                         void            *abd_buf;
  72                         struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
  73                 } abd_linear;
  74                 struct abd_gang {
  75                         list_t abd_gang_chain;
  76                 } abd_gang;
  77         } abd_u;
  78 };
  79
  80 struct scatterlist; /* forward declaration */
  81
  82 struct abd_iter {
  83         /* public interface */
  84         void            *iter_mapaddr;  /* addr corresponding to iter_pos */
  85         size_t          iter_mapsize;   /* length of data valid at mapaddr */
  86
  87         /* private */
  88         abd_t           *iter_abd;      /* ABD being iterated through */
  89         size_t          iter_pos;
  90         size_t          iter_offset;    /* offset in current sg/abd_buf, */
  91                                         /* abd_offset included */
  92         struct scatterlist *iter_sg;    /* current sg */
  93 };
  94
  95 extern abd_t *abd_zero_scatter;
  96
  97 abd_t *abd_gang_get_offset(abd_t *, size_t *);
  98
  99 /*
 100  * OS specific functions
 101  */
 102
 103 abd_t *abd_alloc_struct(size_t);
 104 abd_t *abd_get_offset_scatter(abd_t *, size_t);
 105 void abd_free_struct(abd_t *);
 106 void abd_alloc_chunks(abd_t *, size_t);
 107 void abd_free_chunks(abd_t *);
 108 boolean_t abd_size_alloc_linear(size_t);
 109 void abd_update_scatter_stats(abd_t *, abd_stats_op_t);
 110 void abd_update_linear_stats(abd_t *, abd_stats_op_t);
 111 void abd_verify_scatter(abd_t *);
 112 void abd_free_linear_page(abd_t *);
 113 /* OS specific abd_iter functions */
 114 void abd_iter_init(struct abd_iter  *, abd_t *);
 115 boolean_t abd_iter_at_end(struct abd_iter *);
 116 void abd_iter_advance(struct abd_iter *, size_t);
 117 void abd_iter_map(struct abd_iter *);
 118 void abd_iter_unmap(struct abd_iter *);
 119
 120 /*
 121  * Helper macros
 122  */
 123 #define ABDSTAT(stat)           (abd_stats.stat.value.ui64)
 124 #define ABDSTAT_INCR(stat, val) \
 125         atomic_add_64(&abd_stats.stat.value.ui64, (val))
 126 #define ABDSTAT_BUMP(stat)      ABDSTAT_INCR(stat, 1)
 127 #define ABDSTAT_BUMPDOWN(stat)  ABDSTAT_INCR(stat, -1)
 128
 129 #define ABD_SCATTER(abd)        (abd->abd_u.abd_scatter)
 130 #define ABD_LINEAR_BUF(abd)     (abd->abd_u.abd_linear.abd_buf)
 131 #define ABD_GANG(abd)           (abd->abd_u.abd_gang)
 132
 133 #if defined(_KERNEL)
 134 #if defined(__FreeBSD__)
 135 #define abd_enter_critical(flags)       critical_enter()
 136 #define abd_exit_critical(flags)        critical_exit()
 137 #else
 138 #define abd_enter_critical(flags)       local_irq_save(flags)
 139 #define abd_exit_critical(flags)        local_irq_restore(flags)
 140 #endif
 141 #else /* !_KERNEL */
 142 #define abd_enter_critical(flags)       ((void)0)
 143 #define abd_exit_critical(flags)        ((void)0)
 144 #endif
 145
 146 #ifdef __cplusplus
 147 }
 148 #endif
 149
 150 #endif  /* _ABD_IMPL_H */