Optimize RAIDZ expansion

[zfs.git] / module / zfs / abd.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or https://opensource.org/licenses/CDDL-1.0.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
 * Copyright (c) 2019 by Delphix. All rights reserved.
 */

/*
 * ARC buffer data (ABD).
 *
 * ABDs are an abstract data structure for the ARC which can use two
 * different ways of storing the underlying data:
 *
 * (a) Linear buffer. In this case, all the data in the ABD is stored in one
 *     contiguous buffer in memory (from a zio_[data_]buf_* kmem cache).
 *
 *         +-------------------+
 *         | ABD (linear)      |
 *         |   abd_flags = ... |
 *         |   abd_size = ...  |     +--------------------------------+
 *         |   abd_buf ------------->| raw buffer of size abd_size    |
 *         +-------------------+     +--------------------------------+
 *              no abd_chunks
 *
 * (b) Scattered buffer. In this case, the data in the ABD is split into
 *     equal-sized chunks (from the abd_chunk_cache kmem_cache), with pointers
 *     to the chunks recorded in an array at the end of the ABD structure.
 *
 *         +-------------------+
 *         | ABD (scattered)   |
 *         |   abd_flags = ... |
 *         |   abd_size = ...  |
 *         |   abd_offset = 0  |                           +-----------+
 *         |   abd_chunks[0] ----------------------------->| chunk 0   |
 *         |   abd_chunks[1] ---------------------+        +-----------+
 *         |   ...             |                  |        +-----------+
 *         |   abd_chunks[N-1] ---------+         +------->| chunk 1   |
 *         +-------------------+        |                  +-----------+
 *                                      |                      ...
 *                                      |                  +-----------+
 *                                      +----------------->| chunk N-1 |
 *                                                         +-----------+
 *
 * In addition to directly allocating a linear or scattered ABD, it is also
 * possible to create an ABD by requesting the "sub-ABD" starting at an offset
 * within an existing ABD. In linear buffers this is simple (set abd_buf of
 * the new ABD to the starting point within the original raw buffer), but
 * scattered ABDs are a little more complex. The new ABD makes a copy of the
 * relevant abd_chunks pointers (but not the underlying data). However, to
 * provide arbitrary rather than only chunk-aligned starting offsets, it also
 * tracks an abd_offset field which represents the starting point of the data
 * within the first chunk in abd_chunks. For both linear and scattered ABDs,
 * creating an offset ABD marks the original ABD as the offset's parent, and the
 * original ABD's abd_children refcount is incremented. This data allows us to
 * ensure the root ABD isn't deleted before its children.
 *
 * Most consumers should never need to know what type of ABD they're using --
 * the ABD public API ensures that it's possible to transparently switch from
 * using a linear ABD to a scattered one when doing so would be beneficial.
 *
 * If you need to use the data within an ABD directly, if you know it's linear
 * (because you allocated it) you can use abd_to_buf() to access the underlying
 * raw buffer. Otherwise, you should use one of the abd_borrow_buf* functions
 * which will allocate a raw buffer if necessary. Use the abd_return_buf*
 * functions to return any raw buffers that are no longer necessary when you're
 * done using them.
 *
 * There are a variety of ABD APIs that implement basic buffer operations:
 * compare, copy, read, write, and fill with zeroes. If you need a custom
 * function which progressively accesses the whole ABD, use the abd_iterate_*
 * functions.
 *
 * As an additional feature, linear and scatter ABD's can be stitched together
 * by using the gang ABD type (abd_alloc_gang()). This allows for multiple ABDs
 * to be viewed as a singular ABD.
 *
 * It is possible to make all ABDs linear by setting zfs_abd_scatter_enabled to
 * B_FALSE.
 */

#include <sys/abd_impl.h>
#include <sys/param.h>
#include <sys/zio.h>
#include <sys/zfs_context.h>
#include <sys/zfs_znode.h>

/* see block comment above for description */
int zfs_abd_scatter_enabled = B_TRUE;

void
abd_verify(abd_t *abd)
{
#ifdef ZFS_DEBUG
        if (abd_is_from_pages(abd)) {
                ASSERT3U(abd->abd_size, <=, DMU_MAX_ACCESS);
        } else {
                ASSERT3U(abd->abd_size, <=, SPA_MAXBLOCKSIZE);
        }
        ASSERT3U(abd->abd_flags, ==, abd->abd_flags & (ABD_FLAG_LINEAR |
            ABD_FLAG_OWNER | ABD_FLAG_META | ABD_FLAG_MULTI_ZONE |
            ABD_FLAG_MULTI_CHUNK | ABD_FLAG_LINEAR_PAGE | ABD_FLAG_GANG |
            ABD_FLAG_GANG_FREE | ABD_FLAG_ALLOCD | ABD_FLAG_FROM_PAGES));
        IMPLY(abd->abd_parent != NULL, !(abd->abd_flags & ABD_FLAG_OWNER));
        IMPLY(abd->abd_flags & ABD_FLAG_META, abd->abd_flags & ABD_FLAG_OWNER);
        if (abd_is_linear(abd)) {
                ASSERT3U(abd->abd_size, >, 0);
                ASSERT3P(ABD_LINEAR_BUF(abd), !=, NULL);
        } else if (abd_is_gang(abd)) {
                uint_t child_sizes = 0;
                for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
                    cabd != NULL;
                    cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
                        ASSERT(list_link_active(&cabd->abd_gang_link));
                        child_sizes += cabd->abd_size;
                        abd_verify(cabd);
                }
                ASSERT3U(abd->abd_size, ==, child_sizes);
        } else {
                ASSERT3U(abd->abd_size, >, 0);
                abd_verify_scatter(abd);
        }
#endif
}

void
abd_init_struct(abd_t *abd)
{
        list_link_init(&abd->abd_gang_link);
        mutex_init(&abd->abd_mtx, NULL, MUTEX_DEFAULT, NULL);
        abd->abd_flags = 0;
#ifdef ZFS_DEBUG
        zfs_refcount_create(&abd->abd_children);
        abd->abd_parent = NULL;
#endif
        abd->abd_size = 0;
}

static void
abd_fini_struct(abd_t *abd)
{
        mutex_destroy(&abd->abd_mtx);
        ASSERT(!list_link_active(&abd->abd_gang_link));
#ifdef ZFS_DEBUG
        zfs_refcount_destroy(&abd->abd_children);
#endif
}

abd_t *
abd_alloc_struct(size_t size)
{
        abd_t *abd = abd_alloc_struct_impl(size);
        abd_init_struct(abd);
        abd->abd_flags |= ABD_FLAG_ALLOCD;
        return (abd);
}

void
abd_free_struct(abd_t *abd)
{
        abd_fini_struct(abd);
        abd_free_struct_impl(abd);
}

/*
 * Allocate an ABD, along with its own underlying data buffers. Use this if you
 * don't care whether the ABD is linear or not.
 */
abd_t *
abd_alloc(size_t size, boolean_t is_metadata)
{
        if (abd_size_alloc_linear(size))
                return (abd_alloc_linear(size, is_metadata));

        VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);

        abd_t *abd = abd_alloc_struct(size);
        abd->abd_flags |= ABD_FLAG_OWNER;
        abd->abd_u.abd_scatter.abd_offset = 0;
        abd_alloc_chunks(abd, size);

        if (is_metadata) {
                abd->abd_flags |= ABD_FLAG_META;
        }
        abd->abd_size = size;

        abd_update_scatter_stats(abd, ABDSTAT_INCR);

        return (abd);
}

/*
 * Allocate an ABD that must be linear, along with its own underlying data
 * buffer. Only use this when it would be very annoying to write your ABD
 * consumer with a scattered ABD.
 */
abd_t *
abd_alloc_linear(size_t size, boolean_t is_metadata)
{
        abd_t *abd = abd_alloc_struct(0);

        VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);

        abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_OWNER;
        if (is_metadata) {
                abd->abd_flags |= ABD_FLAG_META;
        }
        abd->abd_size = size;

        if (is_metadata) {
                ABD_LINEAR_BUF(abd) = zio_buf_alloc(size);
        } else {
                ABD_LINEAR_BUF(abd) = zio_data_buf_alloc(size);
        }

        abd_update_linear_stats(abd, ABDSTAT_INCR);

        return (abd);
}

static void
abd_free_linear(abd_t *abd)
{
        if (abd_is_linear_page(abd)) {
                abd_free_linear_page(abd);
                return;
        }

        if (abd->abd_flags & ABD_FLAG_META) {
                zio_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
        } else {
                zio_data_buf_free(ABD_LINEAR_BUF(abd), abd->abd_size);
        }

        abd_update_linear_stats(abd, ABDSTAT_DECR);
}

static void
abd_free_gang(abd_t *abd)
{
        ASSERT(abd_is_gang(abd));
        abd_t *cabd;

        while ((cabd = list_head(&ABD_GANG(abd).abd_gang_chain)) != NULL) {
                /*
                 * We must acquire the child ABDs mutex to ensure that if it
                 * is being added to another gang ABD we will set the link
                 * as inactive when removing it from this gang ABD and before
                 * adding it to the other gang ABD.
                 */
                mutex_enter(&cabd->abd_mtx);
                ASSERT(list_link_active(&cabd->abd_gang_link));
                list_remove(&ABD_GANG(abd).abd_gang_chain, cabd);
                mutex_exit(&cabd->abd_mtx);
                if (cabd->abd_flags & ABD_FLAG_GANG_FREE)
                        abd_free(cabd);
        }
        list_destroy(&ABD_GANG(abd).abd_gang_chain);
}

static void
abd_free_scatter(abd_t *abd)
{
        abd_free_chunks(abd);
        abd_update_scatter_stats(abd, ABDSTAT_DECR);
}

/*
 * Free an ABD.  Use with any kind of abd: those created with abd_alloc_*()
 * and abd_get_*(), including abd_get_offset_struct().
 *
 * If the ABD was created with abd_alloc_*(), the underlying data
 * (scatterlist or linear buffer) will also be freed.  (Subject to ownership
 * changes via abd_*_ownership_of_buf().)
 *
 * Unless the ABD was created with abd_get_offset_struct(), the abd_t will
 * also be freed.
 */
void
abd_free(abd_t *abd)
{
        if (abd == NULL)
                return;

        abd_verify(abd);
#ifdef ZFS_DEBUG
        IMPLY(abd->abd_flags & ABD_FLAG_OWNER, abd->abd_parent == NULL);
#endif

        if (abd_is_gang(abd)) {
                abd_free_gang(abd);
        } else if (abd_is_linear(abd)) {
                if (abd->abd_flags & ABD_FLAG_OWNER)
                        abd_free_linear(abd);
        } else {
                if (abd->abd_flags & ABD_FLAG_OWNER)
                        abd_free_scatter(abd);
        }

#ifdef ZFS_DEBUG
        if (abd->abd_parent != NULL) {
                (void) zfs_refcount_remove_many(&abd->abd_parent->abd_children,
                    abd->abd_size, abd);
        }
#endif

        abd_fini_struct(abd);
        if (abd->abd_flags & ABD_FLAG_ALLOCD)
                abd_free_struct_impl(abd);
}

/*
 * Allocate an ABD of the same format (same metadata flag, same scatterize
 * setting) as another ABD.
 */
abd_t *
abd_alloc_sametype(abd_t *sabd, size_t size)
{
        boolean_t is_metadata = (sabd->abd_flags & ABD_FLAG_META) != 0;
        if (abd_is_linear(sabd) &&
            !abd_is_linear_page(sabd)) {
                return (abd_alloc_linear(size, is_metadata));
        } else {
                return (abd_alloc(size, is_metadata));
        }
}

/*
 * Create gang ABD that will be the head of a list of ABD's. This is used
 * to "chain" scatter/gather lists together when constructing aggregated
 * IO's. To free this abd, abd_free() must be called.
 */
abd_t *
abd_alloc_gang(void)
{
        abd_t *abd = abd_alloc_struct(0);
        abd->abd_flags |= ABD_FLAG_GANG | ABD_FLAG_OWNER;
        list_create(&ABD_GANG(abd).abd_gang_chain,
            sizeof (abd_t), offsetof(abd_t, abd_gang_link));
        return (abd);
}

/*
 * Add a child gang ABD to a parent gang ABDs chained list.
 */
static void
abd_gang_add_gang(abd_t *pabd, abd_t *cabd, boolean_t free_on_free)
{
        ASSERT(abd_is_gang(pabd));
        ASSERT(abd_is_gang(cabd));

        if (free_on_free) {
                /*
                 * If the parent is responsible for freeing the child gang
                 * ABD we will just splice the child's children ABD list to
                 * the parent's list and immediately free the child gang ABD
                 * struct. The parent gang ABDs children from the child gang
                 * will retain all the free_on_free settings after being
                 * added to the parents list.
                 */
#ifdef ZFS_DEBUG
                /*
                 * If cabd had abd_parent, we have to drop it here.  We can't
                 * transfer it to pabd, nor we can clear abd_size leaving it.
                 */
                if (cabd->abd_parent != NULL) {
                        (void) zfs_refcount_remove_many(
                            &cabd->abd_parent->abd_children,
                            cabd->abd_size, cabd);
                        cabd->abd_parent = NULL;
                }
#endif
                pabd->abd_size += cabd->abd_size;
                cabd->abd_size = 0;
                list_move_tail(&ABD_GANG(pabd).abd_gang_chain,
                    &ABD_GANG(cabd).abd_gang_chain);
                ASSERT(list_is_empty(&ABD_GANG(cabd).abd_gang_chain));
                abd_verify(pabd);
                abd_free(cabd);
        } else {
                for (abd_t *child = list_head(&ABD_GANG(cabd).abd_gang_chain);
                    child != NULL;
                    child = list_next(&ABD_GANG(cabd).abd_gang_chain, child)) {
                        /*
                         * We always pass B_FALSE for free_on_free as it is the
                         * original child gang ABDs responsibility to determine
                         * if any of its child ABDs should be free'd on the call
                         * to abd_free().
                         */
                        abd_gang_add(pabd, child, B_FALSE);
                }
                abd_verify(pabd);
        }
}

/*
 * Add a child ABD to a gang ABD's chained list.
 */
void
abd_gang_add(abd_t *pabd, abd_t *cabd, boolean_t free_on_free)
{
        ASSERT(abd_is_gang(pabd));
        abd_t *child_abd = NULL;

        /*
         * If the child being added is a gang ABD, we will add the
         * child's ABDs to the parent gang ABD. This allows us to account
         * for the offset correctly in the parent gang ABD.
         */
        if (abd_is_gang(cabd)) {
                ASSERT(!list_link_active(&cabd->abd_gang_link));
                return (abd_gang_add_gang(pabd, cabd, free_on_free));
        }
        ASSERT(!abd_is_gang(cabd));

        /*
         * In order to verify that an ABD is not already part of
         * another gang ABD, we must lock the child ABD's abd_mtx
         * to check its abd_gang_link status. We unlock the abd_mtx
         * only after it is has been added to a gang ABD, which
         * will update the abd_gang_link's status. See comment below
         * for how an ABD can be in multiple gang ABD's simultaneously.
         */
        mutex_enter(&cabd->abd_mtx);
        if (list_link_active(&cabd->abd_gang_link)) {
                /*
                 * If the child ABD is already part of another
                 * gang ABD then we must allocate a new
                 * ABD to use a separate link. We mark the newly
                 * allocated ABD with ABD_FLAG_GANG_FREE, before
                 * adding it to the gang ABD's list, to make the
                 * gang ABD aware that it is responsible to call
                 * abd_free(). We use abd_get_offset() in order
                 * to just allocate a new ABD but avoid copying the
                 * data over into the newly allocated ABD.
                 *
                 * An ABD may become part of multiple gang ABD's. For
                 * example, when writing ditto bocks, the same ABD
                 * is used to write 2 or 3 locations with 2 or 3
                 * zio_t's. Each of the zio's may be aggregated with
                 * different adjacent zio's. zio aggregation uses gang
                 * zio's, so the single ABD can become part of multiple
                 * gang zio's.
                 *
                 * The ASSERT below is to make sure that if
                 * free_on_free is passed as B_TRUE, the ABD can
                 * not be in multiple gang ABD's. The gang ABD
                 * can not be responsible for cleaning up the child
                 * ABD memory allocation if the ABD can be in
                 * multiple gang ABD's at one time.
                 */
                ASSERT3B(free_on_free, ==, B_FALSE);
                child_abd = abd_get_offset(cabd, 0);
                child_abd->abd_flags |= ABD_FLAG_GANG_FREE;
        } else {
                child_abd = cabd;
                if (free_on_free)
                        child_abd->abd_flags |= ABD_FLAG_GANG_FREE;
        }
        ASSERT3P(child_abd, !=, NULL);

        list_insert_tail(&ABD_GANG(pabd).abd_gang_chain, child_abd);
        mutex_exit(&cabd->abd_mtx);
        pabd->abd_size += child_abd->abd_size;
}

/*
 * Locate the ABD for the supplied offset in the gang ABD.
 * Return a new offset relative to the returned ABD.
 */
abd_t *
abd_gang_get_offset(abd_t *abd, size_t *off)
{
        abd_t *cabd;

        ASSERT(abd_is_gang(abd));
        ASSERT3U(*off, <, abd->abd_size);
        for (cabd = list_head(&ABD_GANG(abd).abd_gang_chain); cabd != NULL;
            cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
                if (*off >= cabd->abd_size)
                        *off -= cabd->abd_size;
                else
                        return (cabd);
        }
        VERIFY3P(cabd, !=, NULL);
        return (cabd);
}

/*
 * Allocate a new ABD, using the provided struct (if non-NULL, and if
 * circumstances allow - otherwise allocate the struct).  The returned ABD will
 * point to offset off of sabd. It shares the underlying buffer data with sabd.
 * Use abd_free() to free.  sabd must not be freed while any derived ABDs exist.
 */
static abd_t *
abd_get_offset_impl(abd_t *abd, abd_t *sabd, size_t off, size_t size)
{
        abd_verify(sabd);
        ASSERT3U(off + size, <=, sabd->abd_size);

        if (abd_is_linear(sabd)) {
                if (abd == NULL)
                        abd = abd_alloc_struct(0);
                /*
                 * Even if this buf is filesystem metadata, we only track that
                 * if we own the underlying data buffer, which is not true in
                 * this case. Therefore, we don't ever use ABD_FLAG_META here.
                 */
                abd->abd_flags |= ABD_FLAG_LINEAR;

                /*
                 * User pages from Direct I/O requests may be in a single page
                 * (ABD_FLAG_LINEAR_PAGE), and we must make sure to still flag
                 * that here for abd. This is required because we have to be
                 * careful when borrowing the buffer from the ABD because we
                 * can not place user pages under write protection on Linux.
                 * See the comments in abd_os.c for abd_borrow_buf(),
                 * abd_borrow_buf_copy(), abd_return_buf() and
                 * abd_return_buf_copy().
                 */
                if (abd_is_from_pages(sabd)) {
                        abd->abd_flags |= ABD_FLAG_FROM_PAGES |
                            ABD_FLAG_LINEAR_PAGE;
                }

                ABD_LINEAR_BUF(abd) = (char *)ABD_LINEAR_BUF(sabd) + off;
        } else if (abd_is_gang(sabd)) {
                size_t left = size;
                if (abd == NULL) {
                        abd = abd_alloc_gang();
                } else {
                        abd->abd_flags |= ABD_FLAG_GANG;
                        list_create(&ABD_GANG(abd).abd_gang_chain,
                            sizeof (abd_t), offsetof(abd_t, abd_gang_link));
                }

                abd->abd_flags &= ~ABD_FLAG_OWNER;
                for (abd_t *cabd = abd_gang_get_offset(sabd, &off);
                    cabd != NULL && left > 0;
                    cabd = list_next(&ABD_GANG(sabd).abd_gang_chain, cabd)) {
                        int csize = MIN(left, cabd->abd_size - off);

                        abd_t *nabd = abd_get_offset_size(cabd, off, csize);
                        abd_gang_add(abd, nabd, B_TRUE);
                        left -= csize;
                        off = 0;
                }
                ASSERT3U(left, ==, 0);
        } else {
                abd = abd_get_offset_scatter(abd, sabd, off, size);
        }

        ASSERT3P(abd, !=, NULL);
        abd->abd_size = size;
#ifdef ZFS_DEBUG
        abd->abd_parent = sabd;
        (void) zfs_refcount_add_many(&sabd->abd_children, abd->abd_size, abd);
#endif
        return (abd);
}

/*
 * Like abd_get_offset_size(), but memory for the abd_t is provided by the
 * caller.  Using this routine can improve performance by avoiding the cost
 * of allocating memory for the abd_t struct, and updating the abd stats.
 * Usually, the provided abd is returned, but in some circumstances (FreeBSD,
 * if sabd is scatter and size is more than 2 pages) a new abd_t may need to
 * be allocated.  Therefore callers should be careful to use the returned
 * abd_t*.
 */
abd_t *
abd_get_offset_struct(abd_t *abd, abd_t *sabd, size_t off, size_t size)
{
        abd_t *result;
        abd_init_struct(abd);
        result = abd_get_offset_impl(abd, sabd, off, size);
        if (result != abd)
                abd_fini_struct(abd);
        return (result);
}

abd_t *
abd_get_offset(abd_t *sabd, size_t off)
{
        size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0;
        VERIFY3U(size, >, 0);
        return (abd_get_offset_impl(NULL, sabd, off, size));
}

abd_t *
abd_get_offset_size(abd_t *sabd, size_t off, size_t size)
{
        ASSERT3U(off + size, <=, sabd->abd_size);
        return (abd_get_offset_impl(NULL, sabd, off, size));
}

/*
 * Return a size scatter ABD containing only zeros.
 */
abd_t *
abd_get_zeros(size_t size)
{
        ASSERT3P(abd_zero_scatter, !=, NULL);
        ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
        return (abd_get_offset_size(abd_zero_scatter, 0, size));
}

/*
 * Create a linear ABD for an existing buf.
 */
static abd_t *
abd_get_from_buf_impl(abd_t *abd, void *buf, size_t size)
{
        VERIFY3U(size, <=, SPA_MAXBLOCKSIZE);

        /*
         * Even if this buf is filesystem metadata, we only track that if we
         * own the underlying data buffer, which is not true in this case.
         * Therefore, we don't ever use ABD_FLAG_META here.
         */
        abd->abd_flags |= ABD_FLAG_LINEAR;
        abd->abd_size = size;

        ABD_LINEAR_BUF(abd) = buf;

        return (abd);
}

abd_t *
abd_get_from_buf(void *buf, size_t size)
{
        abd_t *abd = abd_alloc_struct(0);
        return (abd_get_from_buf_impl(abd, buf, size));
}

abd_t *
abd_get_from_buf_struct(abd_t *abd, void *buf, size_t size)
{
        abd_init_struct(abd);
        return (abd_get_from_buf_impl(abd, buf, size));
}

/*
 * Get the raw buffer associated with a linear ABD.
 */
void *
abd_to_buf(abd_t *abd)
{
        ASSERT(abd_is_linear(abd));
        abd_verify(abd);
        return (ABD_LINEAR_BUF(abd));
}

void
abd_release_ownership_of_buf(abd_t *abd)
{
        ASSERT(abd_is_linear(abd));
        ASSERT(abd->abd_flags & ABD_FLAG_OWNER);

        /*
         * abd_free() needs to handle LINEAR_PAGE ABD's specially.
         * Since that flag does not survive the
         * abd_release_ownership_of_buf() -> abd_get_from_buf() ->
         * abd_take_ownership_of_buf() sequence, we don't allow releasing
         * these "linear but not zio_[data_]buf_alloc()'ed" ABD's.
         */
        ASSERT(!abd_is_linear_page(abd));

        abd_verify(abd);

        abd->abd_flags &= ~ABD_FLAG_OWNER;
        /* Disable this flag since we no longer own the data buffer */
        abd->abd_flags &= ~ABD_FLAG_META;

        abd_update_linear_stats(abd, ABDSTAT_DECR);
}


/*
 * Give this ABD ownership of the buffer that it's storing. Can only be used on
 * linear ABDs which were allocated via abd_get_from_buf(), or ones allocated
 * with abd_alloc_linear() which subsequently released ownership of their buf
 * with abd_release_ownership_of_buf().
 */
void
abd_take_ownership_of_buf(abd_t *abd, boolean_t is_metadata)
{
        ASSERT(abd_is_linear(abd));
        ASSERT(!(abd->abd_flags & ABD_FLAG_OWNER));
        abd_verify(abd);

        abd->abd_flags |= ABD_FLAG_OWNER;
        if (is_metadata) {
                abd->abd_flags |= ABD_FLAG_META;
        }

        abd_update_linear_stats(abd, ABDSTAT_INCR);
}

/*
 * Initializes an abd_iter based on whether the abd is a gang ABD
 * or just a single ABD.
 */
static inline abd_t *
abd_init_abd_iter(abd_t *abd, struct abd_iter *aiter, size_t off)
{
        abd_t *cabd = NULL;

        if (abd_is_gang(abd)) {
                cabd = abd_gang_get_offset(abd, &off);
                if (cabd) {
                        abd_iter_init(aiter, cabd);
                        abd_iter_advance(aiter, off);
                }
        } else {
                abd_iter_init(aiter, abd);
                abd_iter_advance(aiter, off);
        }
        return (cabd);
}

/*
 * Advances an abd_iter. We have to be careful with gang ABD as
 * advancing could mean that we are at the end of a particular ABD and
 * must grab the ABD in the gang ABD's list.
 */
static inline abd_t *
abd_advance_abd_iter(abd_t *abd, abd_t *cabd, struct abd_iter *aiter,
    size_t len)
{
        abd_iter_advance(aiter, len);
        if (abd_is_gang(abd) && abd_iter_at_end(aiter)) {
                ASSERT3P(cabd, !=, NULL);
                cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd);
                if (cabd) {
                        abd_iter_init(aiter, cabd);
                        abd_iter_advance(aiter, 0);
                }
        }
        return (cabd);
}

int
abd_iterate_func(abd_t *abd, size_t off, size_t size,
    abd_iter_func_t *func, void *private)
{
        struct abd_iter aiter;
        int ret = 0;

        if (size == 0)
                return (0);

        abd_verify(abd);
        ASSERT3U(off + size, <=, abd->abd_size);

        abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);

        while (size > 0) {
                IMPLY(abd_is_gang(abd), c_abd != NULL);

                abd_iter_map(&aiter);

                size_t len = MIN(aiter.iter_mapsize, size);
                ASSERT3U(len, >, 0);

                ret = func(aiter.iter_mapaddr, len, private);

                abd_iter_unmap(&aiter);

                if (ret != 0)
                        break;

                size -= len;
                c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
        }

        return (ret);
}

#if defined(__linux__) && defined(_KERNEL)
int
abd_iterate_page_func(abd_t *abd, size_t off, size_t size,
    abd_iter_page_func_t *func, void *private)
{
        struct abd_iter aiter;
        int ret = 0;

        if (size == 0)
                return (0);

        abd_verify(abd);
        ASSERT3U(off + size, <=, abd->abd_size);

        abd_t *c_abd = abd_init_abd_iter(abd, &aiter, off);

        while (size > 0) {
                IMPLY(abd_is_gang(abd), c_abd != NULL);

                abd_iter_page(&aiter);

                size_t len = MIN(aiter.iter_page_dsize, size);
                ASSERT3U(len, >, 0);

                ret = func(aiter.iter_page, aiter.iter_page_doff,
                    len, private);

                aiter.iter_page = NULL;
                aiter.iter_page_doff = 0;
                aiter.iter_page_dsize = 0;

                if (ret != 0)
                        break;

                size -= len;
                c_abd = abd_advance_abd_iter(abd, c_abd, &aiter, len);
        }

        return (ret);
}
#endif

struct buf_arg {
        void *arg_buf;
};

static int
abd_copy_to_buf_off_cb(void *buf, size_t size, void *private)
{
        struct buf_arg *ba_ptr = private;

        (void) memcpy(ba_ptr->arg_buf, buf, size);
        ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;

        return (0);
}

/*
 * Copy abd to buf. (off is the offset in abd.)
 */
void
abd_copy_to_buf_off(void *buf, abd_t *abd, size_t off, size_t size)
{
        struct buf_arg ba_ptr = { buf };

        (void) abd_iterate_func(abd, off, size, abd_copy_to_buf_off_cb,
            &ba_ptr);
}

static int
abd_cmp_buf_off_cb(void *buf, size_t size, void *private)
{
        int ret;
        struct buf_arg *ba_ptr = private;

        ret = memcmp(buf, ba_ptr->arg_buf, size);
        ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;

        return (ret);
}

/*
 * Compare the contents of abd to buf. (off is the offset in abd.)
 */
int
abd_cmp_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
{
        struct buf_arg ba_ptr = { (void *) buf };

        return (abd_iterate_func(abd, off, size, abd_cmp_buf_off_cb, &ba_ptr));
}

static int
abd_copy_from_buf_off_cb(void *buf, size_t size, void *private)
{
        struct buf_arg *ba_ptr = private;

        (void) memcpy(buf, ba_ptr->arg_buf, size);
        ba_ptr->arg_buf = (char *)ba_ptr->arg_buf + size;

        return (0);
}

/*
 * Copy from buf to abd. (off is the offset in abd.)
 */
void
abd_copy_from_buf_off(abd_t *abd, const void *buf, size_t off, size_t size)
{
        struct buf_arg ba_ptr = { (void *) buf };

        (void) abd_iterate_func(abd, off, size, abd_copy_from_buf_off_cb,
            &ba_ptr);
}

static int
abd_zero_off_cb(void *buf, size_t size, void *private)
{
        (void) private;
        (void) memset(buf, 0, size);
        return (0);
}

/*
 * Zero out the abd from a particular offset to the end.
 */
void
abd_zero_off(abd_t *abd, size_t off, size_t size)
{
        (void) abd_iterate_func(abd, off, size, abd_zero_off_cb, NULL);
}

/*
 * Iterate over two ABDs and call func incrementally on the two ABDs' data in
 * equal-sized chunks (passed to func as raw buffers). func could be called many
 * times during this iteration.
 */
int
abd_iterate_func2(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff,
    size_t size, abd_iter_func2_t *func, void *private)
{
        int ret = 0;
        struct abd_iter daiter, saiter;
        abd_t *c_dabd, *c_sabd;

        if (size == 0)
                return (0);

        abd_verify(dabd);
        abd_verify(sabd);

        ASSERT3U(doff + size, <=, dabd->abd_size);
        ASSERT3U(soff + size, <=, sabd->abd_size);

        c_dabd = abd_init_abd_iter(dabd, &daiter, doff);
        c_sabd = abd_init_abd_iter(sabd, &saiter, soff);

        while (size > 0) {
                IMPLY(abd_is_gang(dabd), c_dabd != NULL);
                IMPLY(abd_is_gang(sabd), c_sabd != NULL);

                abd_iter_map(&daiter);
                abd_iter_map(&saiter);

                size_t dlen = MIN(daiter.iter_mapsize, size);
                size_t slen = MIN(saiter.iter_mapsize, size);
                size_t len = MIN(dlen, slen);
                ASSERT(dlen > 0 || slen > 0);

                ret = func(daiter.iter_mapaddr, saiter.iter_mapaddr, len,
                    private);

                abd_iter_unmap(&saiter);
                abd_iter_unmap(&daiter);

                if (ret != 0)
                        break;

                size -= len;
                c_dabd =
                    abd_advance_abd_iter(dabd, c_dabd, &daiter, len);
                c_sabd =
                    abd_advance_abd_iter(sabd, c_sabd, &saiter, len);
        }

        return (ret);
}

static int
abd_copy_off_cb(void *dbuf, void *sbuf, size_t size, void *private)
{
        (void) private;
        (void) memcpy(dbuf, sbuf, size);
        return (0);
}

/*
 * Copy from sabd to dabd starting from soff and doff.
 */
void
abd_copy_off(abd_t *dabd, abd_t *sabd, size_t doff, size_t soff, size_t size)
{
        (void) abd_iterate_func2(dabd, sabd, doff, soff, size,
            abd_copy_off_cb, NULL);
}

static int
abd_cmp_cb(void *bufa, void *bufb, size_t size, void *private)
{
        (void) private;
        return (memcmp(bufa, bufb, size));
}

/*
 * Compares the contents of two ABDs.
 */
int
abd_cmp(abd_t *dabd, abd_t *sabd)
{
        ASSERT3U(dabd->abd_size, ==, sabd->abd_size);
        return (abd_iterate_func2(dabd, sabd, 0, 0, dabd->abd_size,
            abd_cmp_cb, NULL));
}

/*
 * Check if ABD content is all-zeroes.
 */
static int
abd_cmp_zero_off_cb(void *data, size_t len, void *private)
{
        (void) private;

        /* This function can only check whole uint64s. Enforce that. */
        ASSERT0(P2PHASE(len, 8));

        uint64_t *end = (uint64_t *)((char *)data + len);
        for (uint64_t *word = (uint64_t *)data; word < end; word++)
                if (*word != 0)
                        return (1);

        return (0);
}

int
abd_cmp_zero_off(abd_t *abd, size_t off, size_t size)
{
        return (abd_iterate_func(abd, off, size, abd_cmp_zero_off_cb, NULL));
}

/*
 * Iterate over code ABDs and a data ABD and call @func_raidz_gen.
 *
 * @cabds          parity ABDs, must have equal size
 * @dabd           data ABD. Can be NULL (in this case @dsize = 0)
 * @func_raidz_gen should be implemented so that its behaviour
 *                 is the same when taking linear and when taking scatter
 */
void
abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd, size_t off,
    size_t csize, size_t dsize, const unsigned parity,
    void (*func_raidz_gen)(void **, const void *, size_t, size_t))
{
        int i;
        size_t len, dlen;
        struct abd_iter caiters[3];
        struct abd_iter daiter;
        void *caddrs[3], *daddr;
        unsigned long flags __maybe_unused = 0;
        abd_t *c_cabds[3];
        abd_t *c_dabd = NULL;

        ASSERT3U(parity, <=, 3);
        for (i = 0; i < parity; i++) {
                abd_verify(cabds[i]);
                ASSERT3U(off + csize, <=, cabds[i]->abd_size);
                c_cabds[i] = abd_init_abd_iter(cabds[i], &caiters[i], off);
        }

        if (dsize > 0) {
                ASSERT(dabd);
                abd_verify(dabd);
                ASSERT3U(off + dsize, <=, dabd->abd_size);
                c_dabd = abd_init_abd_iter(dabd, &daiter, off);
        }

        abd_enter_critical(flags);
        while (csize > 0) {
                len = csize;
                for (i = 0; i < parity; i++) {
                        IMPLY(abd_is_gang(cabds[i]), c_cabds[i] != NULL);
                        abd_iter_map(&caiters[i]);
                        caddrs[i] = caiters[i].iter_mapaddr;
                        len = MIN(caiters[i].iter_mapsize, len);
                }

                if (dsize > 0) {
                        IMPLY(abd_is_gang(dabd), c_dabd != NULL);
                        abd_iter_map(&daiter);
                        daddr = daiter.iter_mapaddr;
                        len = MIN(daiter.iter_mapsize, len);
                        dlen = len;
                } else {
                        daddr = NULL;
                        dlen = 0;
                }

                /* must be progressive */
                ASSERT3U(len, >, 0);
                /*
                 * The iterated function likely will not do well if each
                 * segment except the last one is not multiple of 512 (raidz).
                 */
                ASSERT3U(((uint64_t)len & 511ULL), ==, 0);

                func_raidz_gen(caddrs, daddr, len, dlen);

                for (i = parity-1; i >= 0; i--) {
                        abd_iter_unmap(&caiters[i]);
                        c_cabds[i] =
                            abd_advance_abd_iter(cabds[i], c_cabds[i],
                            &caiters[i], len);
                }

                if (dsize > 0) {
                        abd_iter_unmap(&daiter);
                        c_dabd =
                            abd_advance_abd_iter(dabd, c_dabd, &daiter,
                            dlen);
                        dsize -= dlen;
                }

                csize -= len;
        }
        abd_exit_critical(flags);
}

/*
 * Iterate over code ABDs and data reconstruction target ABDs and call
 * @func_raidz_rec. Function maps at most 6 pages atomically.
 *
 * @cabds           parity ABDs, must have equal size
 * @tabds           rec target ABDs, at most 3
 * @tsize           size of data target columns
 * @func_raidz_rec  expects syndrome data in target columns. Function
 *                  reconstructs data and overwrites target columns.
 */
void
abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
    size_t tsize, const unsigned parity,
    void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
    const unsigned *mul),
    const unsigned *mul)
{
        int i;
        size_t len;
        struct abd_iter citers[3];
        struct abd_iter xiters[3];
        void *caddrs[3], *xaddrs[3];
        unsigned long flags __maybe_unused = 0;
        abd_t *c_cabds[3];
        abd_t *c_tabds[3];

        ASSERT3U(parity, <=, 3);

        for (i = 0; i < parity; i++) {
                abd_verify(cabds[i]);
                abd_verify(tabds[i]);
                ASSERT3U(tsize, <=, cabds[i]->abd_size);
                ASSERT3U(tsize, <=, tabds[i]->abd_size);
                c_cabds[i] =
                    abd_init_abd_iter(cabds[i], &citers[i], 0);
                c_tabds[i] =
                    abd_init_abd_iter(tabds[i], &xiters[i], 0);
        }

        abd_enter_critical(flags);
        while (tsize > 0) {
                len = tsize;
                for (i = 0; i < parity; i++) {
                        IMPLY(abd_is_gang(cabds[i]), c_cabds[i] != NULL);
                        IMPLY(abd_is_gang(tabds[i]), c_tabds[i] != NULL);
                        abd_iter_map(&citers[i]);
                        abd_iter_map(&xiters[i]);
                        caddrs[i] = citers[i].iter_mapaddr;
                        xaddrs[i] = xiters[i].iter_mapaddr;
                        len = MIN(citers[i].iter_mapsize, len);
                        len = MIN(xiters[i].iter_mapsize, len);
                }

                /* must be progressive */
                ASSERT3S(len, >, 0);
                /*
                 * The iterated function likely will not do well if each
                 * segment except the last one is not multiple of 512 (raidz).
                 */
                ASSERT3U(((uint64_t)len & 511ULL), ==, 0);

                func_raidz_rec(xaddrs, len, caddrs, mul);

                for (i = parity-1; i >= 0; i--) {
                        abd_iter_unmap(&xiters[i]);
                        abd_iter_unmap(&citers[i]);
                        c_tabds[i] =
                            abd_advance_abd_iter(tabds[i], c_tabds[i],
                            &xiters[i], len);
                        c_cabds[i] =
                            abd_advance_abd_iter(cabds[i], c_cabds[i],
                            &citers[i], len);
                }

                tsize -= len;
                ASSERT3S(tsize, >=, 0);
        }
        abd_exit_critical(flags);
}