Linux 6.12 compat: META

[zfs.git] / module / zfs / vdev_raidz_math_impl.h

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or https://opensource.org/licenses/CDDL-1.0.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Gvozden Nešković. All rights reserved.
 */

#ifndef _VDEV_RAIDZ_MATH_IMPL_H
#define _VDEV_RAIDZ_MATH_IMPL_H

#include <sys/types.h>
#include <sys/vdev_raidz_impl.h>

#define raidz_inline inline __attribute__((always_inline))
#ifndef noinline
#define noinline __attribute__((noinline))
#endif

/*
 * Functions calculate multiplication constants for data reconstruction.
 * Coefficients depend on RAIDZ geometry, indexes of failed child vdevs, and
 * used parity columns for reconstruction.
 * @rr                  RAIDZ row
 * @tgtidx              array of missing data indexes
 * @coeff               output array of coefficients. Array must be provided by
 *                      user and must hold minimum MUL_CNT values.
 */
static noinline void
raidz_rec_q_coeff(const raidz_row_t *rr, const int *tgtidx, unsigned *coeff)
{
        const unsigned ncols = rr->rr_cols;
        const unsigned x = tgtidx[TARGET_X];

        coeff[MUL_Q_X] = gf_exp2(255 - (ncols - x - 1));
}

static noinline void
raidz_rec_r_coeff(const raidz_row_t *rr, const int *tgtidx, unsigned *coeff)
{
        const unsigned ncols = rr->rr_cols;
        const unsigned x = tgtidx[TARGET_X];

        coeff[MUL_R_X] = gf_exp4(255 - (ncols - x - 1));
}

static noinline void
raidz_rec_pq_coeff(const raidz_row_t *rr, const int *tgtidx, unsigned *coeff)
{
        const unsigned ncols = rr->rr_cols;
        const unsigned x = tgtidx[TARGET_X];
        const unsigned y = tgtidx[TARGET_Y];
        gf_t a, b, e;

        a = gf_exp2(x + 255 - y);
        b = gf_exp2(255 - (ncols - x - 1));
        e = a ^ 0x01;

        coeff[MUL_PQ_X] = gf_div(a, e);
        coeff[MUL_PQ_Y] = gf_div(b, e);
}

static noinline void
raidz_rec_pr_coeff(const raidz_row_t *rr, const int *tgtidx, unsigned *coeff)
{
        const unsigned ncols = rr->rr_cols;
        const unsigned x = tgtidx[TARGET_X];
        const unsigned y = tgtidx[TARGET_Y];

        gf_t a, b, e;

        a = gf_exp4(x + 255 - y);
        b = gf_exp4(255 - (ncols - x - 1));
        e = a ^ 0x01;

        coeff[MUL_PR_X] = gf_div(a, e);
        coeff[MUL_PR_Y] = gf_div(b, e);
}

static noinline void
raidz_rec_qr_coeff(const raidz_row_t *rr, const int *tgtidx, unsigned *coeff)
{
        const unsigned ncols = rr->rr_cols;
        const unsigned x = tgtidx[TARGET_X];
        const unsigned y = tgtidx[TARGET_Y];

        gf_t nx, ny, nxxy, nxyy, d;

        nx = gf_exp2(ncols - x - 1);
        ny = gf_exp2(ncols - y - 1);
        nxxy = gf_mul(gf_mul(nx, nx), ny);
        nxyy = gf_mul(gf_mul(nx, ny), ny);
        d = nxxy ^ nxyy;

        coeff[MUL_QR_XQ] = ny;
        coeff[MUL_QR_X] = gf_div(ny, d);
        coeff[MUL_QR_YQ] = nx;
        coeff[MUL_QR_Y] = gf_div(nx, d);
}

static noinline void
raidz_rec_pqr_coeff(const raidz_row_t *rr, const int *tgtidx, unsigned *coeff)
{
        const unsigned ncols = rr->rr_cols;
        const unsigned x = tgtidx[TARGET_X];
        const unsigned y = tgtidx[TARGET_Y];
        const unsigned z = tgtidx[TARGET_Z];

        gf_t nx, ny, nz, nxx, nyy, nzz, nyyz, nyzz, xd, yd;

        nx = gf_exp2(ncols - x - 1);
        ny = gf_exp2(ncols - y - 1);
        nz = gf_exp2(ncols - z - 1);

        nxx = gf_exp4(ncols - x - 1);
        nyy = gf_exp4(ncols - y - 1);
        nzz = gf_exp4(ncols - z - 1);

        nyyz = gf_mul(gf_mul(ny, nz), ny);
        nyzz = gf_mul(nzz, ny);

        xd = gf_mul(nxx, ny) ^ gf_mul(nx, nyy) ^ nyyz ^
            gf_mul(nxx, nz) ^ gf_mul(nzz, nx) ^  nyzz;

        yd = gf_inv(ny ^ nz);

        coeff[MUL_PQR_XP] = gf_div(nyyz ^ nyzz, xd);
        coeff[MUL_PQR_XQ] = gf_div(nyy ^ nzz, xd);
        coeff[MUL_PQR_XR] = gf_div(ny ^ nz, xd);
        coeff[MUL_PQR_YU] = nx;
        coeff[MUL_PQR_YP] = gf_mul(nz, yd);
        coeff[MUL_PQR_YQ] = yd;
}

/*
 * Method for zeroing a buffer (can be implemented using SIMD).
 * This method is used by multiple for gen/rec functions.
 *
 * @dc          Destination buffer
 * @dsize       Destination buffer size
 * @private     Unused
 */
static int
raidz_zero_abd_cb(void *dc, size_t dsize, void *private)
{
        v_t *dst = (v_t *)dc;
        size_t i;

        ZERO_DEFINE();

        (void) private; /* unused */

        ZERO(ZERO_D);

        for (i = 0; i < dsize / sizeof (v_t); i += (2 * ZERO_STRIDE)) {
                STORE(dst + i, ZERO_D);
                STORE(dst + i + ZERO_STRIDE, ZERO_D);
        }

        return (0);
}

#define raidz_zero(dabd, size)                                          \
{                                                                       \
        abd_iterate_func(dabd, 0, size, raidz_zero_abd_cb, NULL);       \
}

/*
 * Method for copying two buffers (can be implemented using SIMD).
 * This method is used by multiple for gen/rec functions.
 *
 * @dc          Destination buffer
 * @sc          Source buffer
 * @dsize       Destination buffer size
 * @ssize       Source buffer size
 * @private     Unused
 */
static int
raidz_copy_abd_cb(void *dc, void *sc, size_t size, void *private)
{
        v_t *dst = (v_t *)dc;
        const v_t *src = (v_t *)sc;
        size_t i;

        COPY_DEFINE();

        (void) private; /* unused */

        for (i = 0; i < size / sizeof (v_t); i += (2 * COPY_STRIDE)) {
                LOAD(src + i, COPY_D);
                STORE(dst + i, COPY_D);

                LOAD(src + i + COPY_STRIDE, COPY_D);
                STORE(dst + i + COPY_STRIDE, COPY_D);
        }

        return (0);
}


#define raidz_copy(dabd, sabd, off, size)                               \
{                                                                       \
        abd_iterate_func2(dabd, sabd, off, off, size, raidz_copy_abd_cb, \
            NULL);                                                      \
}

/*
 * Method for adding (XORing) two buffers.
 * Source and destination are XORed together and result is stored in
 * destination buffer. This method is used by multiple for gen/rec functions.
 *
 * @dc          Destination buffer
 * @sc          Source buffer
 * @dsize       Destination buffer size
 * @ssize       Source buffer size
 * @private     Unused
 */
static int
raidz_add_abd_cb(void *dc, void *sc, size_t size, void *private)
{
        v_t *dst = (v_t *)dc;
        const v_t *src = (v_t *)sc;
        size_t i;

        ADD_DEFINE();

        (void) private; /* unused */

        for (i = 0; i < size / sizeof (v_t); i += (2 * ADD_STRIDE)) {
                LOAD(dst + i, ADD_D);
                XOR_ACC(src + i, ADD_D);
                STORE(dst + i, ADD_D);

                LOAD(dst + i + ADD_STRIDE, ADD_D);
                XOR_ACC(src + i + ADD_STRIDE, ADD_D);
                STORE(dst + i + ADD_STRIDE, ADD_D);
        }

        return (0);
}

#define raidz_add(dabd, sabd, off, size)                                \
{                                                                       \
        abd_iterate_func2(dabd, sabd, off, off, size, raidz_add_abd_cb, \
            NULL);                                                      \
}

/*
 * Method for multiplying a buffer with a constant in GF(2^8).
 * Symbols from buffer are multiplied by a constant and result is stored
 * back in the same buffer.
 *
 * @dc          In/Out data buffer.
 * @size        Size of the buffer
 * @private     pointer to the multiplication constant (unsigned)
 */
static int
raidz_mul_abd_cb(void *dc, size_t size, void *private)
{
        const unsigned mul = *((unsigned *)private);
        v_t *d = (v_t *)dc;
        size_t i;

        MUL_DEFINE();

        for (i = 0; i < size / sizeof (v_t); i += (2 * MUL_STRIDE)) {
                LOAD(d + i, MUL_D);
                MUL(mul, MUL_D);
                STORE(d + i, MUL_D);

                LOAD(d + i + MUL_STRIDE, MUL_D);
                MUL(mul, MUL_D);
                STORE(d + i + MUL_STRIDE, MUL_D);
        }

        return (0);
}


/*
 * Syndrome generation/update macros
 *
 * Require LOAD(), XOR(), STORE(), MUL2(), and MUL4() macros
 */
#define P_D_SYNDROME(D, T, t)           \
{                                       \
        LOAD((t), T);                   \
        XOR(D, T);                      \
        STORE((t), T);                  \
}

#define Q_D_SYNDROME(D, T, t)           \
{                                       \
        LOAD((t), T);                   \
        MUL2(T);                        \
        XOR(D, T);                      \
        STORE((t), T);                  \
}

#define Q_SYNDROME(T, t)                \
{                                       \
        LOAD((t), T);                   \
        MUL2(T);                        \
        STORE((t), T);                  \
}

#define R_D_SYNDROME(D, T, t)           \
{                                       \
        LOAD((t), T);                   \
        MUL4(T);                        \
        XOR(D, T);                      \
        STORE((t), T);                  \
}

#define R_SYNDROME(T, t)                \
{                                       \
        LOAD((t), T);                   \
        MUL4(T);                        \
        STORE((t), T);                  \
}


/*
 * PARITY CALCULATION
 *
 * Macros *_SYNDROME are used for parity/syndrome calculation.
 * *_D_SYNDROME() macros are used to calculate syndrome between 0 and
 * length of data column, and *_SYNDROME() macros are only for updating
 * the parity/syndrome if data column is shorter.
 *
 * P parity is calculated using raidz_add_abd().
 *
 * For CPU L2 cache blocking we process 64KB at a time.
 */
#define CHUNK           65536

/*
 * Generate P parity (RAIDZ1)
 *
 * @rr  RAIDZ row
 */
static raidz_inline void
raidz_generate_p_impl(raidz_row_t * const rr)
{
        size_t c;
        const size_t ncols = rr->rr_cols;
        const size_t psize = rr->rr_col[CODE_P].rc_size;
        abd_t *pabd = rr->rr_col[CODE_P].rc_abd;
        size_t off, size;

        raidz_math_begin();

        for (off = 0; off < psize; off += CHUNK) {

                /* start with first data column */
                size = MIN(CHUNK, psize - off);
                raidz_copy(pabd, rr->rr_col[1].rc_abd, off, size);

                for (c = 2; c < ncols; c++) {
                        size = rr->rr_col[c].rc_size;
                        if (size <= off)
                                continue;

                        /* add data column */
                        size = MIN(CHUNK, size - off);
                        abd_t *dabd = rr->rr_col[c].rc_abd;
                        raidz_add(pabd, dabd, off, size);
                }
        }

        raidz_math_end();
}


/*
 * Generate PQ parity (RAIDZ2)
 * The function is called per data column.
 *
 * @c           array of pointers to parity (code) columns
 * @dc          pointer to data column
 * @csize       size of parity columns
 * @dsize       size of data column
 */
static void
raidz_gen_pq_add(void **c, const void *dc, const size_t csize,
    const size_t dsize)
{
        v_t *p = (v_t *)c[0];
        v_t *q = (v_t *)c[1];
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));
        const v_t * const qend = q + (csize / sizeof (v_t));

        GEN_PQ_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += GEN_PQ_STRIDE, p += GEN_PQ_STRIDE,
            q += GEN_PQ_STRIDE) {
                LOAD(d, GEN_PQ_D);
                P_D_SYNDROME(GEN_PQ_D, GEN_PQ_C, p);
                Q_D_SYNDROME(GEN_PQ_D, GEN_PQ_C, q);
        }
        for (; q < qend; q += GEN_PQ_STRIDE) {
                Q_SYNDROME(GEN_PQ_C, q);
        }
}


/*
 * Generate PQ parity (RAIDZ2)
 *
 * @rr  RAIDZ row
 */
static raidz_inline void
raidz_generate_pq_impl(raidz_row_t * const rr)
{
        size_t c;
        const size_t ncols = rr->rr_cols;
        const size_t csize = rr->rr_col[CODE_P].rc_size;
        size_t off, size, dsize;
        abd_t *dabd;
        abd_t *cabds[] = {
                rr->rr_col[CODE_P].rc_abd,
                rr->rr_col[CODE_Q].rc_abd
        };

        raidz_math_begin();

        for (off = 0; off < csize; off += CHUNK) {

                size = MIN(CHUNK, csize - off);
                raidz_copy(cabds[CODE_P], rr->rr_col[2].rc_abd, off, size);
                raidz_copy(cabds[CODE_Q], rr->rr_col[2].rc_abd, off, size);

                for (c = 3; c < ncols; c++) {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                        dsize = (dsize > off) ? MIN(CHUNK, dsize - off) : 0;

                        abd_raidz_gen_iterate(cabds, dabd, off, size, dsize, 2,
                            raidz_gen_pq_add);
                }
        }

        raidz_math_end();
}


/*
 * Generate PQR parity (RAIDZ3)
 * The function is called per data column.
 *
 * @c           array of pointers to parity (code) columns
 * @dc          pointer to data column
 * @csize       size of parity columns
 * @dsize       size of data column
 */
static void
raidz_gen_pqr_add(void **c, const void *dc, const size_t csize,
    const size_t dsize)
{
        v_t *p = (v_t *)c[CODE_P];
        v_t *q = (v_t *)c[CODE_Q];
        v_t *r = (v_t *)c[CODE_R];
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));
        const v_t * const qend = q + (csize / sizeof (v_t));

        GEN_PQR_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += GEN_PQR_STRIDE, p += GEN_PQR_STRIDE,
            q += GEN_PQR_STRIDE, r += GEN_PQR_STRIDE) {
                LOAD(d, GEN_PQR_D);
                P_D_SYNDROME(GEN_PQR_D, GEN_PQR_C, p);
                Q_D_SYNDROME(GEN_PQR_D, GEN_PQR_C, q);
                R_D_SYNDROME(GEN_PQR_D, GEN_PQR_C, r);
        }
        for (; q < qend; q += GEN_PQR_STRIDE, r += GEN_PQR_STRIDE) {
                Q_SYNDROME(GEN_PQR_C, q);
                R_SYNDROME(GEN_PQR_C, r);
        }
}


/*
 * Generate PQR parity (RAIDZ3)
 *
 * @rr  RAIDZ row
 */
static raidz_inline void
raidz_generate_pqr_impl(raidz_row_t * const rr)
{
        size_t c;
        const size_t ncols = rr->rr_cols;
        const size_t csize = rr->rr_col[CODE_P].rc_size;
        size_t off, size, dsize;
        abd_t *dabd;
        abd_t *cabds[] = {
                rr->rr_col[CODE_P].rc_abd,
                rr->rr_col[CODE_Q].rc_abd,
                rr->rr_col[CODE_R].rc_abd
        };

        raidz_math_begin();

        for (off = 0; off < csize; off += CHUNK) {

                size = MIN(CHUNK, csize - off);
                raidz_copy(cabds[CODE_P], rr->rr_col[3].rc_abd, off, size);
                raidz_copy(cabds[CODE_Q], rr->rr_col[3].rc_abd, off, size);
                raidz_copy(cabds[CODE_R], rr->rr_col[3].rc_abd, off, size);

                for (c = 4; c < ncols; c++) {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                        dsize = (dsize > off) ? MIN(CHUNK, dsize - off) : 0;

                        abd_raidz_gen_iterate(cabds, dabd, off, size, dsize, 3,
                            raidz_gen_pqr_add);
                }
        }

        raidz_math_end();
}


/*
 * DATA RECONSTRUCTION
 *
 * Data reconstruction process consists of two phases:
 *      - Syndrome calculation
 *      - Data reconstruction
 *
 * Syndrome is calculated by generating parity using available data columns
 * and zeros in places of erasure. Existing parity is added to corresponding
 * syndrome value to obtain the [P|Q|R]syn values from equation:
 *      P = Psyn + Dx + Dy + Dz
 *      Q = Qsyn + 2^x * Dx + 2^y * Dy + 2^z * Dz
 *      R = Rsyn + 4^x * Dx + 4^y * Dy + 4^z * Dz
 *
 * For data reconstruction phase, the corresponding equations are solved
 * for missing data (Dx, Dy, Dz). This generally involves multiplying known
 * symbols by an coefficient and adding them together. The multiplication
 * constant coefficients are calculated ahead of the operation in
 * raidz_rec_[q|r|pq|pq|qr|pqr]_coeff() functions.
 *
 * IMPLEMENTATION NOTE: RAID-Z block can have complex geometry, with "big"
 * and "short" columns.
 * For this reason, reconstruction is performed in minimum of
 * two steps. First, from offset 0 to short_size, then from short_size to
 * short_size. Calculation functions REC_[*]_BLOCK() are implemented to work
 * over both ranges. The split also enables removal of conditional expressions
 * from loop bodies, improving throughput of SIMD implementations.
 * For the best performance, all functions marked with raidz_inline attribute
 * must be inlined by compiler.
 *
 *    parity          data
 *    columns         columns
 * <----------> <------------------>
 *                   x       y  <----+ missing columns (x, y)
 *                   |       |
 * +---+---+---+---+-v-+---+-v-+---+   ^ 0
 * |   |   |   |   |   |   |   |   |   |
 * |   |   |   |   |   |   |   |   |   |
 * | P | Q | R | D | D | D | D | D |   |
 * |   |   |   | 0 | 1 | 2 | 3 | 4 |   |
 * |   |   |   |   |   |   |   |   |   v
 * |   |   |   |   |   +---+---+---+   ^ short_size
 * |   |   |   |   |   |               |
 * +---+---+---+---+---+               v big_size
 * <------------------> <---------->
 *      big columns     short columns
 *
 */




/*
 * Reconstruct single data column using P parity
 *
 * @syn_method  raidz_add_abd()
 * @rec_method  not applicable
 *
 * @rr          RAIDZ row
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_p_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[TARGET_X];
        const size_t xsize = rr->rr_col[x].rc_size;
        abd_t *xabd = rr->rr_col[x].rc_abd;
        size_t off, size;

        if (xabd == NULL)
                return (1 << CODE_P);

        raidz_math_begin();

        for (off = 0; off < xsize; off += CHUNK) {

                /* copy P into target */
                size = MIN(CHUNK, xsize - off);
                raidz_copy(xabd, rr->rr_col[CODE_P].rc_abd, off, size);

                /* generate p_syndrome */
                for (c = firstdc; c < ncols; c++) {
                        if (c == x)
                                continue;
                        size = rr->rr_col[c].rc_size;
                        if (size <= off)
                                continue;

                        size = MIN(CHUNK, MIN(size, xsize) - off);
                        abd_t *dabd = rr->rr_col[c].rc_abd;
                        raidz_add(xabd, dabd, off, size);
                }
        }

        raidz_math_end();

        return (1 << CODE_P);
}


/*
 * Generate Q syndrome (Qsyn)
 *
 * @xc          array of pointers to syndrome columns
 * @dc          data column (NULL if missing)
 * @xsize       size of syndrome columns
 * @dsize       size of data column (0 if missing)
 */
static void
raidz_syn_q_abd(void **xc, const void *dc, const size_t xsize,
    const size_t dsize)
{
        v_t *x = (v_t *)xc[TARGET_X];
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));
        const v_t * const xend = x + (xsize / sizeof (v_t));

        SYN_Q_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE) {
                LOAD(d, SYN_Q_D);
                Q_D_SYNDROME(SYN_Q_D, SYN_Q_X, x);
        }
        for (; x < xend; x += SYN_STRIDE) {
                Q_SYNDROME(SYN_Q_X, x);
        }
}


/*
 * Reconstruct single data column using Q parity
 *
 * @syn_method  raidz_add_abd()
 * @rec_method  raidz_mul_abd_cb()
 *
 * @rr          RAIDZ row
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_q_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        size_t dsize;
        abd_t *dabd;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[TARGET_X];
        abd_t *xabd = rr->rr_col[x].rc_abd;
        const size_t xsize = rr->rr_col[x].rc_size;
        abd_t *tabds[] = { xabd };

        if (xabd == NULL)
                return (1 << CODE_Q);

        unsigned coeff[MUL_CNT];
        raidz_rec_q_coeff(rr, tgtidx, coeff);

        raidz_math_begin();

        /* Start with first data column if present */
        if (firstdc != x) {
                raidz_copy(xabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
        } else {
                raidz_zero(xabd, xsize);
        }

        /* generate q_syndrome */
        for (c = firstdc+1; c < ncols; c++) {
                if (c == x) {
                        dabd = NULL;
                        dsize = 0;
                } else {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                }

                abd_raidz_gen_iterate(tabds, dabd, 0, xsize, dsize, 1,
                    raidz_syn_q_abd);
        }

        /* add Q to the syndrome */
        raidz_add(xabd, rr->rr_col[CODE_Q].rc_abd, 0, xsize);

        /* transform the syndrome */
        abd_iterate_func(xabd, 0, xsize, raidz_mul_abd_cb, (void*) coeff);

        raidz_math_end();

        return (1 << CODE_Q);
}


/*
 * Generate R syndrome (Rsyn)
 *
 * @xc          array of pointers to syndrome columns
 * @dc          data column (NULL if missing)
 * @tsize       size of syndrome columns
 * @dsize       size of data column (0 if missing)
 */
static void
raidz_syn_r_abd(void **xc, const void *dc, const size_t tsize,
    const size_t dsize)
{
        v_t *x = (v_t *)xc[TARGET_X];
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));
        const v_t * const xend = x + (tsize / sizeof (v_t));

        SYN_R_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE) {
                LOAD(d, SYN_R_D);
                R_D_SYNDROME(SYN_R_D, SYN_R_X, x);
        }
        for (; x < xend; x += SYN_STRIDE) {
                R_SYNDROME(SYN_R_X, x);
        }
}


/*
 * Reconstruct single data column using R parity
 *
 * @syn_method  raidz_add_abd()
 * @rec_method  raidz_mul_abd_cb()
 *
 * @rr          RAIDZ rr
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_r_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        size_t dsize;
        abd_t *dabd;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[TARGET_X];
        const size_t xsize = rr->rr_col[x].rc_size;
        abd_t *xabd = rr->rr_col[x].rc_abd;
        abd_t *tabds[] = { xabd };

        if (xabd == NULL)
                return (1 << CODE_R);

        unsigned coeff[MUL_CNT];
        raidz_rec_r_coeff(rr, tgtidx, coeff);

        raidz_math_begin();

        /* Start with first data column if present */
        if (firstdc != x) {
                raidz_copy(xabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
        } else {
                raidz_zero(xabd, xsize);
        }


        /* generate q_syndrome */
        for (c = firstdc+1; c < ncols; c++) {
                if (c == x) {
                        dabd = NULL;
                        dsize = 0;
                } else {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                }

                abd_raidz_gen_iterate(tabds, dabd, 0, xsize, dsize, 1,
                    raidz_syn_r_abd);
        }

        /* add R to the syndrome */
        raidz_add(xabd, rr->rr_col[CODE_R].rc_abd, 0, xsize);

        /* transform the syndrome */
        abd_iterate_func(xabd, 0, xsize, raidz_mul_abd_cb, (void *)coeff);

        raidz_math_end();

        return (1 << CODE_R);
}


/*
 * Generate P and Q syndromes
 *
 * @xc          array of pointers to syndrome columns
 * @dc          data column (NULL if missing)
 * @tsize       size of syndrome columns
 * @dsize       size of data column (0 if missing)
 */
static void
raidz_syn_pq_abd(void **tc, const void *dc, const size_t tsize,
    const size_t dsize)
{
        v_t *x = (v_t *)tc[TARGET_X];
        v_t *y = (v_t *)tc[TARGET_Y];
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));
        const v_t * const yend = y + (tsize / sizeof (v_t));

        SYN_PQ_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE) {
                LOAD(d, SYN_PQ_D);
                P_D_SYNDROME(SYN_PQ_D, SYN_PQ_X, x);
                Q_D_SYNDROME(SYN_PQ_D, SYN_PQ_X, y);
        }
        for (; y < yend; y += SYN_STRIDE) {
                Q_SYNDROME(SYN_PQ_X, y);
        }
}

/*
 * Reconstruct data using PQ parity and PQ syndromes
 *
 * @tc          syndrome/result columns
 * @tsize       size of syndrome/result columns
 * @c           parity columns
 * @mul         array of multiplication constants
 */
static void
raidz_rec_pq_abd(void **tc, const size_t tsize, void **c,
    const unsigned *mul)
{
        v_t *x = (v_t *)tc[TARGET_X];
        v_t *y = (v_t *)tc[TARGET_Y];
        const v_t * const xend = x + (tsize / sizeof (v_t));
        const v_t *p = (v_t *)c[CODE_P];
        const v_t *q = (v_t *)c[CODE_Q];

        REC_PQ_DEFINE();

        for (; x < xend; x += REC_PQ_STRIDE, y += REC_PQ_STRIDE,
            p += REC_PQ_STRIDE, q += REC_PQ_STRIDE) {
                LOAD(x, REC_PQ_X);
                LOAD(y, REC_PQ_Y);

                XOR_ACC(p, REC_PQ_X);
                XOR_ACC(q, REC_PQ_Y);

                /* Save Pxy */
                COPY(REC_PQ_X,  REC_PQ_T);

                /* Calc X */
                MUL(mul[MUL_PQ_X], REC_PQ_X);
                MUL(mul[MUL_PQ_Y], REC_PQ_Y);
                XOR(REC_PQ_Y,  REC_PQ_X);
                STORE(x, REC_PQ_X);

                /* Calc Y */
                XOR(REC_PQ_T,  REC_PQ_X);
                STORE(y, REC_PQ_X);
        }
}


/*
 * Reconstruct two data columns using PQ parity
 *
 * @syn_method  raidz_syn_pq_abd()
 * @rec_method  raidz_rec_pq_abd()
 *
 * @rr          RAIDZ row
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_pq_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        size_t dsize;
        abd_t *dabd;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[TARGET_X];
        const size_t y = tgtidx[TARGET_Y];
        const size_t xsize = rr->rr_col[x].rc_size;
        const size_t ysize = rr->rr_col[y].rc_size;
        abd_t *xabd = rr->rr_col[x].rc_abd;
        abd_t *yabd = rr->rr_col[y].rc_abd;
        abd_t *tabds[2] = { xabd, yabd };
        abd_t *cabds[] = {
                rr->rr_col[CODE_P].rc_abd,
                rr->rr_col[CODE_Q].rc_abd
        };

        if (xabd == NULL)
                return ((1 << CODE_P) | (1 << CODE_Q));

        unsigned coeff[MUL_CNT];
        raidz_rec_pq_coeff(rr, tgtidx, coeff);

        /*
         * Check if some of targets is shorter then others
         * In this case, shorter target needs to be replaced with
         * new buffer so that syndrome can be calculated.
         */
        if (ysize < xsize) {
                yabd = abd_alloc(xsize, B_FALSE);
                tabds[1] = yabd;
        }

        raidz_math_begin();

        /* Start with first data column if present */
        if (firstdc != x) {
                raidz_copy(xabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
                raidz_copy(yabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
        } else {
                raidz_zero(xabd, xsize);
                raidz_zero(yabd, xsize);
        }

        /* generate q_syndrome */
        for (c = firstdc+1; c < ncols; c++) {
                if (c == x || c == y) {
                        dabd = NULL;
                        dsize = 0;
                } else {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                }

                abd_raidz_gen_iterate(tabds, dabd, 0, xsize, dsize, 2,
                    raidz_syn_pq_abd);
        }

        abd_raidz_rec_iterate(cabds, tabds, xsize, 2, raidz_rec_pq_abd, coeff);

        /* Copy shorter targets back to the original abd buffer */
        if (ysize < xsize)
                raidz_copy(rr->rr_col[y].rc_abd, yabd, 0, ysize);

        raidz_math_end();

        if (ysize < xsize)
                abd_free(yabd);

        return ((1 << CODE_P) | (1 << CODE_Q));
}


/*
 * Generate P and R syndromes
 *
 * @xc          array of pointers to syndrome columns
 * @dc          data column (NULL if missing)
 * @tsize       size of syndrome columns
 * @dsize       size of data column (0 if missing)
 */
static void
raidz_syn_pr_abd(void **c, const void *dc, const size_t tsize,
    const size_t dsize)
{
        v_t *x = (v_t *)c[TARGET_X];
        v_t *y = (v_t *)c[TARGET_Y];
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));
        const v_t * const yend = y + (tsize / sizeof (v_t));

        SYN_PR_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE) {
                LOAD(d, SYN_PR_D);
                P_D_SYNDROME(SYN_PR_D, SYN_PR_X, x);
                R_D_SYNDROME(SYN_PR_D, SYN_PR_X, y);
        }
        for (; y < yend; y += SYN_STRIDE) {
                R_SYNDROME(SYN_PR_X, y);
        }
}

/*
 * Reconstruct data using PR parity and PR syndromes
 *
 * @tc          syndrome/result columns
 * @tsize       size of syndrome/result columns
 * @c           parity columns
 * @mul         array of multiplication constants
 */
static void
raidz_rec_pr_abd(void **t, const size_t tsize, void **c,
    const unsigned *mul)
{
        v_t *x = (v_t *)t[TARGET_X];
        v_t *y = (v_t *)t[TARGET_Y];
        const v_t * const xend = x + (tsize / sizeof (v_t));
        const v_t *p = (v_t *)c[CODE_P];
        const v_t *q = (v_t *)c[CODE_Q];

        REC_PR_DEFINE();

        for (; x < xend; x += REC_PR_STRIDE, y += REC_PR_STRIDE,
            p += REC_PR_STRIDE, q += REC_PR_STRIDE) {
                LOAD(x, REC_PR_X);
                LOAD(y, REC_PR_Y);
                XOR_ACC(p, REC_PR_X);
                XOR_ACC(q, REC_PR_Y);

                /* Save Pxy */
                COPY(REC_PR_X,  REC_PR_T);

                /* Calc X */
                MUL(mul[MUL_PR_X], REC_PR_X);
                MUL(mul[MUL_PR_Y], REC_PR_Y);
                XOR(REC_PR_Y,  REC_PR_X);
                STORE(x, REC_PR_X);

                /* Calc Y */
                XOR(REC_PR_T,  REC_PR_X);
                STORE(y, REC_PR_X);
        }
}


/*
 * Reconstruct two data columns using PR parity
 *
 * @syn_method  raidz_syn_pr_abd()
 * @rec_method  raidz_rec_pr_abd()
 *
 * @rr          RAIDZ row
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_pr_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        size_t dsize;
        abd_t *dabd;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[0];
        const size_t y = tgtidx[1];
        const size_t xsize = rr->rr_col[x].rc_size;
        const size_t ysize = rr->rr_col[y].rc_size;
        abd_t *xabd = rr->rr_col[x].rc_abd;
        abd_t *yabd = rr->rr_col[y].rc_abd;
        abd_t *tabds[2] = { xabd, yabd };
        abd_t *cabds[] = {
                rr->rr_col[CODE_P].rc_abd,
                rr->rr_col[CODE_R].rc_abd
        };

        if (xabd == NULL)
                return ((1 << CODE_P) | (1 << CODE_R));

        unsigned coeff[MUL_CNT];
        raidz_rec_pr_coeff(rr, tgtidx, coeff);

        /*
         * Check if some of targets are shorter then others.
         * They need to be replaced with a new buffer so that syndrome can
         * be calculated on full length.
         */
        if (ysize < xsize) {
                yabd = abd_alloc(xsize, B_FALSE);
                tabds[1] = yabd;
        }

        raidz_math_begin();

        /* Start with first data column if present */
        if (firstdc != x) {
                raidz_copy(xabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
                raidz_copy(yabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
        } else {
                raidz_zero(xabd, xsize);
                raidz_zero(yabd, xsize);
        }

        /* generate q_syndrome */
        for (c = firstdc+1; c < ncols; c++) {
                if (c == x || c == y) {
                        dabd = NULL;
                        dsize = 0;
                } else {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                }

                abd_raidz_gen_iterate(tabds, dabd, 0, xsize, dsize, 2,
                    raidz_syn_pr_abd);
        }

        abd_raidz_rec_iterate(cabds, tabds, xsize, 2, raidz_rec_pr_abd, coeff);

        /*
         * Copy shorter targets back to the original abd buffer
         */
        if (ysize < xsize)
                raidz_copy(rr->rr_col[y].rc_abd, yabd, 0, ysize);

        raidz_math_end();

        if (ysize < xsize)
                abd_free(yabd);

        return ((1 << CODE_P) | (1 << CODE_R));
}


/*
 * Generate Q and R syndromes
 *
 * @xc          array of pointers to syndrome columns
 * @dc          data column (NULL if missing)
 * @tsize       size of syndrome columns
 * @dsize       size of data column (0 if missing)
 */
static void
raidz_syn_qr_abd(void **c, const void *dc, const size_t tsize,
    const size_t dsize)
{
        v_t *x = (v_t *)c[TARGET_X];
        v_t *y = (v_t *)c[TARGET_Y];
        const v_t * const xend = x + (tsize / sizeof (v_t));
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));

        SYN_QR_DEFINE();

        MUL2_SETUP();

        for (; d < dend; d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE) {
                LOAD(d, SYN_PQ_D);
                Q_D_SYNDROME(SYN_QR_D, SYN_QR_X, x);
                R_D_SYNDROME(SYN_QR_D, SYN_QR_X, y);
        }
        for (; x < xend; x += SYN_STRIDE, y += SYN_STRIDE) {
                Q_SYNDROME(SYN_QR_X, x);
                R_SYNDROME(SYN_QR_X, y);
        }
}


/*
 * Reconstruct data using QR parity and QR syndromes
 *
 * @tc          syndrome/result columns
 * @tsize       size of syndrome/result columns
 * @c           parity columns
 * @mul         array of multiplication constants
 */
static void
raidz_rec_qr_abd(void **t, const size_t tsize, void **c,
    const unsigned *mul)
{
        v_t *x = (v_t *)t[TARGET_X];
        v_t *y = (v_t *)t[TARGET_Y];
        const v_t * const xend = x + (tsize / sizeof (v_t));
        const v_t *p = (v_t *)c[CODE_P];
        const v_t *q = (v_t *)c[CODE_Q];

        REC_QR_DEFINE();

        for (; x < xend; x += REC_QR_STRIDE, y += REC_QR_STRIDE,
            p += REC_QR_STRIDE, q += REC_QR_STRIDE) {
                LOAD(x, REC_QR_X);
                LOAD(y, REC_QR_Y);

                XOR_ACC(p, REC_QR_X);
                XOR_ACC(q, REC_QR_Y);

                /* Save Pxy */
                COPY(REC_QR_X,  REC_QR_T);

                /* Calc X */
                MUL(mul[MUL_QR_XQ], REC_QR_X);  /* X = Q * xqm */
                XOR(REC_QR_Y, REC_QR_X);        /* X = R ^ X   */
                MUL(mul[MUL_QR_X], REC_QR_X);   /* X = X * xm  */
                STORE(x, REC_QR_X);

                /* Calc Y */
                MUL(mul[MUL_QR_YQ], REC_QR_T);  /* X = Q * xqm */
                XOR(REC_QR_Y, REC_QR_T);        /* X = R ^ X   */
                MUL(mul[MUL_QR_Y], REC_QR_T);   /* X = X * xm  */
                STORE(y, REC_QR_T);
        }
}


/*
 * Reconstruct two data columns using QR parity
 *
 * @syn_method  raidz_syn_qr_abd()
 * @rec_method  raidz_rec_qr_abd()
 *
 * @rr          RAIDZ row
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_qr_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        size_t dsize;
        abd_t *dabd;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[TARGET_X];
        const size_t y = tgtidx[TARGET_Y];
        const size_t xsize = rr->rr_col[x].rc_size;
        const size_t ysize = rr->rr_col[y].rc_size;
        abd_t *xabd = rr->rr_col[x].rc_abd;
        abd_t *yabd = rr->rr_col[y].rc_abd;
        abd_t *tabds[2] = { xabd, yabd };
        abd_t *cabds[] = {
                rr->rr_col[CODE_Q].rc_abd,
                rr->rr_col[CODE_R].rc_abd
        };

        if (xabd == NULL)
                return ((1 << CODE_Q) | (1 << CODE_R));

        unsigned coeff[MUL_CNT];
        raidz_rec_qr_coeff(rr, tgtidx, coeff);

        /*
         * Check if some of targets is shorter then others
         * In this case, shorter target needs to be replaced with
         * new buffer so that syndrome can be calculated.
         */
        if (ysize < xsize) {
                yabd = abd_alloc(xsize, B_FALSE);
                tabds[1] = yabd;
        }

        raidz_math_begin();

        /* Start with first data column if present */
        if (firstdc != x) {
                raidz_copy(xabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
                raidz_copy(yabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
        } else {
                raidz_zero(xabd, xsize);
                raidz_zero(yabd, xsize);
        }

        /* generate q_syndrome */
        for (c = firstdc+1; c < ncols; c++) {
                if (c == x || c == y) {
                        dabd = NULL;
                        dsize = 0;
                } else {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                }

                abd_raidz_gen_iterate(tabds, dabd, 0, xsize, dsize, 2,
                    raidz_syn_qr_abd);
        }

        abd_raidz_rec_iterate(cabds, tabds, xsize, 2, raidz_rec_qr_abd, coeff);

        /*
         * Copy shorter targets back to the original abd buffer
         */
        if (ysize < xsize)
                raidz_copy(rr->rr_col[y].rc_abd, yabd, 0, ysize);

        raidz_math_end();

        if (ysize < xsize)
                abd_free(yabd);


        return ((1 << CODE_Q) | (1 << CODE_R));
}


/*
 * Generate P, Q, and R syndromes
 *
 * @xc          array of pointers to syndrome columns
 * @dc          data column (NULL if missing)
 * @tsize       size of syndrome columns
 * @dsize       size of data column (0 if missing)
 */
static void
raidz_syn_pqr_abd(void **c, const void *dc, const size_t tsize,
    const size_t dsize)
{
        v_t *x = (v_t *)c[TARGET_X];
        v_t *y = (v_t *)c[TARGET_Y];
        v_t *z = (v_t *)c[TARGET_Z];
        const v_t * const yend = y + (tsize / sizeof (v_t));
        const v_t *d = (const v_t *)dc;
        const v_t * const dend = d + (dsize / sizeof (v_t));

        SYN_PQR_DEFINE();

        MUL2_SETUP();

        for (; d < dend;  d += SYN_STRIDE, x += SYN_STRIDE, y += SYN_STRIDE,
            z += SYN_STRIDE) {
                LOAD(d, SYN_PQR_D);
                P_D_SYNDROME(SYN_PQR_D, SYN_PQR_X, x)
                Q_D_SYNDROME(SYN_PQR_D, SYN_PQR_X, y);
                R_D_SYNDROME(SYN_PQR_D, SYN_PQR_X, z);
        }
        for (; y < yend; y += SYN_STRIDE, z += SYN_STRIDE) {
                Q_SYNDROME(SYN_PQR_X, y);
                R_SYNDROME(SYN_PQR_X, z);
        }
}


/*
 * Reconstruct data using PRQ parity and PQR syndromes
 *
 * @tc          syndrome/result columns
 * @tsize       size of syndrome/result columns
 * @c           parity columns
 * @mul         array of multiplication constants
 */
static void
raidz_rec_pqr_abd(void **t, const size_t tsize, void **c,
    const unsigned * const mul)
{
        v_t *x = (v_t *)t[TARGET_X];
        v_t *y = (v_t *)t[TARGET_Y];
        v_t *z = (v_t *)t[TARGET_Z];
        const v_t * const xend = x + (tsize / sizeof (v_t));
        const v_t *p = (v_t *)c[CODE_P];
        const v_t *q = (v_t *)c[CODE_Q];
        const v_t *r = (v_t *)c[CODE_R];

        REC_PQR_DEFINE();

        for (; x < xend; x += REC_PQR_STRIDE, y += REC_PQR_STRIDE,
            z += REC_PQR_STRIDE, p += REC_PQR_STRIDE, q += REC_PQR_STRIDE,
            r += REC_PQR_STRIDE) {
                LOAD(x, REC_PQR_X);
                LOAD(y, REC_PQR_Y);
                LOAD(z, REC_PQR_Z);

                XOR_ACC(p, REC_PQR_X);
                XOR_ACC(q, REC_PQR_Y);
                XOR_ACC(r, REC_PQR_Z);

                /* Save Pxyz and Qxyz */
                COPY(REC_PQR_X, REC_PQR_XS);
                COPY(REC_PQR_Y, REC_PQR_YS);

                /* Calc X */
                MUL(mul[MUL_PQR_XP], REC_PQR_X);        /* Xp = Pxyz * xp   */
                MUL(mul[MUL_PQR_XQ], REC_PQR_Y);        /* Xq = Qxyz * xq   */
                XOR(REC_PQR_Y, REC_PQR_X);
                MUL(mul[MUL_PQR_XR], REC_PQR_Z);        /* Xr = Rxyz * xr   */
                XOR(REC_PQR_Z, REC_PQR_X);              /* X = Xp + Xq + Xr */
                STORE(x, REC_PQR_X);

                /* Calc Y */
                XOR(REC_PQR_X, REC_PQR_XS);             /* Pyz = Pxyz + X */
                MUL(mul[MUL_PQR_YU], REC_PQR_X);        /* Xq = X * upd_q */
                XOR(REC_PQR_X, REC_PQR_YS);             /* Qyz = Qxyz + Xq */
                COPY(REC_PQR_XS, REC_PQR_X);            /* restore Pyz */
                MUL(mul[MUL_PQR_YP], REC_PQR_X);        /* Yp = Pyz * yp */
                MUL(mul[MUL_PQR_YQ], REC_PQR_YS);       /* Yq = Qyz * yq */
                XOR(REC_PQR_X, REC_PQR_YS);             /* Y = Yp + Yq */
                STORE(y, REC_PQR_YS);

                /* Calc Z */
                XOR(REC_PQR_XS, REC_PQR_YS);            /* Z = Pz = Pyz + Y */
                STORE(z, REC_PQR_YS);
        }
}


/*
 * Reconstruct three data columns using PQR parity
 *
 * @syn_method  raidz_syn_pqr_abd()
 * @rec_method  raidz_rec_pqr_abd()
 *
 * @rr          RAIDZ row
 * @tgtidx      array of missing data indexes
 */
static raidz_inline int
raidz_reconstruct_pqr_impl(raidz_row_t *rr, const int *tgtidx)
{
        size_t c;
        size_t dsize;
        abd_t *dabd;
        const size_t firstdc = rr->rr_firstdatacol;
        const size_t ncols = rr->rr_cols;
        const size_t x = tgtidx[TARGET_X];
        const size_t y = tgtidx[TARGET_Y];
        const size_t z = tgtidx[TARGET_Z];
        const size_t xsize = rr->rr_col[x].rc_size;
        const size_t ysize = rr->rr_col[y].rc_size;
        const size_t zsize = rr->rr_col[z].rc_size;
        abd_t *xabd = rr->rr_col[x].rc_abd;
        abd_t *yabd = rr->rr_col[y].rc_abd;
        abd_t *zabd = rr->rr_col[z].rc_abd;
        abd_t *tabds[] = { xabd, yabd, zabd };
        abd_t *cabds[] = {
                rr->rr_col[CODE_P].rc_abd,
                rr->rr_col[CODE_Q].rc_abd,
                rr->rr_col[CODE_R].rc_abd
        };

        if (xabd == NULL)
                return ((1 << CODE_P) | (1 << CODE_Q) | (1 << CODE_R));

        unsigned coeff[MUL_CNT];
        raidz_rec_pqr_coeff(rr, tgtidx, coeff);

        /*
         * Check if some of targets is shorter then others
         * In this case, shorter target needs to be replaced with
         * new buffer so that syndrome can be calculated.
         */
        if (ysize < xsize) {
                yabd = abd_alloc(xsize, B_FALSE);
                tabds[1] = yabd;
        }
        if (zsize < xsize) {
                zabd = abd_alloc(xsize, B_FALSE);
                tabds[2] = zabd;
        }

        raidz_math_begin();

        /* Start with first data column if present */
        if (firstdc != x) {
                raidz_copy(xabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
                raidz_copy(yabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
                raidz_copy(zabd, rr->rr_col[firstdc].rc_abd, 0, xsize);
        } else {
                raidz_zero(xabd, xsize);
                raidz_zero(yabd, xsize);
                raidz_zero(zabd, xsize);
        }

        /* generate q_syndrome */
        for (c = firstdc+1; c < ncols; c++) {
                if (c == x || c == y || c == z) {
                        dabd = NULL;
                        dsize = 0;
                } else {
                        dabd = rr->rr_col[c].rc_abd;
                        dsize = rr->rr_col[c].rc_size;
                }

                abd_raidz_gen_iterate(tabds, dabd, 0, xsize, dsize, 3,
                    raidz_syn_pqr_abd);
        }

        abd_raidz_rec_iterate(cabds, tabds, xsize, 3, raidz_rec_pqr_abd, coeff);

        /*
         * Copy shorter targets back to the original abd buffer
         */
        if (ysize < xsize)
                raidz_copy(rr->rr_col[y].rc_abd, yabd, 0, ysize);
        if (zsize < xsize)
                raidz_copy(rr->rr_col[z].rc_abd, zabd, 0, zsize);

        raidz_math_end();

        if (ysize < xsize)
                abd_free(yabd);
        if (zsize < xsize)
                abd_free(zabd);

        return ((1 << CODE_P) | (1 << CODE_Q) | (1 << CODE_R));
}

#endif /* _VDEV_RAIDZ_MATH_IMPL_H */