module/zfs/vdev_raidz_math_aarch64_neonx2.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or https://opensource.org/licenses/CDDL-1.0.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (C) 2016 Romain Dolbeau. All rights reserved.
  23  */
  24
  25 #include <sys/isa_defs.h>
  26
  27 #if defined(__aarch64__)
  28
  29 #include "vdev_raidz_math_aarch64_neon_common.h"
  30
  31 #define SYN_STRIDE              4
  32
  33 #define ZERO_STRIDE             8
  34 #define ZERO_DEFINE()   \
  35         GEN_X_DEFINE_0_3()      \
  36         GEN_X_DEFINE_4_5()      \
  37         GEN_X_DEFINE_6_7()
  38 #define ZERO_D                  0, 1, 2, 3, 4, 5, 6, 7
  39
  40 #define COPY_STRIDE             8
  41 #define COPY_DEFINE()   \
  42         GEN_X_DEFINE_0_3()      \
  43         GEN_X_DEFINE_4_5()      \
  44         GEN_X_DEFINE_6_7()
  45 #define COPY_D                  0, 1, 2, 3, 4, 5, 6, 7
  46
  47 #define ADD_STRIDE              8
  48 #define ADD_DEFINE()    \
  49         GEN_X_DEFINE_0_3()      \
  50         GEN_X_DEFINE_4_5()      \
  51         GEN_X_DEFINE_6_7()
  52 #define ADD_D                   0, 1, 2, 3, 4, 5, 6, 7
  53
  54 #define MUL_STRIDE              4
  55 #define MUL_DEFINE()    \
  56         GEN_X_DEFINE_0_3()  \
  57         GEN_X_DEFINE_33_36()
  58 #define MUL_D                   0, 1, 2, 3
  59
  60 #define GEN_P_DEFINE() \
  61         GEN_X_DEFINE_0_3() \
  62         GEN_X_DEFINE_33_36()
  63 #define GEN_P_STRIDE            4
  64 #define GEN_P_P                 0, 1, 2, 3
  65
  66 #define GEN_PQ_DEFINE() \
  67         GEN_X_DEFINE_0_3()      \
  68         GEN_X_DEFINE_4_5()      \
  69         GEN_X_DEFINE_6_7()      \
  70         GEN_X_DEFINE_16()       \
  71         GEN_X_DEFINE_17()       \
  72         GEN_X_DEFINE_33_36()
  73 #define GEN_PQ_STRIDE           4
  74 #define GEN_PQ_D                0, 1, 2, 3
  75 #define GEN_PQ_C                4, 5, 6, 7
  76
  77 #define GEN_PQR_DEFINE() \
  78         GEN_X_DEFINE_0_3()      \
  79         GEN_X_DEFINE_4_5()      \
  80         GEN_X_DEFINE_6_7()      \
  81         GEN_X_DEFINE_16()       \
  82         GEN_X_DEFINE_17()       \
  83         GEN_X_DEFINE_33_36()
  84 #define GEN_PQR_STRIDE          4
  85 #define GEN_PQR_D               0, 1, 2, 3
  86 #define GEN_PQR_C               4, 5, 6, 7
  87
  88 #define SYN_Q_DEFINE() \
  89         GEN_X_DEFINE_0_3()      \
  90         GEN_X_DEFINE_4_5()      \
  91         GEN_X_DEFINE_6_7()      \
  92         GEN_X_DEFINE_16()       \
  93         GEN_X_DEFINE_17()       \
  94         GEN_X_DEFINE_33_36()
  95 #define SYN_Q_STRIDE            4
  96 #define SYN_Q_D                 0, 1, 2, 3
  97 #define SYN_Q_X                 4, 5, 6, 7
  98
  99 #define SYN_R_DEFINE() \
 100         GEN_X_DEFINE_0_3()      \
 101         GEN_X_DEFINE_4_5()      \
 102         GEN_X_DEFINE_6_7()      \
 103         GEN_X_DEFINE_16()       \
 104         GEN_X_DEFINE_17()       \
 105         GEN_X_DEFINE_33_36()
 106 #define SYN_R_STRIDE            4
 107 #define SYN_R_D                 0, 1, 2, 3
 108 #define SYN_R_X                 4, 5, 6, 7
 109
 110 #define SYN_PQ_DEFINE() \
 111         GEN_X_DEFINE_0_3()      \
 112         GEN_X_DEFINE_4_5()      \
 113         GEN_X_DEFINE_6_7()      \
 114         GEN_X_DEFINE_16()       \
 115         GEN_X_DEFINE_17()       \
 116         GEN_X_DEFINE_33_36()
 117 #define SYN_PQ_STRIDE           4
 118 #define SYN_PQ_D                0, 1, 2, 3
 119 #define SYN_PQ_X                4, 5, 6, 7
 120
 121 #define REC_PQ_DEFINE() \
 122         GEN_X_DEFINE_0_3()      \
 123         GEN_X_DEFINE_4_5()      \
 124         GEN_X_DEFINE_6_7()      \
 125         GEN_X_DEFINE_8_9()      \
 126         GEN_X_DEFINE_22_23()    \
 127         GEN_X_DEFINE_33_36()
 128 #define REC_PQ_STRIDE           4
 129 #define REC_PQ_X                0, 1, 2, 3
 130 #define REC_PQ_Y                4, 5, 6, 7
 131 #define REC_PQ_T                8, 9, 22, 23
 132
 133 #define SYN_PR_DEFINE() \
 134         GEN_X_DEFINE_0_3()      \
 135         GEN_X_DEFINE_4_5()      \
 136         GEN_X_DEFINE_6_7()      \
 137         GEN_X_DEFINE_16()       \
 138         GEN_X_DEFINE_17()       \
 139         GEN_X_DEFINE_33_36()
 140 #define SYN_PR_STRIDE           4
 141 #define SYN_PR_D                0, 1, 2, 3
 142 #define SYN_PR_X                4, 5, 6, 7
 143
 144 #define REC_PR_DEFINE() \
 145         GEN_X_DEFINE_0_3()      \
 146         GEN_X_DEFINE_4_5()      \
 147         GEN_X_DEFINE_6_7()      \
 148         GEN_X_DEFINE_8_9()      \
 149         GEN_X_DEFINE_22_23()    \
 150         GEN_X_DEFINE_33_36()
 151 #define REC_PR_STRIDE           4
 152 #define REC_PR_X                0, 1, 2, 3
 153 #define REC_PR_Y                4, 5, 6, 7
 154 #define REC_PR_T                8, 9, 22, 23
 155
 156 #define SYN_QR_DEFINE() \
 157         GEN_X_DEFINE_0_3()      \
 158         GEN_X_DEFINE_4_5()      \
 159         GEN_X_DEFINE_6_7()      \
 160         GEN_X_DEFINE_16()       \
 161         GEN_X_DEFINE_17()       \
 162         GEN_X_DEFINE_33_36()
 163 #define SYN_QR_STRIDE           4
 164 #define SYN_QR_D                0, 1, 2, 3
 165 #define SYN_QR_X                4, 5, 6, 7
 166
 167 #define REC_QR_DEFINE() \
 168         GEN_X_DEFINE_0_3()      \
 169         GEN_X_DEFINE_4_5()      \
 170         GEN_X_DEFINE_6_7()      \
 171         GEN_X_DEFINE_8_9()      \
 172         GEN_X_DEFINE_22_23()    \
 173         GEN_X_DEFINE_33_36()
 174 #define REC_QR_STRIDE           4
 175 #define REC_QR_X                0, 1, 2, 3
 176 #define REC_QR_Y                4, 5, 6, 7
 177 #define REC_QR_T                8, 9, 22, 23
 178
 179 #define SYN_PQR_DEFINE() \
 180         GEN_X_DEFINE_0_3()      \
 181         GEN_X_DEFINE_4_5()      \
 182         GEN_X_DEFINE_6_7()      \
 183         GEN_X_DEFINE_16()       \
 184         GEN_X_DEFINE_17()       \
 185         GEN_X_DEFINE_33_36()
 186 #define SYN_PQR_STRIDE           4
 187 #define SYN_PQR_D                0, 1, 2, 3
 188 #define SYN_PQR_X                4, 5, 6, 7
 189
 190 #define REC_PQR_DEFINE() \
 191         GEN_X_DEFINE_0_3()      \
 192         GEN_X_DEFINE_4_5()      \
 193         GEN_X_DEFINE_6_7()      \
 194         GEN_X_DEFINE_8_9()      \
 195         GEN_X_DEFINE_31()       \
 196         GEN_X_DEFINE_32()       \
 197         GEN_X_DEFINE_33_36()
 198 #define REC_PQR_STRIDE          2
 199 #define REC_PQR_X               0, 1
 200 #define REC_PQR_Y               2, 3
 201 #define REC_PQR_Z               4, 5
 202 #define REC_PQR_XS              6, 7
 203 #define REC_PQR_YS              8, 9
 204
 205 #include <sys/vdev_raidz_impl.h>
 206 #include "vdev_raidz_math_impl.h"
 207
 208 DEFINE_GEN_METHODS(aarch64_neonx2);
 209 /*
 210  * If compiled with -O0, gcc doesn't do any stack frame coalescing
 211  * and -Wframe-larger-than=1024 is triggered in debug mode.
 212  */
 213 #if defined(__GNUC__) && !defined(__clang__)
 214 #pragma GCC diagnostic ignored "-Wframe-larger-than="
 215 #endif
 216 DEFINE_REC_METHODS(aarch64_neonx2);
 217 #if defined(__GNUC__) && !defined(__clang__)
 218 #pragma GCC diagnostic pop
 219 #endif
 220
 221 static boolean_t
 222 raidz_will_aarch64_neonx2_work(void)
 223 {
 224         return (kfpu_allowed());
 225 }
 226
 227 const raidz_impl_ops_t vdev_raidz_aarch64_neonx2_impl = {
 228         .init = NULL,
 229         .fini = NULL,
 230         .gen = RAIDZ_GEN_METHODS(aarch64_neonx2),
 231         .rec = RAIDZ_REC_METHODS(aarch64_neonx2),
 232         .is_supported = &raidz_will_aarch64_neonx2_work,
 233         .name = "aarch64_neonx2"
 234 };
 235
 236 #endif /* defined(__aarch64__) */