gcc/config/aarch64/aarch64-simd-pragma-builtins.def

   1 /* AArch64 SIMD pragma builtins
   2    Copyright (C) 2024-2025 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 #undef ENTRY_BINARY
  22 #define ENTRY_BINARY(N, T0, T1, T2, U, F)       \
  23   ENTRY (N, binary, T0, T1, T2, none, U, F)
  24
  25 #undef ENTRY_BINARY_LANE
  26 #define ENTRY_BINARY_LANE(N, T0, T1, T2, U, F)  \
  27   ENTRY (N, binary_lane, T0, T1, T2, none, U, F)
  28
  29 #undef ENTRY_BINARY_TWO_LANES
  30 #define ENTRY_BINARY_TWO_LANES(N, T0, T1, T2, U, F)     \
  31   ENTRY (N, binary_two_lanes, T0, T1, T2, none, U, F)
  32
  33 #undef ENTRY_LOAD
  34 #define ENTRY_LOAD(N, T0, T1, U)                        \
  35   ENTRY (N, load, T0, T1, none, none, U, LOAD)
  36
  37 #undef ENTRY_LOAD_LANE
  38 #define ENTRY_LOAD_LANE(N, T0, T1, T2, U)               \
  39   ENTRY (N, load_lane, T0, T1, T2, none, U, LOAD)
  40
  41 #undef ENTRY_STORE
  42 #define ENTRY_STORE(N, T0, T1, U)                       \
  43   ENTRY (N, store, T0, T1, none, none, U, STORE)
  44
  45 #undef ENTRY_STORE_LANE
  46 #define ENTRY_STORE_LANE(N, T0, T1, U)                  \
  47   ENTRY (N, store_lane, T0, T1, none, none, U, STORE)
  48
  49 #undef ENTRY_TERNARY
  50 #define ENTRY_TERNARY(N, T0, T1, T2, T3, U, F)  \
  51   ENTRY (N, ternary, T0, T1, T2, T3, U, F)
  52
  53 #undef ENTRY_TERNARY_LANE
  54 #define ENTRY_TERNARY_LANE(N, T0, T1, T2, T3, U, F)     \
  55   ENTRY (N, ternary_lane, T0, T1, T2, T3, U, F)
  56
  57 #undef ENTRY_UNARY
  58 #define ENTRY_UNARY(N, T0, T1, U, F)    \
  59   ENTRY (N, unary, T0, T1, none, none, U, F)
  60
  61 #undef ENTRY_UNARY_LANE
  62 #define ENTRY_UNARY_LANE(N, T0, T1, U, F)       \
  63   ENTRY (N, unary_lane, T0, T1, none, none, U, F)
  64
  65 #undef ENTRY_BINARY_VHSDF
  66 #define ENTRY_BINARY_VHSDF(NAME, UNSPEC, FLAGS)                 \
  67   ENTRY_BINARY (NAME##_f16, f16, f16, f16, UNSPEC, FLAGS)       \
  68   ENTRY_BINARY (NAME##q_f16, f16q, f16q, f16q, UNSPEC, FLAGS)   \
  69   ENTRY_BINARY (NAME##_f32, f32, f32, f32, UNSPEC, FLAGS)       \
  70   ENTRY_BINARY (NAME##q_f32, f32q, f32q, f32q, UNSPEC, FLAGS)   \
  71   ENTRY_BINARY (NAME##q_f64, f64q, f64q, f64q, UNSPEC, FLAGS)
  72
  73 #undef ENTRY_BINARY_VHSDF_SIGNED
  74 #define ENTRY_BINARY_VHSDF_SIGNED(NAME, UNSPEC, FLAGS)          \
  75   ENTRY_BINARY (NAME##_f16, f16, f16, s16, UNSPEC, FLAGS)       \
  76   ENTRY_BINARY (NAME##q_f16, f16q, f16q, s16q, UNSPEC, FLAGS)   \
  77   ENTRY_BINARY (NAME##_f32, f32, f32, s32, UNSPEC, FLAGS)       \
  78   ENTRY_BINARY (NAME##q_f32, f32q, f32q, s32q, UNSPEC, FLAGS)   \
  79   ENTRY_BINARY (NAME##q_f64, f64q, f64q, s64q, UNSPEC, FLAGS)
  80
  81 #undef ENTRY_TERNARY_VLUT8
  82 #define ENTRY_TERNARY_VLUT8(T)                                  \
  83   ENTRY_BINARY_LANE (vluti2_lane_##T##8, T##8q, T##8, u8,       \
  84                      UNSPEC_LUTI2, QUIET)                       \
  85   ENTRY_BINARY_LANE (vluti2_laneq_##T##8, T##8q, T##8, u8q,     \
  86                      UNSPEC_LUTI2, QUIET)                       \
  87   ENTRY_BINARY_LANE (vluti2q_lane_##T##8, T##8q, T##8q, u8,     \
  88                      UNSPEC_LUTI2, QUIET)                       \
  89   ENTRY_BINARY_LANE (vluti2q_laneq_##T##8, T##8q, T##8q, u8q,   \
  90                      UNSPEC_LUTI2, QUIET)                       \
  91   ENTRY_BINARY_LANE (vluti4q_lane_##T##8, T##8q, T##8q, u8,     \
  92                      UNSPEC_LUTI4, QUIET)                       \
  93   ENTRY_BINARY_LANE (vluti4q_laneq_##T##8, T##8q, T##8q, u8q,   \
  94                      UNSPEC_LUTI4, QUIET)
  95
  96 #undef ENTRY_TERNARY_VLUT16
  97 #define ENTRY_TERNARY_VLUT16(T)                                         \
  98   ENTRY_BINARY_LANE (vluti2_lane_##T##16, T##16q, T##16, u8,            \
  99                      UNSPEC_LUTI2, QUIET)                               \
 100   ENTRY_BINARY_LANE (vluti2_laneq_##T##16, T##16q, T##16, u8q,          \
 101                      UNSPEC_LUTI2, QUIET)                               \
 102   ENTRY_BINARY_LANE (vluti2q_lane_##T##16, T##16q, T##16q, u8,          \
 103                      UNSPEC_LUTI2, QUIET)                               \
 104   ENTRY_BINARY_LANE (vluti2q_laneq_##T##16, T##16q, T##16q, u8q,        \
 105                      UNSPEC_LUTI2, QUIET)                               \
 106   ENTRY_BINARY_LANE (vluti4q_lane_##T##16_x2, T##16q, T##16qx2, u8,     \
 107                      UNSPEC_LUTI4, QUIET)                               \
 108   ENTRY_BINARY_LANE (vluti4q_laneq_##T##16_x2, T##16q, T##16qx2, u8q,   \
 109                      UNSPEC_LUTI4, QUIET)
 110
 111 #undef ENTRY_UNARY_VQ_BHF
 112 #define ENTRY_UNARY_VQ_BHF(N, T1, UNSPEC, FLAGS)                \
 113   ENTRY_UNARY (N##_bf16_mf8_fpm, bf16q, T1, UNSPEC, FLAGS)      \
 114   ENTRY_UNARY (N##_f16_mf8_fpm, f16q, T1, UNSPEC, FLAGS)
 115
 116 #undef ENTRY_VDOT_FPM
 117 #define ENTRY_VDOT_FPM(T)                                               \
 118   ENTRY_TERNARY (vdot_##T##_mf8_fpm, T, T, mf8, mf8,                    \
 119                  UNSPEC_FDOT_FP8, FP8)                                  \
 120   ENTRY_TERNARY (vdotq_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q,           \
 121                  UNSPEC_FDOT_FP8, FP8)                                  \
 122   ENTRY_TERNARY_LANE (vdot_lane_##T##_mf8_fpm, T, T, mf8, mf8,          \
 123                       UNSPEC_FDOT_LANE_FP8, FP8)                        \
 124   ENTRY_TERNARY_LANE (vdot_laneq_##T##_mf8_fpm, T, T, mf8, mf8q,        \
 125                       UNSPEC_FDOT_LANE_FP8, FP8)                        \
 126   ENTRY_TERNARY_LANE (vdotq_lane_##T##_mf8_fpm, T##q, T##q, mf8q, mf8,  \
 127                       UNSPEC_FDOT_LANE_FP8, FP8)                        \
 128   ENTRY_TERNARY_LANE (vdotq_laneq_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q,\
 129                       UNSPEC_FDOT_LANE_FP8, FP8)
 130
 131 #undef ENTRY_FMA_FPM
 132 #define ENTRY_FMA_FPM(N, T, U)                                          \
 133   ENTRY_TERNARY (N##q_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q, U, FP8)    \
 134   ENTRY_TERNARY_LANE (N##q_lane_##T##_mf8_fpm, T##q, T##q, mf8q, mf8, U, FP8) \
 135   ENTRY_TERNARY_LANE (N##q_laneq_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q, U, FP8)
 136
 137 // faminmax
 138 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FAMINMAX)
 139 ENTRY_BINARY_VHSDF (vamax, UNSPEC_FAMAX, FP)
 140 ENTRY_BINARY_VHSDF (vamin, UNSPEC_FAMIN, FP)
 141 #undef REQUIRED_EXTENSIONS
 142
 143 // lut
 144 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_LUT)
 145 ENTRY_TERNARY_VLUT8 (p)
 146 ENTRY_TERNARY_VLUT8 (s)
 147 ENTRY_TERNARY_VLUT8 (u)
 148 ENTRY_TERNARY_VLUT8 (mf)
 149
 150 ENTRY_TERNARY_VLUT16 (bf)
 151 ENTRY_TERNARY_VLUT16 (f)
 152 ENTRY_TERNARY_VLUT16 (p)
 153 ENTRY_TERNARY_VLUT16 (s)
 154 ENTRY_TERNARY_VLUT16 (u)
 155 #undef REQUIRED_EXTENSIONS
 156
 157 // fpm conversion
 158 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8)
 159 ENTRY_UNARY_VQ_BHF (vcvt1, mf8, UNSPEC_F1CVTL_FP8, FP8)
 160 ENTRY_UNARY_VQ_BHF (vcvt1_high, mf8q, UNSPEC_F1CVTL2_FP8, FP8)
 161 ENTRY_UNARY_VQ_BHF (vcvt1_low, mf8q, UNSPEC_F1CVTL_FP8, FP8)
 162 ENTRY_UNARY_VQ_BHF (vcvt2, mf8, UNSPEC_F2CVTL_FP8, FP8)
 163 ENTRY_UNARY_VQ_BHF (vcvt2_high, mf8q, UNSPEC_F2CVTL2_FP8, FP8)
 164 ENTRY_UNARY_VQ_BHF (vcvt2_low, mf8q, UNSPEC_F2CVTL_FP8, FP8)
 165
 166 ENTRY_BINARY (vcvt_mf8_f16_fpm, mf8, f16, f16, UNSPEC_FCVTN_FP8, FP8)
 167 ENTRY_BINARY (vcvtq_mf8_f16_fpm, mf8q, f16q, f16q, UNSPEC_FCVTN_FP8, FP8)
 168 ENTRY_BINARY (vcvt_mf8_f32_fpm, mf8, f32q, f32q, UNSPEC_FCVTN_FP8, FP8)
 169
 170 ENTRY_TERNARY (vcvt_high_mf8_f32_fpm, mf8q, mf8, f32q, f32q,
 171                UNSPEC_FCVTN2_FP8, FP8)
 172 #undef REQUIRED_EXTENSIONS
 173
 174 // fpm scaling
 175 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8)
 176 ENTRY_BINARY_VHSDF_SIGNED (vscale, UNSPEC_FSCALE, FP)
 177 #undef REQUIRED_EXTENSIONS
 178
 179 // fpm dot2 product
 180 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8DOT2)
 181 ENTRY_VDOT_FPM (f16)
 182 #undef REQUIRED_EXTENSIONS
 183
 184 // fpm dot4 product
 185 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8DOT4)
 186 ENTRY_VDOT_FPM (f32)
 187 #undef REQUIRED_EXTENSIONS
 188
 189 // fp8 multiply-add
 190 #define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8FMA)
 191 ENTRY_FMA_FPM (vmlalb, f16, UNSPEC_FMLALB_FP8)
 192 ENTRY_FMA_FPM (vmlalt, f16, UNSPEC_FMLALT_FP8)
 193 ENTRY_FMA_FPM (vmlallbb, f32, UNSPEC_FMLALLBB_FP8)
 194 ENTRY_FMA_FPM (vmlallbt, f32, UNSPEC_FMLALLBT_FP8)
 195 ENTRY_FMA_FPM (vmlalltb, f32, UNSPEC_FMLALLTB_FP8)
 196 ENTRY_FMA_FPM (vmlalltt, f32, UNSPEC_FMLALLTT_FP8)
 197 #undef REQUIRED_EXTENSIONS
 198
 199 // bsl
 200 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 201 ENTRY_TERNARY (vbsl_mf8, mf8, u8, mf8, mf8, UNSPEC_BSL, QUIET)
 202 ENTRY_TERNARY (vbslq_mf8, mf8q, u8q, mf8q, mf8q, UNSPEC_BSL, QUIET)
 203 #undef REQUIRED_EXTENSIONS
 204
 205 // combine
 206 #define REQUIRED_EXTENSIONS nonstreaming_only (NONE)
 207 ENTRY_BINARY (vcombine_mf8, mf8q, mf8, mf8, UNSPEC_COMBINE, QUIET)
 208 #undef REQUIRED_EXTENSIONS
 209
 210 // copy_lane
 211 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 212 ENTRY_BINARY_TWO_LANES (vcopy_lane_mf8, mf8, mf8, mf8,
 213                         UNSPEC_VEC_COPY, QUIET)
 214 ENTRY_BINARY_TWO_LANES (vcopyq_lane_mf8, mf8q, mf8q, mf8,
 215                         UNSPEC_VEC_COPY, QUIET)
 216 ENTRY_BINARY_TWO_LANES (vcopy_laneq_mf8, mf8, mf8, mf8q,
 217                         UNSPEC_VEC_COPY, QUIET)
 218 ENTRY_BINARY_TWO_LANES (vcopyq_laneq_mf8, mf8q, mf8q, mf8q,
 219                         UNSPEC_VEC_COPY, QUIET)
 220 #undef REQUIRED_EXTENSIONS
 221
 222 // create
 223 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 224 ENTRY_UNARY (vcreate_mf8, mf8, u64_scalar, UNSPEC_VCREATE, QUIET)
 225 #undef REQUIRED_EXTENSIONS
 226
 227 // dup
 228 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 229 ENTRY_UNARY (vdup_n_mf8, mf8, mf8_scalar, UNSPEC_DUP, QUIET)
 230 ENTRY_UNARY (vdupq_n_mf8, mf8q, mf8_scalar, UNSPEC_DUP, QUIET)
 231
 232 ENTRY_UNARY_LANE (vdup_lane_mf8, mf8, mf8, UNSPEC_DUP_LANE, QUIET)
 233 ENTRY_UNARY_LANE (vdupq_lane_mf8, mf8q, mf8, UNSPEC_DUP_LANE, QUIET)
 234 ENTRY_UNARY_LANE (vdup_laneq_mf8, mf8, mf8q, UNSPEC_DUP_LANE, QUIET)
 235 ENTRY_UNARY_LANE (vdupq_laneq_mf8, mf8q, mf8q, UNSPEC_DUP_LANE, QUIET)
 236 #undef REQUIRED_EXTENSIONS
 237
 238 // dupb_lane
 239 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 240 ENTRY_UNARY_LANE (vdupb_lane_mf8, mf8_scalar, mf8, UNSPEC_GET_LANE, QUIET)
 241 ENTRY_UNARY_LANE (vdupb_laneq_mf8, mf8_scalar, mf8q, UNSPEC_GET_LANE, QUIET)
 242 #undef REQUIRED_EXTENSIONS
 243
 244 // ext
 245 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 246 ENTRY_BINARY_LANE (vext_mf8, mf8, mf8, mf8, UNSPEC_EXT, QUIET)
 247 ENTRY_BINARY_LANE (vextq_mf8, mf8q, mf8q, mf8q, UNSPEC_EXT, QUIET)
 248 #undef REQUIRED_EXTENSIONS
 249
 250 // ld1
 251 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 252 ENTRY_LOAD (vld1_mf8, mf8, mf8_scalar_const_ptr, UNSPEC_LD1)
 253 ENTRY_LOAD (vld1q_mf8, mf8q, mf8_scalar_const_ptr, UNSPEC_LD1)
 254 ENTRY_LOAD (vld1_dup_mf8, mf8, mf8_scalar_const_ptr, UNSPEC_DUP)
 255 ENTRY_LOAD (vld1q_dup_mf8, mf8q, mf8_scalar_const_ptr, UNSPEC_DUP)
 256
 257 ENTRY_LOAD_LANE (vld1_lane_mf8, mf8, mf8_scalar_const_ptr, mf8,
 258                  UNSPEC_SET_LANE)
 259 ENTRY_LOAD_LANE (vld1q_lane_mf8, mf8q, mf8_scalar_const_ptr, mf8q,
 260                  UNSPEC_SET_LANE)
 261 #undef REQUIRED_EXTENSIONS
 262
 263 // ld<n>
 264 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 265 ENTRY_LOAD (vld1_mf8_x2, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD1x2)
 266 ENTRY_LOAD (vld1q_mf8_x2, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD1x2)
 267 ENTRY_LOAD (vld2_mf8, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD2)
 268 ENTRY_LOAD (vld2q_mf8, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD2)
 269 ENTRY_LOAD (vld2_dup_mf8, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD2_DUP)
 270 ENTRY_LOAD (vld2q_dup_mf8, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD2_DUP)
 271 ENTRY_LOAD_LANE (vld2_lane_mf8, mf8x2, mf8_scalar_const_ptr, mf8x2,
 272                  UNSPEC_LD2_LANE)
 273 ENTRY_LOAD_LANE (vld2q_lane_mf8, mf8qx2, mf8_scalar_const_ptr, mf8qx2,
 274                  UNSPEC_LD2_LANE)
 275
 276 ENTRY_LOAD (vld1_mf8_x3, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD1x3)
 277 ENTRY_LOAD (vld1q_mf8_x3, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD1x3)
 278 ENTRY_LOAD (vld3_mf8, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD3)
 279 ENTRY_LOAD (vld3q_mf8, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD3)
 280 ENTRY_LOAD (vld3_dup_mf8, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD3_DUP)
 281 ENTRY_LOAD (vld3q_dup_mf8, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD3_DUP)
 282 ENTRY_LOAD_LANE (vld3_lane_mf8, mf8x3, mf8_scalar_const_ptr, mf8x3,
 283                  UNSPEC_LD3_LANE)
 284 ENTRY_LOAD_LANE (vld3q_lane_mf8, mf8qx3, mf8_scalar_const_ptr, mf8qx3,
 285                  UNSPEC_LD3_LANE)
 286
 287 ENTRY_LOAD (vld1_mf8_x4, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD1x4)
 288 ENTRY_LOAD (vld1q_mf8_x4, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD1x4)
 289 ENTRY_LOAD (vld4_mf8, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD4)
 290 ENTRY_LOAD (vld4q_mf8, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD4)
 291 ENTRY_LOAD (vld4_dup_mf8, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD4_DUP)
 292 ENTRY_LOAD (vld4q_dup_mf8, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD4_DUP)
 293 ENTRY_LOAD_LANE (vld4_lane_mf8, mf8x4, mf8_scalar_const_ptr, mf8x4,
 294                  UNSPEC_LD4_LANE)
 295 ENTRY_LOAD_LANE (vld4q_lane_mf8, mf8qx4, mf8_scalar_const_ptr, mf8qx4,
 296                  UNSPEC_LD4_LANE)
 297 #undef REQUIRED_EXTENSIONS
 298
 299 // mov
 300 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 301 ENTRY_UNARY (vmov_n_mf8, mf8, mf8_scalar, UNSPEC_DUP, QUIET)
 302 ENTRY_UNARY (vmovq_n_mf8, mf8q, mf8_scalar, UNSPEC_DUP, QUIET)
 303 #undef REQUIRED_EXTENSIONS
 304
 305 // rev
 306 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 307 ENTRY_UNARY (vrev64_mf8, mf8, mf8, UNSPEC_REV64, QUIET)
 308 ENTRY_UNARY (vrev64q_mf8, mf8q, mf8q, UNSPEC_REV64, QUIET)
 309
 310 ENTRY_UNARY (vrev32_mf8, mf8, mf8, UNSPEC_REV32, QUIET)
 311 ENTRY_UNARY (vrev32q_mf8, mf8q, mf8q, UNSPEC_REV32, QUIET)
 312
 313 ENTRY_UNARY (vrev16_mf8, mf8, mf8, UNSPEC_REV16, QUIET)
 314 ENTRY_UNARY (vrev16q_mf8, mf8q, mf8q, UNSPEC_REV16, QUIET)
 315 #undef REQUIRED_EXTENSIONS
 316
 317 // set_lane
 318 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 319 ENTRY_BINARY_LANE (vset_lane_mf8, mf8, mf8_scalar, mf8, UNSPEC_SET_LANE, QUIET)
 320 ENTRY_BINARY_LANE (vsetq_lane_mf8, mf8q, mf8_scalar, mf8q, UNSPEC_SET_LANE, QUIET)
 321 #undef REQUIRED_EXTENSIONS
 322
 323 // st1
 324 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 325 ENTRY_STORE (vst1_mf8, mf8_scalar_ptr, mf8, UNSPEC_ST1)
 326 ENTRY_STORE (vst1q_mf8, mf8_scalar_ptr, mf8q, UNSPEC_ST1)
 327
 328 ENTRY_STORE_LANE (vst1_lane_mf8, mf8_scalar_ptr, mf8, UNSPEC_ST1_LANE)
 329 ENTRY_STORE_LANE (vst1q_lane_mf8, mf8_scalar_ptr, mf8q, UNSPEC_ST1_LANE)
 330 #undef REQUIRED_EXTENSIONS
 331
 332 // st<n>
 333 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 334 ENTRY_STORE (vst2_mf8, mf8_scalar_ptr, mf8x2, UNSPEC_ST2)
 335 ENTRY_STORE (vst2q_mf8, mf8_scalar_ptr, mf8qx2, UNSPEC_ST2)
 336 ENTRY_STORE (vst1_mf8_x2, mf8_scalar_ptr, mf8x2, UNSPEC_ST1x2)
 337 ENTRY_STORE (vst1q_mf8_x2, mf8_scalar_ptr, mf8qx2, UNSPEC_ST1x2)
 338 ENTRY_STORE_LANE (vst2_lane_mf8, mf8_scalar_ptr, mf8x2, UNSPEC_ST2_LANE)
 339 ENTRY_STORE_LANE (vst2q_lane_mf8, mf8_scalar_ptr, mf8qx2, UNSPEC_ST2_LANE)
 340
 341 ENTRY_STORE (vst3_mf8, mf8_scalar_ptr, mf8x3, UNSPEC_ST3)
 342 ENTRY_STORE (vst3q_mf8, mf8_scalar_ptr, mf8qx3, UNSPEC_ST3)
 343 ENTRY_STORE (vst1_mf8_x3, mf8_scalar_ptr, mf8x3, UNSPEC_ST1x3)
 344 ENTRY_STORE (vst1q_mf8_x3, mf8_scalar_ptr, mf8qx3, UNSPEC_ST1x3)
 345 ENTRY_STORE_LANE (vst3_lane_mf8, mf8_scalar_ptr, mf8x3, UNSPEC_ST3_LANE)
 346 ENTRY_STORE_LANE (vst3q_lane_mf8, mf8_scalar_ptr, mf8qx3, UNSPEC_ST3_LANE)
 347
 348 ENTRY_STORE (vst4_mf8, mf8_scalar_ptr, mf8x4, UNSPEC_ST4)
 349 ENTRY_STORE (vst4q_mf8, mf8_scalar_ptr, mf8qx4, UNSPEC_ST4)
 350 ENTRY_STORE (vst1_mf8_x4, mf8_scalar_ptr, mf8x4, UNSPEC_ST1x4)
 351 ENTRY_STORE (vst1q_mf8_x4, mf8_scalar_ptr, mf8qx4, UNSPEC_ST1x4)
 352 ENTRY_STORE_LANE (vst4_lane_mf8, mf8_scalar_ptr, mf8x4, UNSPEC_ST4_LANE)
 353 ENTRY_STORE_LANE (vst4q_lane_mf8, mf8_scalar_ptr, mf8qx4, UNSPEC_ST4_LANE)
 354 #undef REQUIRED_EXTENSIONS
 355
 356 // tbl<n>
 357 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 358 ENTRY_BINARY (vtbl1_mf8, mf8, mf8, u8, UNSPEC_TBL, QUIET)
 359 ENTRY_BINARY (vtbl2_mf8, mf8, mf8x2, u8, UNSPEC_TBL, QUIET)
 360 ENTRY_BINARY (vtbl3_mf8, mf8, mf8x3, u8, UNSPEC_TBL, QUIET)
 361 ENTRY_BINARY (vtbl4_mf8, mf8, mf8x4, u8, UNSPEC_TBL, QUIET)
 362
 363 ENTRY_BINARY (vqtbl1_mf8, mf8, mf8q, u8, UNSPEC_TBL, QUIET)
 364 ENTRY_BINARY (vqtbl1q_mf8, mf8q, mf8q, u8q, UNSPEC_TBL, QUIET)
 365 ENTRY_BINARY (vqtbl2_mf8, mf8, mf8qx2, u8, UNSPEC_TBL, QUIET)
 366 ENTRY_BINARY (vqtbl2q_mf8, mf8q, mf8qx2, u8q, UNSPEC_TBL, QUIET)
 367 ENTRY_BINARY (vqtbl3_mf8, mf8, mf8qx3, u8, UNSPEC_TBL, QUIET)
 368 ENTRY_BINARY (vqtbl3q_mf8, mf8q, mf8qx3, u8q, UNSPEC_TBL, QUIET)
 369 ENTRY_BINARY (vqtbl4_mf8, mf8, mf8qx4, u8, UNSPEC_TBL, QUIET)
 370 ENTRY_BINARY (vqtbl4q_mf8, mf8q, mf8qx4, u8q, UNSPEC_TBL, QUIET)
 371 #undef REQUIRED_EXTENSIONS
 372
 373 // tbx<n>
 374 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 375 ENTRY_TERNARY (vtbx1_mf8, mf8, mf8, mf8, u8, UNSPEC_TBX, QUIET)
 376 ENTRY_TERNARY (vtbx2_mf8, mf8, mf8, mf8x2, u8, UNSPEC_TBX, QUIET)
 377 ENTRY_TERNARY (vtbx3_mf8, mf8, mf8, mf8x3, u8, UNSPEC_TBX, QUIET)
 378 ENTRY_TERNARY (vtbx4_mf8, mf8, mf8, mf8x4, u8, UNSPEC_TBX, QUIET)
 379
 380 ENTRY_TERNARY (vqtbx1_mf8, mf8, mf8, mf8q, u8, UNSPEC_TBX, QUIET)
 381 ENTRY_TERNARY (vqtbx1q_mf8, mf8q, mf8q, mf8q, u8q, UNSPEC_TBX, QUIET)
 382 ENTRY_TERNARY (vqtbx2_mf8, mf8, mf8, mf8qx2, u8, UNSPEC_TBX, QUIET)
 383 ENTRY_TERNARY (vqtbx2q_mf8, mf8q, mf8q, mf8qx2, u8q, UNSPEC_TBX, QUIET)
 384 ENTRY_TERNARY (vqtbx3_mf8, mf8, mf8, mf8qx3, u8, UNSPEC_TBX, QUIET)
 385 ENTRY_TERNARY (vqtbx3q_mf8, mf8q, mf8q, mf8qx3, u8q, UNSPEC_TBX, QUIET)
 386 ENTRY_TERNARY (vqtbx4_mf8, mf8, mf8, mf8qx4, u8, UNSPEC_TBX, QUIET)
 387 ENTRY_TERNARY (vqtbx4q_mf8, mf8q, mf8q, mf8qx4, u8q, UNSPEC_TBX, QUIET)
 388 #undef REQUIRED_EXTENSIONS
 389
 390 // trn<n>
 391 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 392 ENTRY_BINARY (vtrn1_mf8, mf8, mf8, mf8, UNSPEC_TRN1, QUIET)
 393 ENTRY_BINARY (vtrn1q_mf8, mf8q, mf8q, mf8q, UNSPEC_TRN1, QUIET)
 394 ENTRY_BINARY (vtrn2_mf8, mf8, mf8, mf8, UNSPEC_TRN2, QUIET)
 395 ENTRY_BINARY (vtrn2q_mf8, mf8q, mf8q, mf8q, UNSPEC_TRN2, QUIET)
 396 ENTRY_BINARY (vtrn_mf8, mf8x2, mf8, mf8, UNSPEC_TRN, QUIET)
 397 ENTRY_BINARY (vtrnq_mf8, mf8qx2, mf8q, mf8q, UNSPEC_TRN, QUIET)
 398 #undef REQUIRED_EXTENSIONS
 399
 400 // uzp<n>
 401 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 402 ENTRY_BINARY (vuzp1_mf8, mf8, mf8, mf8, UNSPEC_UZP1, QUIET)
 403 ENTRY_BINARY (vuzp1q_mf8, mf8q, mf8q, mf8q, UNSPEC_UZP1, QUIET)
 404 ENTRY_BINARY (vuzp2_mf8, mf8, mf8, mf8, UNSPEC_UZP2, QUIET)
 405 ENTRY_BINARY (vuzp2q_mf8, mf8q, mf8q, mf8q, UNSPEC_UZP2, QUIET)
 406 ENTRY_BINARY (vuzp_mf8, mf8x2, mf8, mf8, UNSPEC_UZP, QUIET)
 407 ENTRY_BINARY (vuzpq_mf8, mf8qx2, mf8q, mf8q, UNSPEC_UZP, QUIET)
 408 #undef REQUIRED_EXTENSIONS
 409
 410 // zip<n>
 411 #define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
 412 ENTRY_BINARY (vzip1_mf8, mf8, mf8, mf8, UNSPEC_ZIP1, QUIET)
 413 ENTRY_BINARY (vzip1q_mf8, mf8q, mf8q, mf8q, UNSPEC_ZIP1, QUIET)
 414 ENTRY_BINARY (vzip2_mf8, mf8, mf8, mf8, UNSPEC_ZIP2, QUIET)
 415 ENTRY_BINARY (vzip2q_mf8, mf8q, mf8q, mf8q, UNSPEC_ZIP2, QUIET)
 416 ENTRY_BINARY (vzip_mf8, mf8x2, mf8, mf8, UNSPEC_ZIP, QUIET)
 417 ENTRY_BINARY (vzipq_mf8, mf8qx2, mf8q, mf8q, UNSPEC_ZIP, QUIET)
 418 #undef REQUIRED_EXTENSIONS