clang/test/CodeGen/arm-mve-intrinsics/scatter-gather.c

   1 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
   2 // RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
   3 // RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
   4
   5 // REQUIRES: aarch64-registered-target || arm-registered-target
   6
   7 #include <arm_mve.h>
   8
   9 // CHECK-LABEL: @test_vldrbq_gather_offset_s16(
  10 // CHECK-NEXT:  entry:
  11 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 8, i32 0, i32 0)
  12 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
  13 //
  14 int16x8_t test_vldrbq_gather_offset_s16(const int8_t *base, uint16x8_t offset)
  15 {
  16 #ifdef POLYMORPHIC
  17     return vldrbq_gather_offset(base, offset);
  18 #else /* POLYMORPHIC */
  19     return vldrbq_gather_offset_s16(base, offset);
  20 #endif /* POLYMORPHIC */
  21 }
  22
  23 // CHECK-LABEL: @test_vldrbq_gather_offset_s32(
  24 // CHECK-NEXT:  entry:
  25 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 8, i32 0, i32 0)
  26 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
  27 //
  28 int32x4_t test_vldrbq_gather_offset_s32(const int8_t *base, uint32x4_t offset)
  29 {
  30 #ifdef POLYMORPHIC
  31     return vldrbq_gather_offset(base, offset);
  32 #else /* POLYMORPHIC */
  33     return vldrbq_gather_offset_s32(base, offset);
  34 #endif /* POLYMORPHIC */
  35 }
  36
  37 // CHECK-LABEL: @test_vldrbq_gather_offset_s8(
  38 // CHECK-NEXT:  entry:
  39 // CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.v16i8.p0.v16i8(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], i32 8, i32 0, i32 0)
  40 // CHECK-NEXT:    ret <16 x i8> [[TMP0]]
  41 //
  42 int8x16_t test_vldrbq_gather_offset_s8(const int8_t *base, uint8x16_t offset)
  43 {
  44 #ifdef POLYMORPHIC
  45     return vldrbq_gather_offset(base, offset);
  46 #else /* POLYMORPHIC */
  47     return vldrbq_gather_offset_s8(base, offset);
  48 #endif /* POLYMORPHIC */
  49 }
  50
  51 // CHECK-LABEL: @test_vldrbq_gather_offset_u16(
  52 // CHECK-NEXT:  entry:
  53 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 8, i32 0, i32 1)
  54 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
  55 //
  56 uint16x8_t test_vldrbq_gather_offset_u16(const uint8_t *base, uint16x8_t offset)
  57 {
  58 #ifdef POLYMORPHIC
  59     return vldrbq_gather_offset(base, offset);
  60 #else /* POLYMORPHIC */
  61     return vldrbq_gather_offset_u16(base, offset);
  62 #endif /* POLYMORPHIC */
  63 }
  64
  65 // CHECK-LABEL: @test_vldrbq_gather_offset_u32(
  66 // CHECK-NEXT:  entry:
  67 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 8, i32 0, i32 1)
  68 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
  69 //
  70 uint32x4_t test_vldrbq_gather_offset_u32(const uint8_t *base, uint32x4_t offset)
  71 {
  72 #ifdef POLYMORPHIC
  73     return vldrbq_gather_offset(base, offset);
  74 #else /* POLYMORPHIC */
  75     return vldrbq_gather_offset_u32(base, offset);
  76 #endif /* POLYMORPHIC */
  77 }
  78
  79 // CHECK-LABEL: @test_vldrbq_gather_offset_u8(
  80 // CHECK-NEXT:  entry:
  81 // CHECK-NEXT:    [[TMP0:%.*]] = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.v16i8.p0.v16i8(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], i32 8, i32 0, i32 1)
  82 // CHECK-NEXT:    ret <16 x i8> [[TMP0]]
  83 //
  84 uint8x16_t test_vldrbq_gather_offset_u8(const uint8_t *base, uint8x16_t offset)
  85 {
  86 #ifdef POLYMORPHIC
  87     return vldrbq_gather_offset(base, offset);
  88 #else /* POLYMORPHIC */
  89     return vldrbq_gather_offset_u8(base, offset);
  90 #endif /* POLYMORPHIC */
  91 }
  92
  93 // CHECK-LABEL: @test_vldrbq_gather_offset_z_s16(
  94 // CHECK-NEXT:  entry:
  95 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
  96 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
  97 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 8, i32 0, i32 0, <8 x i1> [[TMP1]])
  98 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
  99 //
 100 int16x8_t test_vldrbq_gather_offset_z_s16(const int8_t *base, uint16x8_t offset, mve_pred16_t p)
 101 {
 102 #ifdef POLYMORPHIC
 103     return vldrbq_gather_offset_z(base, offset, p);
 104 #else /* POLYMORPHIC */
 105     return vldrbq_gather_offset_z_s16(base, offset, p);
 106 #endif /* POLYMORPHIC */
 107 }
 108
 109 // CHECK-LABEL: @test_vldrbq_gather_offset_z_s32(
 110 // CHECK-NEXT:  entry:
 111 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 112 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 113 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 8, i32 0, i32 0, <4 x i1> [[TMP1]])
 114 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 115 //
 116 int32x4_t test_vldrbq_gather_offset_z_s32(const int8_t *base, uint32x4_t offset, mve_pred16_t p)
 117 {
 118 #ifdef POLYMORPHIC
 119     return vldrbq_gather_offset_z(base, offset, p);
 120 #else /* POLYMORPHIC */
 121     return vldrbq_gather_offset_z_s32(base, offset, p);
 122 #endif /* POLYMORPHIC */
 123 }
 124
 125 // CHECK-LABEL: @test_vldrbq_gather_offset_z_s8(
 126 // CHECK-NEXT:  entry:
 127 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 128 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
 129 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.predicated.v16i8.p0.v16i8.v16i1(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], i32 8, i32 0, i32 0, <16 x i1> [[TMP1]])
 130 // CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 131 //
 132 int8x16_t test_vldrbq_gather_offset_z_s8(const int8_t *base, uint8x16_t offset, mve_pred16_t p)
 133 {
 134 #ifdef POLYMORPHIC
 135     return vldrbq_gather_offset_z(base, offset, p);
 136 #else /* POLYMORPHIC */
 137     return vldrbq_gather_offset_z_s8(base, offset, p);
 138 #endif /* POLYMORPHIC */
 139 }
 140
 141 // CHECK-LABEL: @test_vldrbq_gather_offset_z_u16(
 142 // CHECK-NEXT:  entry:
 143 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 144 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 145 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 8, i32 0, i32 1, <8 x i1> [[TMP1]])
 146 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 147 //
 148 uint16x8_t test_vldrbq_gather_offset_z_u16(const uint8_t *base, uint16x8_t offset, mve_pred16_t p)
 149 {
 150 #ifdef POLYMORPHIC
 151     return vldrbq_gather_offset_z(base, offset, p);
 152 #else /* POLYMORPHIC */
 153     return vldrbq_gather_offset_z_u16(base, offset, p);
 154 #endif /* POLYMORPHIC */
 155 }
 156
 157 // CHECK-LABEL: @test_vldrbq_gather_offset_z_u32(
 158 // CHECK-NEXT:  entry:
 159 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 160 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 161 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 8, i32 0, i32 1, <4 x i1> [[TMP1]])
 162 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 163 //
 164 uint32x4_t test_vldrbq_gather_offset_z_u32(const uint8_t *base, uint32x4_t offset, mve_pred16_t p)
 165 {
 166 #ifdef POLYMORPHIC
 167     return vldrbq_gather_offset_z(base, offset, p);
 168 #else /* POLYMORPHIC */
 169     return vldrbq_gather_offset_z_u32(base, offset, p);
 170 #endif /* POLYMORPHIC */
 171 }
 172
 173 // CHECK-LABEL: @test_vldrbq_gather_offset_z_u8(
 174 // CHECK-NEXT:  entry:
 175 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 176 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
 177 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i8> @llvm.arm.mve.vldr.gather.offset.predicated.v16i8.p0.v16i8.v16i1(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], i32 8, i32 0, i32 1, <16 x i1> [[TMP1]])
 178 // CHECK-NEXT:    ret <16 x i8> [[TMP2]]
 179 //
 180 uint8x16_t test_vldrbq_gather_offset_z_u8(const uint8_t *base, uint8x16_t offset, mve_pred16_t p)
 181 {
 182 #ifdef POLYMORPHIC
 183     return vldrbq_gather_offset_z(base, offset, p);
 184 #else /* POLYMORPHIC */
 185     return vldrbq_gather_offset_z_u8(base, offset, p);
 186 #endif /* POLYMORPHIC */
 187 }
 188
 189 // CHECK-LABEL: @test_vldrdq_gather_base_s64(
 190 // CHECK-NEXT:  entry:
 191 // CHECK-NEXT:    [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.base.v2i64.v2i64(<2 x i64> [[ADDR:%.*]], i32 616)
 192 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 193 //
 194 int64x2_t test_vldrdq_gather_base_s64(uint64x2_t addr)
 195 {
 196     return vldrdq_gather_base_s64(addr, 0x268);
 197 }
 198
 199 // CHECK-LABEL: @test_vldrdq_gather_base_u64(
 200 // CHECK-NEXT:  entry:
 201 // CHECK-NEXT:    [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.base.v2i64.v2i64(<2 x i64> [[ADDR:%.*]], i32 -336)
 202 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 203 //
 204 uint64x2_t test_vldrdq_gather_base_u64(uint64x2_t addr)
 205 {
 206     return vldrdq_gather_base_u64(addr, -0x150);
 207 }
 208
 209 // CHECK-LABEL: @test_vldrdq_gather_base_wb_s64(
 210 // CHECK-NEXT:  entry:
 211 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
 212 // CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64> [[TMP0]], i32 576)
 213 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP1]], 1
 214 // CHECK-NEXT:    store <2 x i64> [[TMP2]], ptr [[ADDR]], align 8
 215 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP1]], 0
 216 // CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 217 //
 218 int64x2_t test_vldrdq_gather_base_wb_s64(uint64x2_t *addr)
 219 {
 220     return vldrdq_gather_base_wb_s64(addr, 0x240);
 221 }
 222
 223 // CHECK-LABEL: @test_vldrdq_gather_base_wb_u64(
 224 // CHECK-NEXT:  entry:
 225 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
 226 // CHECK-NEXT:    [[TMP1:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.v2i64.v2i64(<2 x i64> [[TMP0]], i32 -328)
 227 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP1]], 1
 228 // CHECK-NEXT:    store <2 x i64> [[TMP2]], ptr [[ADDR]], align 8
 229 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP1]], 0
 230 // CHECK-NEXT:    ret <2 x i64> [[TMP3]]
 231 //
 232 uint64x2_t test_vldrdq_gather_base_wb_u64(uint64x2_t *addr)
 233 {
 234     return vldrdq_gather_base_wb_u64(addr, -0x148);
 235 }
 236
 237 // CHECK-LABEL: @test_vldrdq_gather_base_wb_z_s64(
 238 // CHECK-NEXT:  entry:
 239 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
 240 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 241 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP1]])
 242 // CHECK-NEXT:    [[TMP3:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v2i1(<2 x i64> [[TMP0]], i32 664, <2 x i1> [[TMP2]])
 243 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP3]], 1
 244 // CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[ADDR]], align 8
 245 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP3]], 0
 246 // CHECK-NEXT:    ret <2 x i64> [[TMP5]]
 247 //
 248 int64x2_t test_vldrdq_gather_base_wb_z_s64(uint64x2_t *addr, mve_pred16_t p)
 249 {
 250     return vldrdq_gather_base_wb_z_s64(addr, 0x298, p);
 251 }
 252
 253 // CHECK-LABEL: @test_vldrdq_gather_base_wb_z_u64(
 254 // CHECK-NEXT:  entry:
 255 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
 256 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 257 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP1]])
 258 // CHECK-NEXT:    [[TMP3:%.*]] = call { <2 x i64>, <2 x i64> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v2i64.v2i64.v2i1(<2 x i64> [[TMP0]], i32 656, <2 x i1> [[TMP2]])
 259 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP3]], 1
 260 // CHECK-NEXT:    store <2 x i64> [[TMP4]], ptr [[ADDR]], align 8
 261 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[TMP3]], 0
 262 // CHECK-NEXT:    ret <2 x i64> [[TMP5]]
 263 //
 264 uint64x2_t test_vldrdq_gather_base_wb_z_u64(uint64x2_t *addr, mve_pred16_t p)
 265 {
 266     return vldrdq_gather_base_wb_z_u64(addr, 0x290, p);
 267 }
 268
 269 // CHECK-LABEL: @test_vldrdq_gather_base_z_s64(
 270 // CHECK-NEXT:  entry:
 271 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 272 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
 273 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v2i1(<2 x i64> [[ADDR:%.*]], i32 888, <2 x i1> [[TMP1]])
 274 // CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 275 //
 276 int64x2_t test_vldrdq_gather_base_z_s64(uint64x2_t addr, mve_pred16_t p)
 277 {
 278     return vldrdq_gather_base_z_s64(addr, 0x378, p);
 279 }
 280
 281 // CHECK-LABEL: @test_vldrdq_gather_base_z_u64(
 282 // CHECK-NEXT:  entry:
 283 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 284 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
 285 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.base.predicated.v2i64.v2i64.v2i1(<2 x i64> [[ADDR:%.*]], i32 -1000, <2 x i1> [[TMP1]])
 286 // CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 287 //
 288 uint64x2_t test_vldrdq_gather_base_z_u64(uint64x2_t addr, mve_pred16_t p)
 289 {
 290     return vldrdq_gather_base_z_u64(addr, -0x3e8, p);
 291 }
 292
 293 // CHECK-LABEL: @test_vldrdq_gather_offset_s64(
 294 // CHECK-NEXT:  entry:
 295 // CHECK-NEXT:    [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 0, i32 0)
 296 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 297 //
 298 int64x2_t test_vldrdq_gather_offset_s64(const int64_t *base, uint64x2_t offset)
 299 {
 300 #ifdef POLYMORPHIC
 301     return vldrdq_gather_offset(base, offset);
 302 #else /* POLYMORPHIC */
 303     return vldrdq_gather_offset_s64(base, offset);
 304 #endif /* POLYMORPHIC */
 305 }
 306
 307 // CHECK-LABEL: @test_vldrdq_gather_offset_u64(
 308 // CHECK-NEXT:  entry:
 309 // CHECK-NEXT:    [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 0, i32 1)
 310 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 311 //
 312 uint64x2_t test_vldrdq_gather_offset_u64(const uint64_t *base, uint64x2_t offset)
 313 {
 314 #ifdef POLYMORPHIC
 315     return vldrdq_gather_offset(base, offset);
 316 #else /* POLYMORPHIC */
 317     return vldrdq_gather_offset_u64(base, offset);
 318 #endif /* POLYMORPHIC */
 319 }
 320
 321 // CHECK-LABEL: @test_vldrdq_gather_offset_z_s64(
 322 // CHECK-NEXT:  entry:
 323 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 324 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
 325 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 0, i32 0, <2 x i1> [[TMP1]])
 326 // CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 327 //
 328 int64x2_t test_vldrdq_gather_offset_z_s64(const int64_t *base, uint64x2_t offset, mve_pred16_t p)
 329 {
 330 #ifdef POLYMORPHIC
 331     return vldrdq_gather_offset_z(base, offset, p);
 332 #else /* POLYMORPHIC */
 333     return vldrdq_gather_offset_z_s64(base, offset, p);
 334 #endif /* POLYMORPHIC */
 335 }
 336
 337 // CHECK-LABEL: @test_vldrdq_gather_offset_z_u64(
 338 // CHECK-NEXT:  entry:
 339 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 340 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
 341 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 0, i32 1, <2 x i1> [[TMP1]])
 342 // CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 343 //
 344 uint64x2_t test_vldrdq_gather_offset_z_u64(const uint64_t *base, uint64x2_t offset, mve_pred16_t p)
 345 {
 346 #ifdef POLYMORPHIC
 347     return vldrdq_gather_offset_z(base, offset, p);
 348 #else /* POLYMORPHIC */
 349     return vldrdq_gather_offset_z_u64(base, offset, p);
 350 #endif /* POLYMORPHIC */
 351 }
 352
 353 // CHECK-LABEL: @test_vldrdq_gather_shifted_offset_s64(
 354 // CHECK-NEXT:  entry:
 355 // CHECK-NEXT:    [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 3, i32 0)
 356 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 357 //
 358 int64x2_t test_vldrdq_gather_shifted_offset_s64(const int64_t *base, uint64x2_t offset)
 359 {
 360 #ifdef POLYMORPHIC
 361     return vldrdq_gather_shifted_offset(base, offset);
 362 #else /* POLYMORPHIC */
 363     return vldrdq_gather_shifted_offset_s64(base, offset);
 364 #endif /* POLYMORPHIC */
 365 }
 366
 367 // CHECK-LABEL: @test_vldrdq_gather_shifted_offset_u64(
 368 // CHECK-NEXT:  entry:
 369 // CHECK-NEXT:    [[TMP0:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.v2i64.p0.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 3, i32 1)
 370 // CHECK-NEXT:    ret <2 x i64> [[TMP0]]
 371 //
 372 uint64x2_t test_vldrdq_gather_shifted_offset_u64(const uint64_t *base, uint64x2_t offset)
 373 {
 374 #ifdef POLYMORPHIC
 375     return vldrdq_gather_shifted_offset(base, offset);
 376 #else /* POLYMORPHIC */
 377     return vldrdq_gather_shifted_offset_u64(base, offset);
 378 #endif /* POLYMORPHIC */
 379 }
 380
 381 // CHECK-LABEL: @test_vldrdq_gather_shifted_offset_z_s64(
 382 // CHECK-NEXT:  entry:
 383 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 384 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
 385 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 3, i32 0, <2 x i1> [[TMP1]])
 386 // CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 387 //
 388 int64x2_t test_vldrdq_gather_shifted_offset_z_s64(const int64_t *base, uint64x2_t offset, mve_pred16_t p)
 389 {
 390 #ifdef POLYMORPHIC
 391     return vldrdq_gather_shifted_offset_z(base, offset, p);
 392 #else /* POLYMORPHIC */
 393     return vldrdq_gather_shifted_offset_z_s64(base, offset, p);
 394 #endif /* POLYMORPHIC */
 395 }
 396
 397 // CHECK-LABEL: @test_vldrdq_gather_shifted_offset_z_u64(
 398 // CHECK-NEXT:  entry:
 399 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 400 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
 401 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i64> @llvm.arm.mve.vldr.gather.offset.predicated.v2i64.p0.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], i32 64, i32 3, i32 1, <2 x i1> [[TMP1]])
 402 // CHECK-NEXT:    ret <2 x i64> [[TMP2]]
 403 //
 404 uint64x2_t test_vldrdq_gather_shifted_offset_z_u64(const uint64_t *base, uint64x2_t offset, mve_pred16_t p)
 405 {
 406 #ifdef POLYMORPHIC
 407     return vldrdq_gather_shifted_offset_z(base, offset, p);
 408 #else /* POLYMORPHIC */
 409     return vldrdq_gather_shifted_offset_z_u64(base, offset, p);
 410 #endif /* POLYMORPHIC */
 411 }
 412
 413 // CHECK-LABEL: @test_vldrhq_gather_offset_f16(
 414 // CHECK-NEXT:  entry:
 415 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vldr.gather.offset.v8f16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 0, i32 0)
 416 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
 417 //
 418 float16x8_t test_vldrhq_gather_offset_f16(const float16_t *base, uint16x8_t offset)
 419 {
 420 #ifdef POLYMORPHIC
 421     return vldrhq_gather_offset(base, offset);
 422 #else /* POLYMORPHIC */
 423     return vldrhq_gather_offset_f16(base, offset);
 424 #endif /* POLYMORPHIC */
 425 }
 426
 427 // CHECK-LABEL: @test_vldrhq_gather_offset_s16(
 428 // CHECK-NEXT:  entry:
 429 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 0, i32 0)
 430 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
 431 //
 432 int16x8_t test_vldrhq_gather_offset_s16(const int16_t *base, uint16x8_t offset)
 433 {
 434 #ifdef POLYMORPHIC
 435     return vldrhq_gather_offset(base, offset);
 436 #else /* POLYMORPHIC */
 437     return vldrhq_gather_offset_s16(base, offset);
 438 #endif /* POLYMORPHIC */
 439 }
 440
 441 // CHECK-LABEL: @test_vldrhq_gather_offset_s32(
 442 // CHECK-NEXT:  entry:
 443 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 0, i32 0)
 444 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 445 //
 446 int32x4_t test_vldrhq_gather_offset_s32(const int16_t *base, uint32x4_t offset)
 447 {
 448 #ifdef POLYMORPHIC
 449     return vldrhq_gather_offset(base, offset);
 450 #else /* POLYMORPHIC */
 451     return vldrhq_gather_offset_s32(base, offset);
 452 #endif /* POLYMORPHIC */
 453 }
 454
 455 // CHECK-LABEL: @test_vldrhq_gather_offset_u16(
 456 // CHECK-NEXT:  entry:
 457 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 0, i32 1)
 458 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
 459 //
 460 uint16x8_t test_vldrhq_gather_offset_u16(const uint16_t *base, uint16x8_t offset)
 461 {
 462 #ifdef POLYMORPHIC
 463     return vldrhq_gather_offset(base, offset);
 464 #else /* POLYMORPHIC */
 465     return vldrhq_gather_offset_u16(base, offset);
 466 #endif /* POLYMORPHIC */
 467 }
 468
 469 // CHECK-LABEL: @test_vldrhq_gather_offset_u32(
 470 // CHECK-NEXT:  entry:
 471 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 0, i32 1)
 472 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 473 //
 474 uint32x4_t test_vldrhq_gather_offset_u32(const uint16_t *base, uint32x4_t offset)
 475 {
 476 #ifdef POLYMORPHIC
 477     return vldrhq_gather_offset(base, offset);
 478 #else /* POLYMORPHIC */
 479     return vldrhq_gather_offset_u32(base, offset);
 480 #endif /* POLYMORPHIC */
 481 }
 482
 483 // CHECK-LABEL: @test_vldrhq_gather_offset_z_f16(
 484 // CHECK-NEXT:  entry:
 485 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 486 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 487 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vldr.gather.offset.predicated.v8f16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 0, i32 0, <8 x i1> [[TMP1]])
 488 // CHECK-NEXT:    ret <8 x half> [[TMP2]]
 489 //
 490 float16x8_t test_vldrhq_gather_offset_z_f16(const float16_t *base, uint16x8_t offset, mve_pred16_t p)
 491 {
 492 #ifdef POLYMORPHIC
 493     return vldrhq_gather_offset_z(base, offset, p);
 494 #else /* POLYMORPHIC */
 495     return vldrhq_gather_offset_z_f16(base, offset, p);
 496 #endif /* POLYMORPHIC */
 497 }
 498
 499 // CHECK-LABEL: @test_vldrhq_gather_offset_z_s16(
 500 // CHECK-NEXT:  entry:
 501 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 502 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 503 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 0, i32 0, <8 x i1> [[TMP1]])
 504 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 505 //
 506 int16x8_t test_vldrhq_gather_offset_z_s16(const int16_t *base, uint16x8_t offset, mve_pred16_t p)
 507 {
 508 #ifdef POLYMORPHIC
 509     return vldrhq_gather_offset_z(base, offset, p);
 510 #else /* POLYMORPHIC */
 511     return vldrhq_gather_offset_z_s16(base, offset, p);
 512 #endif /* POLYMORPHIC */
 513 }
 514
 515 // CHECK-LABEL: @test_vldrhq_gather_offset_z_s32(
 516 // CHECK-NEXT:  entry:
 517 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 518 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 519 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 0, i32 0, <4 x i1> [[TMP1]])
 520 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 521 //
 522 int32x4_t test_vldrhq_gather_offset_z_s32(const int16_t *base, uint32x4_t offset, mve_pred16_t p)
 523 {
 524 #ifdef POLYMORPHIC
 525     return vldrhq_gather_offset_z(base, offset, p);
 526 #else /* POLYMORPHIC */
 527     return vldrhq_gather_offset_z_s32(base, offset, p);
 528 #endif /* POLYMORPHIC */
 529 }
 530
 531 // CHECK-LABEL: @test_vldrhq_gather_offset_z_u16(
 532 // CHECK-NEXT:  entry:
 533 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 534 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 535 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 0, i32 1, <8 x i1> [[TMP1]])
 536 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 537 //
 538 uint16x8_t test_vldrhq_gather_offset_z_u16(const uint16_t *base, uint16x8_t offset, mve_pred16_t p)
 539 {
 540 #ifdef POLYMORPHIC
 541     return vldrhq_gather_offset_z(base, offset, p);
 542 #else /* POLYMORPHIC */
 543     return vldrhq_gather_offset_z_u16(base, offset, p);
 544 #endif /* POLYMORPHIC */
 545 }
 546
 547 // CHECK-LABEL: @test_vldrhq_gather_offset_z_u32(
 548 // CHECK-NEXT:  entry:
 549 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 550 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 551 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 0, i32 1, <4 x i1> [[TMP1]])
 552 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 553 //
 554 uint32x4_t test_vldrhq_gather_offset_z_u32(const uint16_t *base, uint32x4_t offset, mve_pred16_t p)
 555 {
 556 #ifdef POLYMORPHIC
 557     return vldrhq_gather_offset_z(base, offset, p);
 558 #else /* POLYMORPHIC */
 559     return vldrhq_gather_offset_z_u32(base, offset, p);
 560 #endif /* POLYMORPHIC */
 561 }
 562
 563 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_f16(
 564 // CHECK-NEXT:  entry:
 565 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.arm.mve.vldr.gather.offset.v8f16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 1, i32 0)
 566 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
 567 //
 568 float16x8_t test_vldrhq_gather_shifted_offset_f16(const float16_t *base, uint16x8_t offset)
 569 {
 570 #ifdef POLYMORPHIC
 571     return vldrhq_gather_shifted_offset(base, offset);
 572 #else /* POLYMORPHIC */
 573     return vldrhq_gather_shifted_offset_f16(base, offset);
 574 #endif /* POLYMORPHIC */
 575 }
 576
 577 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_s16(
 578 // CHECK-NEXT:  entry:
 579 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 1, i32 0)
 580 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
 581 //
 582 int16x8_t test_vldrhq_gather_shifted_offset_s16(const int16_t *base, uint16x8_t offset)
 583 {
 584 #ifdef POLYMORPHIC
 585     return vldrhq_gather_shifted_offset(base, offset);
 586 #else /* POLYMORPHIC */
 587     return vldrhq_gather_shifted_offset_s16(base, offset);
 588 #endif /* POLYMORPHIC */
 589 }
 590
 591 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_s32(
 592 // CHECK-NEXT:  entry:
 593 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 1, i32 0)
 594 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 595 //
 596 int32x4_t test_vldrhq_gather_shifted_offset_s32(const int16_t *base, uint32x4_t offset)
 597 {
 598 #ifdef POLYMORPHIC
 599     return vldrhq_gather_shifted_offset(base, offset);
 600 #else /* POLYMORPHIC */
 601     return vldrhq_gather_shifted_offset_s32(base, offset);
 602 #endif /* POLYMORPHIC */
 603 }
 604
 605 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_u16(
 606 // CHECK-NEXT:  entry:
 607 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.v8i16.p0.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 1, i32 1)
 608 // CHECK-NEXT:    ret <8 x i16> [[TMP0]]
 609 //
 610 uint16x8_t test_vldrhq_gather_shifted_offset_u16(const uint16_t *base, uint16x8_t offset)
 611 {
 612 #ifdef POLYMORPHIC
 613     return vldrhq_gather_shifted_offset(base, offset);
 614 #else /* POLYMORPHIC */
 615     return vldrhq_gather_shifted_offset_u16(base, offset);
 616 #endif /* POLYMORPHIC */
 617 }
 618
 619 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_u32(
 620 // CHECK-NEXT:  entry:
 621 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 1, i32 1)
 622 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 623 //
 624 uint32x4_t test_vldrhq_gather_shifted_offset_u32(const uint16_t *base, uint32x4_t offset)
 625 {
 626 #ifdef POLYMORPHIC
 627     return vldrhq_gather_shifted_offset(base, offset);
 628 #else /* POLYMORPHIC */
 629     return vldrhq_gather_shifted_offset_u32(base, offset);
 630 #endif /* POLYMORPHIC */
 631 }
 632
 633 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_z_f16(
 634 // CHECK-NEXT:  entry:
 635 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 636 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 637 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.vldr.gather.offset.predicated.v8f16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 1, i32 0, <8 x i1> [[TMP1]])
 638 // CHECK-NEXT:    ret <8 x half> [[TMP2]]
 639 //
 640 float16x8_t test_vldrhq_gather_shifted_offset_z_f16(const float16_t *base, uint16x8_t offset, mve_pred16_t p)
 641 {
 642 #ifdef POLYMORPHIC
 643     return vldrhq_gather_shifted_offset_z(base, offset, p);
 644 #else /* POLYMORPHIC */
 645     return vldrhq_gather_shifted_offset_z_f16(base, offset, p);
 646 #endif /* POLYMORPHIC */
 647 }
 648
 649 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_z_s16(
 650 // CHECK-NEXT:  entry:
 651 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 652 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 653 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 1, i32 0, <8 x i1> [[TMP1]])
 654 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 655 //
 656 int16x8_t test_vldrhq_gather_shifted_offset_z_s16(const int16_t *base, uint16x8_t offset, mve_pred16_t p)
 657 {
 658 #ifdef POLYMORPHIC
 659     return vldrhq_gather_shifted_offset_z(base, offset, p);
 660 #else /* POLYMORPHIC */
 661     return vldrhq_gather_shifted_offset_z_s16(base, offset, p);
 662 #endif /* POLYMORPHIC */
 663 }
 664
 665 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_z_s32(
 666 // CHECK-NEXT:  entry:
 667 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 668 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 669 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 1, i32 0, <4 x i1> [[TMP1]])
 670 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 671 //
 672 int32x4_t test_vldrhq_gather_shifted_offset_z_s32(const int16_t *base, uint32x4_t offset, mve_pred16_t p)
 673 {
 674 #ifdef POLYMORPHIC
 675     return vldrhq_gather_shifted_offset_z(base, offset, p);
 676 #else /* POLYMORPHIC */
 677     return vldrhq_gather_shifted_offset_z_s32(base, offset, p);
 678 #endif /* POLYMORPHIC */
 679 }
 680
 681 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_z_u16(
 682 // CHECK-NEXT:  entry:
 683 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 684 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 685 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i16> @llvm.arm.mve.vldr.gather.offset.predicated.v8i16.p0.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], i32 16, i32 1, i32 1, <8 x i1> [[TMP1]])
 686 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 687 //
 688 uint16x8_t test_vldrhq_gather_shifted_offset_z_u16(const uint16_t *base, uint16x8_t offset, mve_pred16_t p)
 689 {
 690 #ifdef POLYMORPHIC
 691     return vldrhq_gather_shifted_offset_z(base, offset, p);
 692 #else /* POLYMORPHIC */
 693     return vldrhq_gather_shifted_offset_z_u16(base, offset, p);
 694 #endif /* POLYMORPHIC */
 695 }
 696
 697 // CHECK-LABEL: @test_vldrhq_gather_shifted_offset_z_u32(
 698 // CHECK-NEXT:  entry:
 699 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 700 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 701 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 16, i32 1, i32 1, <4 x i1> [[TMP1]])
 702 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 703 //
 704 uint32x4_t test_vldrhq_gather_shifted_offset_z_u32(const uint16_t *base, uint32x4_t offset, mve_pred16_t p)
 705 {
 706 #ifdef POLYMORPHIC
 707     return vldrhq_gather_shifted_offset_z(base, offset, p);
 708 #else /* POLYMORPHIC */
 709     return vldrhq_gather_shifted_offset_z_u32(base, offset, p);
 710 #endif /* POLYMORPHIC */
 711 }
 712
 713 // CHECK-LABEL: @test_vldrwq_gather_base_f32(
 714 // CHECK-NEXT:  entry:
 715 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vldr.gather.base.v4f32.v4i32(<4 x i32> [[ADDR:%.*]], i32 12)
 716 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
 717 //
 718 float32x4_t test_vldrwq_gather_base_f32(uint32x4_t addr)
 719 {
 720     return vldrwq_gather_base_f32(addr, 0xc);
 721 }
 722
 723 // CHECK-LABEL: @test_vldrwq_gather_base_s32(
 724 // CHECK-NEXT:  entry:
 725 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32> [[ADDR:%.*]], i32 400)
 726 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 727 //
 728 int32x4_t test_vldrwq_gather_base_s32(uint32x4_t addr)
 729 {
 730     return vldrwq_gather_base_s32(addr, 0x190);
 731 }
 732
 733 // CHECK-LABEL: @test_vldrwq_gather_base_u32(
 734 // CHECK-NEXT:  entry:
 735 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.base.v4i32.v4i32(<4 x i32> [[ADDR:%.*]], i32 284)
 736 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 737 //
 738 uint32x4_t test_vldrwq_gather_base_u32(uint32x4_t addr)
 739 {
 740     return vldrwq_gather_base_u32(addr, 0x11c);
 741 }
 742
 743 // CHECK-LABEL: @test_vldrwq_gather_base_wb_f32(
 744 // CHECK-NEXT:  entry:
 745 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
 746 // CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4f32.v4i32(<4 x i32> [[TMP0]], i32 -64)
 747 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP1]], 1
 748 // CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[ADDR]], align 8
 749 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP1]], 0
 750 // CHECK-NEXT:    ret <4 x float> [[TMP3]]
 751 //
 752 float32x4_t test_vldrwq_gather_base_wb_f32(uint32x4_t *addr)
 753 {
 754     return vldrwq_gather_base_wb_f32(addr, -0x40);
 755 }
 756
 757 // CHECK-LABEL: @test_vldrwq_gather_base_wb_s32(
 758 // CHECK-NEXT:  entry:
 759 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
 760 // CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> [[TMP0]], i32 80)
 761 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP1]], 1
 762 // CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[ADDR]], align 8
 763 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP1]], 0
 764 // CHECK-NEXT:    ret <4 x i32> [[TMP3]]
 765 //
 766 int32x4_t test_vldrwq_gather_base_wb_s32(uint32x4_t *addr)
 767 {
 768     return vldrwq_gather_base_wb_s32(addr, 0x50);
 769 }
 770
 771 // CHECK-LABEL: @test_vldrwq_gather_base_wb_u32(
 772 // CHECK-NEXT:  entry:
 773 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
 774 // CHECK-NEXT:    [[TMP1:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.v4i32.v4i32(<4 x i32> [[TMP0]], i32 480)
 775 // CHECK-NEXT:    [[TMP2:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP1]], 1
 776 // CHECK-NEXT:    store <4 x i32> [[TMP2]], ptr [[ADDR]], align 8
 777 // CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP1]], 0
 778 // CHECK-NEXT:    ret <4 x i32> [[TMP3]]
 779 //
 780 uint32x4_t test_vldrwq_gather_base_wb_u32(uint32x4_t *addr)
 781 {
 782     return vldrwq_gather_base_wb_u32(addr, 0x1e0);
 783 }
 784
 785 // CHECK-LABEL: @test_vldrwq_gather_base_wb_z_f32(
 786 // CHECK-NEXT:  entry:
 787 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
 788 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 789 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 790 // CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP0]], i32 -352, <4 x i1> [[TMP2]])
 791 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP3]], 1
 792 // CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[ADDR]], align 8
 793 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP3]], 0
 794 // CHECK-NEXT:    ret <4 x float> [[TMP5]]
 795 //
 796 float32x4_t test_vldrwq_gather_base_wb_z_f32(uint32x4_t *addr, mve_pred16_t p)
 797 {
 798     return vldrwq_gather_base_wb_z_f32(addr, -0x160, p);
 799 }
 800
 801 // CHECK-LABEL: @test_vldrwq_gather_base_wb_z_s32(
 802 // CHECK-NEXT:  entry:
 803 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
 804 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 805 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 806 // CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> [[TMP0]], i32 276, <4 x i1> [[TMP2]])
 807 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP3]], 1
 808 // CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[ADDR]], align 8
 809 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP3]], 0
 810 // CHECK-NEXT:    ret <4 x i32> [[TMP5]]
 811 //
 812 int32x4_t test_vldrwq_gather_base_wb_z_s32(uint32x4_t *addr, mve_pred16_t p)
 813 {
 814     return vldrwq_gather_base_wb_z_s32(addr, 0x114, p);
 815 }
 816
 817 // CHECK-LABEL: @test_vldrwq_gather_base_wb_z_u32(
 818 // CHECK-NEXT:  entry:
 819 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
 820 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 821 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 822 // CHECK-NEXT:    [[TMP3:%.*]] = call { <4 x i32>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> [[TMP0]], i32 88, <4 x i1> [[TMP2]])
 823 // CHECK-NEXT:    [[TMP4:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP3]], 1
 824 // CHECK-NEXT:    store <4 x i32> [[TMP4]], ptr [[ADDR]], align 8
 825 // CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[TMP3]], 0
 826 // CHECK-NEXT:    ret <4 x i32> [[TMP5]]
 827 //
 828 uint32x4_t test_vldrwq_gather_base_wb_z_u32(uint32x4_t *addr, mve_pred16_t p)
 829 {
 830     return vldrwq_gather_base_wb_z_u32(addr, 0x58, p);
 831 }
 832
 833 // CHECK-LABEL: @test_vldrwq_gather_base_z_f32(
 834 // CHECK-NEXT:  entry:
 835 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 836 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 837 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vldr.gather.base.predicated.v4f32.v4i32.v4i1(<4 x i32> [[ADDR:%.*]], i32 300, <4 x i1> [[TMP1]])
 838 // CHECK-NEXT:    ret <4 x float> [[TMP2]]
 839 //
 840 float32x4_t test_vldrwq_gather_base_z_f32(uint32x4_t addr, mve_pred16_t p)
 841 {
 842     return vldrwq_gather_base_z_f32(addr, 0x12c, p);
 843 }
 844
 845 // CHECK-LABEL: @test_vldrwq_gather_base_z_s32(
 846 // CHECK-NEXT:  entry:
 847 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 848 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 849 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.base.predicated.v4i32.v4i32.v4i1(<4 x i32> [[ADDR:%.*]], i32 440, <4 x i1> [[TMP1]])
 850 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 851 //
 852 int32x4_t test_vldrwq_gather_base_z_s32(uint32x4_t addr, mve_pred16_t p)
 853 {
 854     return vldrwq_gather_base_z_s32(addr, 0x1b8, p);
 855 }
 856
 857 // CHECK-LABEL: @test_vldrwq_gather_base_z_u32(
 858 // CHECK-NEXT:  entry:
 859 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 860 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 861 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.base.predicated.v4i32.v4i32.v4i1(<4 x i32> [[ADDR:%.*]], i32 -300, <4 x i1> [[TMP1]])
 862 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 863 //
 864 uint32x4_t test_vldrwq_gather_base_z_u32(uint32x4_t addr, mve_pred16_t p)
 865 {
 866     return vldrwq_gather_base_z_u32(addr, -0x12c, p);
 867 }
 868
 869 // CHECK-LABEL: @test_vldrwq_gather_offset_f32(
 870 // CHECK-NEXT:  entry:
 871 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vldr.gather.offset.v4f32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 0, i32 0)
 872 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
 873 //
 874 float32x4_t test_vldrwq_gather_offset_f32(const float32_t *base, uint32x4_t offset)
 875 {
 876 #ifdef POLYMORPHIC
 877     return vldrwq_gather_offset(base, offset);
 878 #else /* POLYMORPHIC */
 879     return vldrwq_gather_offset_f32(base, offset);
 880 #endif /* POLYMORPHIC */
 881 }
 882
 883 // CHECK-LABEL: @test_vldrwq_gather_offset_s32(
 884 // CHECK-NEXT:  entry:
 885 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 0, i32 0)
 886 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 887 //
 888 int32x4_t test_vldrwq_gather_offset_s32(const int32_t *base, uint32x4_t offset)
 889 {
 890 #ifdef POLYMORPHIC
 891     return vldrwq_gather_offset(base, offset);
 892 #else /* POLYMORPHIC */
 893     return vldrwq_gather_offset_s32(base, offset);
 894 #endif /* POLYMORPHIC */
 895 }
 896
 897 // CHECK-LABEL: @test_vldrwq_gather_offset_u32(
 898 // CHECK-NEXT:  entry:
 899 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 0, i32 1)
 900 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 901 //
 902 uint32x4_t test_vldrwq_gather_offset_u32(const uint32_t *base, uint32x4_t offset)
 903 {
 904 #ifdef POLYMORPHIC
 905     return vldrwq_gather_offset(base, offset);
 906 #else /* POLYMORPHIC */
 907     return vldrwq_gather_offset_u32(base, offset);
 908 #endif /* POLYMORPHIC */
 909 }
 910
 911 // CHECK-LABEL: @test_vldrwq_gather_offset_z_f32(
 912 // CHECK-NEXT:  entry:
 913 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 914 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 915 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vldr.gather.offset.predicated.v4f32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 0, i32 0, <4 x i1> [[TMP1]])
 916 // CHECK-NEXT:    ret <4 x float> [[TMP2]]
 917 //
 918 float32x4_t test_vldrwq_gather_offset_z_f32(const float32_t *base, uint32x4_t offset, mve_pred16_t p)
 919 {
 920 #ifdef POLYMORPHIC
 921     return vldrwq_gather_offset_z(base, offset, p);
 922 #else /* POLYMORPHIC */
 923     return vldrwq_gather_offset_z_f32(base, offset, p);
 924 #endif /* POLYMORPHIC */
 925 }
 926
 927 // CHECK-LABEL: @test_vldrwq_gather_offset_z_s32(
 928 // CHECK-NEXT:  entry:
 929 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 930 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 931 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 0, i32 0, <4 x i1> [[TMP1]])
 932 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 933 //
 934 int32x4_t test_vldrwq_gather_offset_z_s32(const int32_t *base, uint32x4_t offset, mve_pred16_t p)
 935 {
 936 #ifdef POLYMORPHIC
 937     return vldrwq_gather_offset_z(base, offset, p);
 938 #else /* POLYMORPHIC */
 939     return vldrwq_gather_offset_z_s32(base, offset, p);
 940 #endif /* POLYMORPHIC */
 941 }
 942
 943 // CHECK-LABEL: @test_vldrwq_gather_offset_z_u32(
 944 // CHECK-NEXT:  entry:
 945 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 946 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 947 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 0, i32 1, <4 x i1> [[TMP1]])
 948 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 949 //
 950 uint32x4_t test_vldrwq_gather_offset_z_u32(const uint32_t *base, uint32x4_t offset, mve_pred16_t p)
 951 {
 952 #ifdef POLYMORPHIC
 953     return vldrwq_gather_offset_z(base, offset, p);
 954 #else /* POLYMORPHIC */
 955     return vldrwq_gather_offset_z_u32(base, offset, p);
 956 #endif /* POLYMORPHIC */
 957 }
 958
 959 // CHECK-LABEL: @test_vldrwq_gather_shifted_offset_f32(
 960 // CHECK-NEXT:  entry:
 961 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.arm.mve.vldr.gather.offset.v4f32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 2, i32 0)
 962 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
 963 //
 964 float32x4_t test_vldrwq_gather_shifted_offset_f32(const float32_t *base, uint32x4_t offset)
 965 {
 966 #ifdef POLYMORPHIC
 967     return vldrwq_gather_shifted_offset(base, offset);
 968 #else /* POLYMORPHIC */
 969     return vldrwq_gather_shifted_offset_f32(base, offset);
 970 #endif /* POLYMORPHIC */
 971 }
 972
 973 // CHECK-LABEL: @test_vldrwq_gather_shifted_offset_s32(
 974 // CHECK-NEXT:  entry:
 975 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 2, i32 0)
 976 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 977 //
 978 int32x4_t test_vldrwq_gather_shifted_offset_s32(const int32_t *base, uint32x4_t offset)
 979 {
 980 #ifdef POLYMORPHIC
 981     return vldrwq_gather_shifted_offset(base, offset);
 982 #else /* POLYMORPHIC */
 983     return vldrwq_gather_shifted_offset_s32(base, offset);
 984 #endif /* POLYMORPHIC */
 985 }
 986
 987 // CHECK-LABEL: @test_vldrwq_gather_shifted_offset_u32(
 988 // CHECK-NEXT:  entry:
 989 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.v4i32.p0.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 2, i32 1)
 990 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 991 //
 992 uint32x4_t test_vldrwq_gather_shifted_offset_u32(const uint32_t *base, uint32x4_t offset)
 993 {
 994 #ifdef POLYMORPHIC
 995     return vldrwq_gather_shifted_offset(base, offset);
 996 #else /* POLYMORPHIC */
 997     return vldrwq_gather_shifted_offset_u32(base, offset);
 998 #endif /* POLYMORPHIC */
 999 }
1000
1001 // CHECK-LABEL: @test_vldrwq_gather_shifted_offset_z_f32(
1002 // CHECK-NEXT:  entry:
1003 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1004 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1005 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.vldr.gather.offset.predicated.v4f32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 2, i32 0, <4 x i1> [[TMP1]])
1006 // CHECK-NEXT:    ret <4 x float> [[TMP2]]
1007 //
1008 float32x4_t test_vldrwq_gather_shifted_offset_z_f32(const float32_t *base, uint32x4_t offset, mve_pred16_t p)
1009 {
1010 #ifdef POLYMORPHIC
1011     return vldrwq_gather_shifted_offset_z(base, offset, p);
1012 #else /* POLYMORPHIC */
1013     return vldrwq_gather_shifted_offset_z_f32(base, offset, p);
1014 #endif /* POLYMORPHIC */
1015 }
1016
1017 // CHECK-LABEL: @test_vldrwq_gather_shifted_offset_z_s32(
1018 // CHECK-NEXT:  entry:
1019 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1020 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1021 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 2, i32 0, <4 x i1> [[TMP1]])
1022 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
1023 //
1024 int32x4_t test_vldrwq_gather_shifted_offset_z_s32(const int32_t *base, uint32x4_t offset, mve_pred16_t p)
1025 {
1026 #ifdef POLYMORPHIC
1027     return vldrwq_gather_shifted_offset_z(base, offset, p);
1028 #else /* POLYMORPHIC */
1029     return vldrwq_gather_shifted_offset_z_s32(base, offset, p);
1030 #endif /* POLYMORPHIC */
1031 }
1032
1033 // CHECK-LABEL: @test_vldrwq_gather_shifted_offset_z_u32(
1034 // CHECK-NEXT:  entry:
1035 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1036 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1037 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vldr.gather.offset.predicated.v4i32.p0.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], i32 32, i32 2, i32 1, <4 x i1> [[TMP1]])
1038 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
1039 //
1040 uint32x4_t test_vldrwq_gather_shifted_offset_z_u32(const uint32_t *base, uint32x4_t offset, mve_pred16_t p)
1041 {
1042 #ifdef POLYMORPHIC
1043     return vldrwq_gather_shifted_offset_z(base, offset, p);
1044 #else /* POLYMORPHIC */
1045     return vldrwq_gather_shifted_offset_z_u32(base, offset, p);
1046 #endif /* POLYMORPHIC */
1047 }
1048
1049 // CHECK-LABEL: @test_vstrbq_scatter_offset_p_s16(
1050 // CHECK-NEXT:  entry:
1051 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1052 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1053 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 8, i32 0, <8 x i1> [[TMP1]])
1054 // CHECK-NEXT:    ret void
1055 //
1056 void test_vstrbq_scatter_offset_p_s16(int8_t *base, uint16x8_t offset, int16x8_t value, mve_pred16_t p)
1057 {
1058 #ifdef POLYMORPHIC
1059     vstrbq_scatter_offset_p(base, offset, value, p);
1060 #else /* POLYMORPHIC */
1061     vstrbq_scatter_offset_p_s16(base, offset, value, p);
1062 #endif /* POLYMORPHIC */
1063 }
1064
1065 // CHECK-LABEL: @test_vstrbq_scatter_offset_p_s32(
1066 // CHECK-NEXT:  entry:
1067 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1068 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1069 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 8, i32 0, <4 x i1> [[TMP1]])
1070 // CHECK-NEXT:    ret void
1071 //
1072 void test_vstrbq_scatter_offset_p_s32(int8_t *base, uint32x4_t offset, int32x4_t value, mve_pred16_t p)
1073 {
1074 #ifdef POLYMORPHIC
1075     vstrbq_scatter_offset_p(base, offset, value, p);
1076 #else /* POLYMORPHIC */
1077     vstrbq_scatter_offset_p_s32(base, offset, value, p);
1078 #endif /* POLYMORPHIC */
1079 }
1080
1081 // CHECK-LABEL: @test_vstrbq_scatter_offset_p_s8(
1082 // CHECK-NEXT:  entry:
1083 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1084 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
1085 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v16i8.v16i8.v16i1(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], <16 x i8> [[VALUE:%.*]], i32 8, i32 0, <16 x i1> [[TMP1]])
1086 // CHECK-NEXT:    ret void
1087 //
1088 void test_vstrbq_scatter_offset_p_s8(int8_t *base, uint8x16_t offset, int8x16_t value, mve_pred16_t p)
1089 {
1090 #ifdef POLYMORPHIC
1091     vstrbq_scatter_offset_p(base, offset, value, p);
1092 #else /* POLYMORPHIC */
1093     vstrbq_scatter_offset_p_s8(base, offset, value, p);
1094 #endif /* POLYMORPHIC */
1095 }
1096
1097 // CHECK-LABEL: @test_vstrbq_scatter_offset_p_u16(
1098 // CHECK-NEXT:  entry:
1099 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1100 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1101 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 8, i32 0, <8 x i1> [[TMP1]])
1102 // CHECK-NEXT:    ret void
1103 //
1104 void test_vstrbq_scatter_offset_p_u16(uint8_t *base, uint16x8_t offset, uint16x8_t value, mve_pred16_t p)
1105 {
1106 #ifdef POLYMORPHIC
1107     vstrbq_scatter_offset_p(base, offset, value, p);
1108 #else /* POLYMORPHIC */
1109     vstrbq_scatter_offset_p_u16(base, offset, value, p);
1110 #endif /* POLYMORPHIC */
1111 }
1112
1113 // CHECK-LABEL: @test_vstrbq_scatter_offset_p_u32(
1114 // CHECK-NEXT:  entry:
1115 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1116 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1117 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 8, i32 0, <4 x i1> [[TMP1]])
1118 // CHECK-NEXT:    ret void
1119 //
1120 void test_vstrbq_scatter_offset_p_u32(uint8_t *base, uint32x4_t offset, uint32x4_t value, mve_pred16_t p)
1121 {
1122 #ifdef POLYMORPHIC
1123     vstrbq_scatter_offset_p(base, offset, value, p);
1124 #else /* POLYMORPHIC */
1125     vstrbq_scatter_offset_p_u32(base, offset, value, p);
1126 #endif /* POLYMORPHIC */
1127 }
1128
1129 // CHECK-LABEL: @test_vstrbq_scatter_offset_p_u8(
1130 // CHECK-NEXT:  entry:
1131 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1132 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP0]])
1133 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v16i8.v16i8.v16i1(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], <16 x i8> [[VALUE:%.*]], i32 8, i32 0, <16 x i1> [[TMP1]])
1134 // CHECK-NEXT:    ret void
1135 //
1136 void test_vstrbq_scatter_offset_p_u8(uint8_t *base, uint8x16_t offset, uint8x16_t value, mve_pred16_t p)
1137 {
1138 #ifdef POLYMORPHIC
1139     vstrbq_scatter_offset_p(base, offset, value, p);
1140 #else /* POLYMORPHIC */
1141     vstrbq_scatter_offset_p_u8(base, offset, value, p);
1142 #endif /* POLYMORPHIC */
1143 }
1144
1145 // CHECK-LABEL: @test_vstrbq_scatter_offset_s16(
1146 // CHECK-NEXT:  entry:
1147 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 8, i32 0)
1148 // CHECK-NEXT:    ret void
1149 //
1150 void test_vstrbq_scatter_offset_s16(int8_t *base, uint16x8_t offset, int16x8_t value)
1151 {
1152 #ifdef POLYMORPHIC
1153     vstrbq_scatter_offset(base, offset, value);
1154 #else /* POLYMORPHIC */
1155     vstrbq_scatter_offset_s16(base, offset, value);
1156 #endif /* POLYMORPHIC */
1157 }
1158
1159 // CHECK-LABEL: @test_vstrbq_scatter_offset_s32(
1160 // CHECK-NEXT:  entry:
1161 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 8, i32 0)
1162 // CHECK-NEXT:    ret void
1163 //
1164 void test_vstrbq_scatter_offset_s32(int8_t *base, uint32x4_t offset, int32x4_t value)
1165 {
1166 #ifdef POLYMORPHIC
1167     vstrbq_scatter_offset(base, offset, value);
1168 #else /* POLYMORPHIC */
1169     vstrbq_scatter_offset_s32(base, offset, value);
1170 #endif /* POLYMORPHIC */
1171 }
1172
1173 // CHECK-LABEL: @test_vstrbq_scatter_offset_s8(
1174 // CHECK-NEXT:  entry:
1175 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v16i8.v16i8(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], <16 x i8> [[VALUE:%.*]], i32 8, i32 0)
1176 // CHECK-NEXT:    ret void
1177 //
1178 void test_vstrbq_scatter_offset_s8(int8_t *base, uint8x16_t offset, int8x16_t value)
1179 {
1180 #ifdef POLYMORPHIC
1181     vstrbq_scatter_offset(base, offset, value);
1182 #else /* POLYMORPHIC */
1183     vstrbq_scatter_offset_s8(base, offset, value);
1184 #endif /* POLYMORPHIC */
1185 }
1186
1187 // CHECK-LABEL: @test_vstrbq_scatter_offset_u16(
1188 // CHECK-NEXT:  entry:
1189 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 8, i32 0)
1190 // CHECK-NEXT:    ret void
1191 //
1192 void test_vstrbq_scatter_offset_u16(uint8_t *base, uint16x8_t offset, uint16x8_t value)
1193 {
1194 #ifdef POLYMORPHIC
1195     vstrbq_scatter_offset(base, offset, value);
1196 #else /* POLYMORPHIC */
1197     vstrbq_scatter_offset_u16(base, offset, value);
1198 #endif /* POLYMORPHIC */
1199 }
1200
1201 // CHECK-LABEL: @test_vstrbq_scatter_offset_u32(
1202 // CHECK-NEXT:  entry:
1203 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 8, i32 0)
1204 // CHECK-NEXT:    ret void
1205 //
1206 void test_vstrbq_scatter_offset_u32(uint8_t *base, uint32x4_t offset, uint32x4_t value)
1207 {
1208 #ifdef POLYMORPHIC
1209     vstrbq_scatter_offset(base, offset, value);
1210 #else /* POLYMORPHIC */
1211     vstrbq_scatter_offset_u32(base, offset, value);
1212 #endif /* POLYMORPHIC */
1213 }
1214
1215 // CHECK-LABEL: @test_vstrbq_scatter_offset_u8(
1216 // CHECK-NEXT:  entry:
1217 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v16i8.v16i8(ptr [[BASE:%.*]], <16 x i8> [[OFFSET:%.*]], <16 x i8> [[VALUE:%.*]], i32 8, i32 0)
1218 // CHECK-NEXT:    ret void
1219 //
1220 void test_vstrbq_scatter_offset_u8(uint8_t *base, uint8x16_t offset, uint8x16_t value)
1221 {
1222 #ifdef POLYMORPHIC
1223     vstrbq_scatter_offset(base, offset, value);
1224 #else /* POLYMORPHIC */
1225     vstrbq_scatter_offset_u8(base, offset, value);
1226 #endif /* POLYMORPHIC */
1227 }
1228
1229 // CHECK-LABEL: @test_vstrdq_scatter_base_p_s64(
1230 // CHECK-NEXT:  entry:
1231 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1232 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
1233 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v2i1(<2 x i64> [[ADDR:%.*]], i32 888, <2 x i64> [[VALUE:%.*]], <2 x i1> [[TMP1]])
1234 // CHECK-NEXT:    ret void
1235 //
1236 void test_vstrdq_scatter_base_p_s64(uint64x2_t addr, int64x2_t value, mve_pred16_t p)
1237 {
1238 #ifdef POLYMORPHIC
1239     vstrdq_scatter_base_p(addr, 0x378, value, p);
1240 #else /* POLYMORPHIC */
1241     vstrdq_scatter_base_p_s64(addr, 0x378, value, p);
1242 #endif /* POLYMORPHIC */
1243 }
1244
1245 // CHECK-LABEL: @test_vstrdq_scatter_base_p_u64(
1246 // CHECK-NEXT:  entry:
1247 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1248 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
1249 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.predicated.v2i64.v2i64.v2i1(<2 x i64> [[ADDR:%.*]], i32 264, <2 x i64> [[VALUE:%.*]], <2 x i1> [[TMP1]])
1250 // CHECK-NEXT:    ret void
1251 //
1252 void test_vstrdq_scatter_base_p_u64(uint64x2_t addr, uint64x2_t value, mve_pred16_t p)
1253 {
1254 #ifdef POLYMORPHIC
1255     vstrdq_scatter_base_p(addr, 0x108, value, p);
1256 #else /* POLYMORPHIC */
1257     vstrdq_scatter_base_p_u64(addr, 0x108, value, p);
1258 #endif /* POLYMORPHIC */
1259 }
1260
1261 // CHECK-LABEL: @test_vstrdq_scatter_base_s64(
1262 // CHECK-NEXT:  entry:
1263 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.v2i64.v2i64(<2 x i64> [[ADDR:%.*]], i32 408, <2 x i64> [[VALUE:%.*]])
1264 // CHECK-NEXT:    ret void
1265 //
1266 void test_vstrdq_scatter_base_s64(uint64x2_t addr, int64x2_t value)
1267 {
1268 #ifdef POLYMORPHIC
1269     vstrdq_scatter_base(addr, 0x198, value);
1270 #else /* POLYMORPHIC */
1271     vstrdq_scatter_base_s64(addr, 0x198, value);
1272 #endif /* POLYMORPHIC */
1273 }
1274
1275 // CHECK-LABEL: @test_vstrdq_scatter_base_u64(
1276 // CHECK-NEXT:  entry:
1277 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.v2i64.v2i64(<2 x i64> [[ADDR:%.*]], i32 -472, <2 x i64> [[VALUE:%.*]])
1278 // CHECK-NEXT:    ret void
1279 //
1280 void test_vstrdq_scatter_base_u64(uint64x2_t addr, uint64x2_t value)
1281 {
1282 #ifdef POLYMORPHIC
1283     vstrdq_scatter_base(addr, -0x1d8, value);
1284 #else /* POLYMORPHIC */
1285     vstrdq_scatter_base_u64(addr, -0x1d8, value);
1286 #endif /* POLYMORPHIC */
1287 }
1288
1289 // CHECK-LABEL: @test_vstrdq_scatter_base_wb_p_s64(
1290 // CHECK-NEXT:  entry:
1291 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
1292 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1293 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP1]])
1294 // CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v2i1(<2 x i64> [[TMP0]], i32 248, <2 x i64> [[VALUE:%.*]], <2 x i1> [[TMP2]])
1295 // CHECK-NEXT:    store <2 x i64> [[TMP3]], ptr [[ADDR]], align 8
1296 // CHECK-NEXT:    ret void
1297 //
1298 void test_vstrdq_scatter_base_wb_p_s64(uint64x2_t *addr, int64x2_t value, mve_pred16_t p)
1299 {
1300 #ifdef POLYMORPHIC
1301     vstrdq_scatter_base_wb_p(addr, 0xf8, value, p);
1302 #else /* POLYMORPHIC */
1303     vstrdq_scatter_base_wb_p_s64(addr, 0xf8, value, p);
1304 #endif /* POLYMORPHIC */
1305 }
1306
1307 // CHECK-LABEL: @test_vstrdq_scatter_base_wb_p_u64(
1308 // CHECK-NEXT:  entry:
1309 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
1310 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1311 // CHECK-NEXT:    [[TMP2:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP1]])
1312 // CHECK-NEXT:    [[TMP3:%.*]] = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v2i64.v2i64.v2i1(<2 x i64> [[TMP0]], i32 136, <2 x i64> [[VALUE:%.*]], <2 x i1> [[TMP2]])
1313 // CHECK-NEXT:    store <2 x i64> [[TMP3]], ptr [[ADDR]], align 8
1314 // CHECK-NEXT:    ret void
1315 //
1316 void test_vstrdq_scatter_base_wb_p_u64(uint64x2_t *addr, uint64x2_t value, mve_pred16_t p)
1317 {
1318 #ifdef POLYMORPHIC
1319     vstrdq_scatter_base_wb_p(addr, 0x88, value, p);
1320 #else /* POLYMORPHIC */
1321     vstrdq_scatter_base_wb_p_u64(addr, 0x88, value, p);
1322 #endif /* POLYMORPHIC */
1323 }
1324
1325 // CHECK-LABEL: @test_vstrdq_scatter_base_wb_s64(
1326 // CHECK-NEXT:  entry:
1327 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
1328 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.v2i64.v2i64(<2 x i64> [[TMP0]], i32 208, <2 x i64> [[VALUE:%.*]])
1329 // CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr [[ADDR]], align 8
1330 // CHECK-NEXT:    ret void
1331 //
1332 void test_vstrdq_scatter_base_wb_s64(uint64x2_t *addr, int64x2_t value)
1333 {
1334 #ifdef POLYMORPHIC
1335     vstrdq_scatter_base_wb(addr, 0xd0, value);
1336 #else /* POLYMORPHIC */
1337     vstrdq_scatter_base_wb_s64(addr, 0xd0, value);
1338 #endif /* POLYMORPHIC */
1339 }
1340
1341 // CHECK-LABEL: @test_vstrdq_scatter_base_wb_u64(
1342 // CHECK-NEXT:  entry:
1343 // CHECK-NEXT:    [[TMP0:%.*]] = load <2 x i64>, ptr [[ADDR:%.*]], align 8
1344 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i64> @llvm.arm.mve.vstr.scatter.base.wb.v2i64.v2i64(<2 x i64> [[TMP0]], i32 -168, <2 x i64> [[VALUE:%.*]])
1345 // CHECK-NEXT:    store <2 x i64> [[TMP1]], ptr [[ADDR]], align 8
1346 // CHECK-NEXT:    ret void
1347 //
1348 void test_vstrdq_scatter_base_wb_u64(uint64x2_t *addr, uint64x2_t value)
1349 {
1350 #ifdef POLYMORPHIC
1351     vstrdq_scatter_base_wb(addr, -0xa8, value);
1352 #else /* POLYMORPHIC */
1353     vstrdq_scatter_base_wb_u64(addr, -0xa8, value);
1354 #endif /* POLYMORPHIC */
1355 }
1356
1357 // CHECK-LABEL: @test_vstrdq_scatter_offset_p_s64(
1358 // CHECK-NEXT:  entry:
1359 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1360 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
1361 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 0, <2 x i1> [[TMP1]])
1362 // CHECK-NEXT:    ret void
1363 //
1364 void test_vstrdq_scatter_offset_p_s64(int64_t *base, uint64x2_t offset, int64x2_t value, mve_pred16_t p)
1365 {
1366 #ifdef POLYMORPHIC
1367     vstrdq_scatter_offset_p(base, offset, value, p);
1368 #else /* POLYMORPHIC */
1369     vstrdq_scatter_offset_p_s64(base, offset, value, p);
1370 #endif /* POLYMORPHIC */
1371 }
1372
1373 // CHECK-LABEL: @test_vstrdq_scatter_offset_p_u64(
1374 // CHECK-NEXT:  entry:
1375 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1376 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
1377 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 0, <2 x i1> [[TMP1]])
1378 // CHECK-NEXT:    ret void
1379 //
1380 void test_vstrdq_scatter_offset_p_u64(uint64_t *base, uint64x2_t offset, uint64x2_t value, mve_pred16_t p)
1381 {
1382 #ifdef POLYMORPHIC
1383     vstrdq_scatter_offset_p(base, offset, value, p);
1384 #else /* POLYMORPHIC */
1385     vstrdq_scatter_offset_p_u64(base, offset, value, p);
1386 #endif /* POLYMORPHIC */
1387 }
1388
1389 // CHECK-LABEL: @test_vstrdq_scatter_offset_s64(
1390 // CHECK-NEXT:  entry:
1391 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v2i64.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 0)
1392 // CHECK-NEXT:    ret void
1393 //
1394 void test_vstrdq_scatter_offset_s64(int64_t *base, uint64x2_t offset, int64x2_t value)
1395 {
1396 #ifdef POLYMORPHIC
1397     vstrdq_scatter_offset(base, offset, value);
1398 #else /* POLYMORPHIC */
1399     vstrdq_scatter_offset_s64(base, offset, value);
1400 #endif /* POLYMORPHIC */
1401 }
1402
1403 // CHECK-LABEL: @test_vstrdq_scatter_offset_u64(
1404 // CHECK-NEXT:  entry:
1405 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v2i64.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 0)
1406 // CHECK-NEXT:    ret void
1407 //
1408 void test_vstrdq_scatter_offset_u64(uint64_t *base, uint64x2_t offset, uint64x2_t value)
1409 {
1410 #ifdef POLYMORPHIC
1411     vstrdq_scatter_offset(base, offset, value);
1412 #else /* POLYMORPHIC */
1413     vstrdq_scatter_offset_u64(base, offset, value);
1414 #endif /* POLYMORPHIC */
1415 }
1416
1417 // CHECK-LABEL: @test_vstrdq_scatter_shifted_offset_p_s64(
1418 // CHECK-NEXT:  entry:
1419 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1420 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
1421 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 3, <2 x i1> [[TMP1]])
1422 // CHECK-NEXT:    ret void
1423 //
1424 void test_vstrdq_scatter_shifted_offset_p_s64(int64_t *base, uint64x2_t offset, int64x2_t value, mve_pred16_t p)
1425 {
1426 #ifdef POLYMORPHIC
1427     vstrdq_scatter_shifted_offset_p(base, offset, value, p);
1428 #else /* POLYMORPHIC */
1429     vstrdq_scatter_shifted_offset_p_s64(base, offset, value, p);
1430 #endif /* POLYMORPHIC */
1431 }
1432
1433 // CHECK-LABEL: @test_vstrdq_scatter_shifted_offset_p_u64(
1434 // CHECK-NEXT:  entry:
1435 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1436 // CHECK-NEXT:    [[TMP1:%.*]] = call <2 x i1> @llvm.arm.mve.pred.i2v.v2i1(i32 [[TMP0]])
1437 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v2i64.v2i64.v2i1(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 3, <2 x i1> [[TMP1]])
1438 // CHECK-NEXT:    ret void
1439 //
1440 void test_vstrdq_scatter_shifted_offset_p_u64(uint64_t *base, uint64x2_t offset, uint64x2_t value, mve_pred16_t p)
1441 {
1442 #ifdef POLYMORPHIC
1443     vstrdq_scatter_shifted_offset_p(base, offset, value, p);
1444 #else /* POLYMORPHIC */
1445     vstrdq_scatter_shifted_offset_p_u64(base, offset, value, p);
1446 #endif /* POLYMORPHIC */
1447 }
1448
1449 // CHECK-LABEL: @test_vstrdq_scatter_shifted_offset_s64(
1450 // CHECK-NEXT:  entry:
1451 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v2i64.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 3)
1452 // CHECK-NEXT:    ret void
1453 //
1454 void test_vstrdq_scatter_shifted_offset_s64(int64_t *base, uint64x2_t offset, int64x2_t value)
1455 {
1456 #ifdef POLYMORPHIC
1457     vstrdq_scatter_shifted_offset(base, offset, value);
1458 #else /* POLYMORPHIC */
1459     vstrdq_scatter_shifted_offset_s64(base, offset, value);
1460 #endif /* POLYMORPHIC */
1461 }
1462
1463 // CHECK-LABEL: @test_vstrdq_scatter_shifted_offset_u64(
1464 // CHECK-NEXT:  entry:
1465 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v2i64.v2i64(ptr [[BASE:%.*]], <2 x i64> [[OFFSET:%.*]], <2 x i64> [[VALUE:%.*]], i32 64, i32 3)
1466 // CHECK-NEXT:    ret void
1467 //
1468 void test_vstrdq_scatter_shifted_offset_u64(uint64_t *base, uint64x2_t offset, uint64x2_t value)
1469 {
1470 #ifdef POLYMORPHIC
1471     vstrdq_scatter_shifted_offset(base, offset, value);
1472 #else /* POLYMORPHIC */
1473     vstrdq_scatter_shifted_offset_u64(base, offset, value);
1474 #endif /* POLYMORPHIC */
1475 }
1476
1477 // CHECK-LABEL: @test_vstrhq_scatter_offset_f16(
1478 // CHECK-NEXT:  entry:
1479 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8f16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x half> [[VALUE:%.*]], i32 16, i32 0)
1480 // CHECK-NEXT:    ret void
1481 //
1482 void test_vstrhq_scatter_offset_f16(float16_t *base, uint16x8_t offset, float16x8_t value)
1483 {
1484 #ifdef POLYMORPHIC
1485     vstrhq_scatter_offset(base, offset, value);
1486 #else /* POLYMORPHIC */
1487     vstrhq_scatter_offset_f16(base, offset, value);
1488 #endif /* POLYMORPHIC */
1489 }
1490
1491 // CHECK-LABEL: @test_vstrhq_scatter_offset_p_f16(
1492 // CHECK-NEXT:  entry:
1493 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1494 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1495 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8f16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x half> [[VALUE:%.*]], i32 16, i32 0, <8 x i1> [[TMP1]])
1496 // CHECK-NEXT:    ret void
1497 //
1498 void test_vstrhq_scatter_offset_p_f16(float16_t *base, uint16x8_t offset, float16x8_t value, mve_pred16_t p)
1499 {
1500 #ifdef POLYMORPHIC
1501     vstrhq_scatter_offset_p(base, offset, value, p);
1502 #else /* POLYMORPHIC */
1503     vstrhq_scatter_offset_p_f16(base, offset, value, p);
1504 #endif /* POLYMORPHIC */
1505 }
1506
1507 // CHECK-LABEL: @test_vstrhq_scatter_offset_p_s16(
1508 // CHECK-NEXT:  entry:
1509 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1510 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1511 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 0, <8 x i1> [[TMP1]])
1512 // CHECK-NEXT:    ret void
1513 //
1514 void test_vstrhq_scatter_offset_p_s16(int16_t *base, uint16x8_t offset, int16x8_t value, mve_pred16_t p)
1515 {
1516 #ifdef POLYMORPHIC
1517     vstrhq_scatter_offset_p(base, offset, value, p);
1518 #else /* POLYMORPHIC */
1519     vstrhq_scatter_offset_p_s16(base, offset, value, p);
1520 #endif /* POLYMORPHIC */
1521 }
1522
1523 // CHECK-LABEL: @test_vstrhq_scatter_offset_p_s32(
1524 // CHECK-NEXT:  entry:
1525 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1526 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1527 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 0, <4 x i1> [[TMP1]])
1528 // CHECK-NEXT:    ret void
1529 //
1530 void test_vstrhq_scatter_offset_p_s32(int16_t *base, uint32x4_t offset, int32x4_t value, mve_pred16_t p)
1531 {
1532 #ifdef POLYMORPHIC
1533     vstrhq_scatter_offset_p(base, offset, value, p);
1534 #else /* POLYMORPHIC */
1535     vstrhq_scatter_offset_p_s32(base, offset, value, p);
1536 #endif /* POLYMORPHIC */
1537 }
1538
1539 // CHECK-LABEL: @test_vstrhq_scatter_offset_p_u16(
1540 // CHECK-NEXT:  entry:
1541 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1542 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1543 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 0, <8 x i1> [[TMP1]])
1544 // CHECK-NEXT:    ret void
1545 //
1546 void test_vstrhq_scatter_offset_p_u16(uint16_t *base, uint16x8_t offset, uint16x8_t value, mve_pred16_t p)
1547 {
1548 #ifdef POLYMORPHIC
1549     vstrhq_scatter_offset_p(base, offset, value, p);
1550 #else /* POLYMORPHIC */
1551     vstrhq_scatter_offset_p_u16(base, offset, value, p);
1552 #endif /* POLYMORPHIC */
1553 }
1554
1555 // CHECK-LABEL: @test_vstrhq_scatter_offset_p_u32(
1556 // CHECK-NEXT:  entry:
1557 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1558 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1559 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 0, <4 x i1> [[TMP1]])
1560 // CHECK-NEXT:    ret void
1561 //
1562 void test_vstrhq_scatter_offset_p_u32(uint16_t *base, uint32x4_t offset, uint32x4_t value, mve_pred16_t p)
1563 {
1564 #ifdef POLYMORPHIC
1565     vstrhq_scatter_offset_p(base, offset, value, p);
1566 #else /* POLYMORPHIC */
1567     vstrhq_scatter_offset_p_u32(base, offset, value, p);
1568 #endif /* POLYMORPHIC */
1569 }
1570
1571 // CHECK-LABEL: @test_vstrhq_scatter_offset_s16(
1572 // CHECK-NEXT:  entry:
1573 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 0)
1574 // CHECK-NEXT:    ret void
1575 //
1576 void test_vstrhq_scatter_offset_s16(int16_t *base, uint16x8_t offset, int16x8_t value)
1577 {
1578 #ifdef POLYMORPHIC
1579     vstrhq_scatter_offset(base, offset, value);
1580 #else /* POLYMORPHIC */
1581     vstrhq_scatter_offset_s16(base, offset, value);
1582 #endif /* POLYMORPHIC */
1583 }
1584
1585 // CHECK-LABEL: @test_vstrhq_scatter_offset_s32(
1586 // CHECK-NEXT:  entry:
1587 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 0)
1588 // CHECK-NEXT:    ret void
1589 //
1590 void test_vstrhq_scatter_offset_s32(int16_t *base, uint32x4_t offset, int32x4_t value)
1591 {
1592 #ifdef POLYMORPHIC
1593     vstrhq_scatter_offset(base, offset, value);
1594 #else /* POLYMORPHIC */
1595     vstrhq_scatter_offset_s32(base, offset, value);
1596 #endif /* POLYMORPHIC */
1597 }
1598
1599 // CHECK-LABEL: @test_vstrhq_scatter_offset_u16(
1600 // CHECK-NEXT:  entry:
1601 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 0)
1602 // CHECK-NEXT:    ret void
1603 //
1604 void test_vstrhq_scatter_offset_u16(uint16_t *base, uint16x8_t offset, uint16x8_t value)
1605 {
1606 #ifdef POLYMORPHIC
1607     vstrhq_scatter_offset(base, offset, value);
1608 #else /* POLYMORPHIC */
1609     vstrhq_scatter_offset_u16(base, offset, value);
1610 #endif /* POLYMORPHIC */
1611 }
1612
1613 // CHECK-LABEL: @test_vstrhq_scatter_offset_u32(
1614 // CHECK-NEXT:  entry:
1615 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 0)
1616 // CHECK-NEXT:    ret void
1617 //
1618 void test_vstrhq_scatter_offset_u32(uint16_t *base, uint32x4_t offset, uint32x4_t value)
1619 {
1620 #ifdef POLYMORPHIC
1621     vstrhq_scatter_offset(base, offset, value);
1622 #else /* POLYMORPHIC */
1623     vstrhq_scatter_offset_u32(base, offset, value);
1624 #endif /* POLYMORPHIC */
1625 }
1626
1627 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_f16(
1628 // CHECK-NEXT:  entry:
1629 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8f16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x half> [[VALUE:%.*]], i32 16, i32 1)
1630 // CHECK-NEXT:    ret void
1631 //
1632 void test_vstrhq_scatter_shifted_offset_f16(float16_t *base, uint16x8_t offset, float16x8_t value)
1633 {
1634 #ifdef POLYMORPHIC
1635     vstrhq_scatter_shifted_offset(base, offset, value);
1636 #else /* POLYMORPHIC */
1637     vstrhq_scatter_shifted_offset_f16(base, offset, value);
1638 #endif /* POLYMORPHIC */
1639 }
1640
1641 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_p_f16(
1642 // CHECK-NEXT:  entry:
1643 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1644 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1645 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8f16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x half> [[VALUE:%.*]], i32 16, i32 1, <8 x i1> [[TMP1]])
1646 // CHECK-NEXT:    ret void
1647 //
1648 void test_vstrhq_scatter_shifted_offset_p_f16(float16_t *base, uint16x8_t offset, float16x8_t value, mve_pred16_t p)
1649 {
1650 #ifdef POLYMORPHIC
1651     vstrhq_scatter_shifted_offset_p(base, offset, value, p);
1652 #else /* POLYMORPHIC */
1653     vstrhq_scatter_shifted_offset_p_f16(base, offset, value, p);
1654 #endif /* POLYMORPHIC */
1655 }
1656
1657 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_p_s16(
1658 // CHECK-NEXT:  entry:
1659 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1660 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1661 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 1, <8 x i1> [[TMP1]])
1662 // CHECK-NEXT:    ret void
1663 //
1664 void test_vstrhq_scatter_shifted_offset_p_s16(int16_t *base, uint16x8_t offset, int16x8_t value, mve_pred16_t p)
1665 {
1666 #ifdef POLYMORPHIC
1667     vstrhq_scatter_shifted_offset_p(base, offset, value, p);
1668 #else /* POLYMORPHIC */
1669     vstrhq_scatter_shifted_offset_p_s16(base, offset, value, p);
1670 #endif /* POLYMORPHIC */
1671 }
1672
1673 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_p_s32(
1674 // CHECK-NEXT:  entry:
1675 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1676 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1677 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 1, <4 x i1> [[TMP1]])
1678 // CHECK-NEXT:    ret void
1679 //
1680 void test_vstrhq_scatter_shifted_offset_p_s32(int16_t *base, uint32x4_t offset, int32x4_t value, mve_pred16_t p)
1681 {
1682 #ifdef POLYMORPHIC
1683     vstrhq_scatter_shifted_offset_p(base, offset, value, p);
1684 #else /* POLYMORPHIC */
1685     vstrhq_scatter_shifted_offset_p_s32(base, offset, value, p);
1686 #endif /* POLYMORPHIC */
1687 }
1688
1689 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_p_u16(
1690 // CHECK-NEXT:  entry:
1691 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1692 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
1693 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v8i16.v8i16.v8i1(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 1, <8 x i1> [[TMP1]])
1694 // CHECK-NEXT:    ret void
1695 //
1696 void test_vstrhq_scatter_shifted_offset_p_u16(uint16_t *base, uint16x8_t offset, uint16x8_t value, mve_pred16_t p)
1697 {
1698 #ifdef POLYMORPHIC
1699     vstrhq_scatter_shifted_offset_p(base, offset, value, p);
1700 #else /* POLYMORPHIC */
1701     vstrhq_scatter_shifted_offset_p_u16(base, offset, value, p);
1702 #endif /* POLYMORPHIC */
1703 }
1704
1705 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_p_u32(
1706 // CHECK-NEXT:  entry:
1707 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1708 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1709 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 1, <4 x i1> [[TMP1]])
1710 // CHECK-NEXT:    ret void
1711 //
1712 void test_vstrhq_scatter_shifted_offset_p_u32(uint16_t *base, uint32x4_t offset, uint32x4_t value, mve_pred16_t p)
1713 {
1714 #ifdef POLYMORPHIC
1715     vstrhq_scatter_shifted_offset_p(base, offset, value, p);
1716 #else /* POLYMORPHIC */
1717     vstrhq_scatter_shifted_offset_p_u32(base, offset, value, p);
1718 #endif /* POLYMORPHIC */
1719 }
1720
1721 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_s16(
1722 // CHECK-NEXT:  entry:
1723 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 1)
1724 // CHECK-NEXT:    ret void
1725 //
1726 void test_vstrhq_scatter_shifted_offset_s16(int16_t *base, uint16x8_t offset, int16x8_t value)
1727 {
1728 #ifdef POLYMORPHIC
1729     vstrhq_scatter_shifted_offset(base, offset, value);
1730 #else /* POLYMORPHIC */
1731     vstrhq_scatter_shifted_offset_s16(base, offset, value);
1732 #endif /* POLYMORPHIC */
1733 }
1734
1735 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_s32(
1736 // CHECK-NEXT:  entry:
1737 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 1)
1738 // CHECK-NEXT:    ret void
1739 //
1740 void test_vstrhq_scatter_shifted_offset_s32(int16_t *base, uint32x4_t offset, int32x4_t value)
1741 {
1742 #ifdef POLYMORPHIC
1743     vstrhq_scatter_shifted_offset(base, offset, value);
1744 #else /* POLYMORPHIC */
1745     vstrhq_scatter_shifted_offset_s32(base, offset, value);
1746 #endif /* POLYMORPHIC */
1747 }
1748
1749 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_u16(
1750 // CHECK-NEXT:  entry:
1751 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v8i16.v8i16(ptr [[BASE:%.*]], <8 x i16> [[OFFSET:%.*]], <8 x i16> [[VALUE:%.*]], i32 16, i32 1)
1752 // CHECK-NEXT:    ret void
1753 //
1754 void test_vstrhq_scatter_shifted_offset_u16(uint16_t *base, uint16x8_t offset, uint16x8_t value)
1755 {
1756 #ifdef POLYMORPHIC
1757     vstrhq_scatter_shifted_offset(base, offset, value);
1758 #else /* POLYMORPHIC */
1759     vstrhq_scatter_shifted_offset_u16(base, offset, value);
1760 #endif /* POLYMORPHIC */
1761 }
1762
1763 // CHECK-LABEL: @test_vstrhq_scatter_shifted_offset_u32(
1764 // CHECK-NEXT:  entry:
1765 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 16, i32 1)
1766 // CHECK-NEXT:    ret void
1767 //
1768 void test_vstrhq_scatter_shifted_offset_u32(uint16_t *base, uint32x4_t offset, uint32x4_t value)
1769 {
1770 #ifdef POLYMORPHIC
1771     vstrhq_scatter_shifted_offset(base, offset, value);
1772 #else /* POLYMORPHIC */
1773     vstrhq_scatter_shifted_offset_u32(base, offset, value);
1774 #endif /* POLYMORPHIC */
1775 }
1776
1777 // CHECK-LABEL: @test_vstrwq_scatter_base_f32(
1778 // CHECK-NEXT:  entry:
1779 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4f32(<4 x i32> [[ADDR:%.*]], i32 380, <4 x float> [[VALUE:%.*]])
1780 // CHECK-NEXT:    ret void
1781 //
1782 void test_vstrwq_scatter_base_f32(uint32x4_t addr, float32x4_t value)
1783 {
1784 #ifdef POLYMORPHIC
1785     vstrwq_scatter_base(addr, 0x17c, value);
1786 #else /* POLYMORPHIC */
1787     vstrwq_scatter_base_f32(addr, 0x17c, value);
1788 #endif /* POLYMORPHIC */
1789 }
1790
1791 // CHECK-LABEL: @test_vstrwq_scatter_base_p_f32(
1792 // CHECK-NEXT:  entry:
1793 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1794 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1795 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4f32.v4i1(<4 x i32> [[ADDR:%.*]], i32 -400, <4 x float> [[VALUE:%.*]], <4 x i1> [[TMP1]])
1796 // CHECK-NEXT:    ret void
1797 //
1798 void test_vstrwq_scatter_base_p_f32(uint32x4_t addr, float32x4_t value, mve_pred16_t p)
1799 {
1800 #ifdef POLYMORPHIC
1801     vstrwq_scatter_base_p(addr, -0x190, value, p);
1802 #else /* POLYMORPHIC */
1803     vstrwq_scatter_base_p_f32(addr, -0x190, value, p);
1804 #endif /* POLYMORPHIC */
1805 }
1806
1807 // CHECK-LABEL: @test_vstrwq_scatter_base_p_s32(
1808 // CHECK-NEXT:  entry:
1809 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1810 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1811 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4i32.v4i1(<4 x i32> [[ADDR:%.*]], i32 48, <4 x i32> [[VALUE:%.*]], <4 x i1> [[TMP1]])
1812 // CHECK-NEXT:    ret void
1813 //
1814 void test_vstrwq_scatter_base_p_s32(uint32x4_t addr, int32x4_t value, mve_pred16_t p)
1815 {
1816 #ifdef POLYMORPHIC
1817     vstrwq_scatter_base_p(addr, 0x30, value, p);
1818 #else /* POLYMORPHIC */
1819     vstrwq_scatter_base_p_s32(addr, 0x30, value, p);
1820 #endif /* POLYMORPHIC */
1821 }
1822
1823 // CHECK-LABEL: @test_vstrwq_scatter_base_p_u32(
1824 // CHECK-NEXT:  entry:
1825 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1826 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1827 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.predicated.v4i32.v4i32.v4i1(<4 x i32> [[ADDR:%.*]], i32 -376, <4 x i32> [[VALUE:%.*]], <4 x i1> [[TMP1]])
1828 // CHECK-NEXT:    ret void
1829 //
1830 void test_vstrwq_scatter_base_p_u32(uint32x4_t addr, uint32x4_t value, mve_pred16_t p)
1831 {
1832 #ifdef POLYMORPHIC
1833     vstrwq_scatter_base_p(addr, -0x178, value, p);
1834 #else /* POLYMORPHIC */
1835     vstrwq_scatter_base_p_u32(addr, -0x178, value, p);
1836 #endif /* POLYMORPHIC */
1837 }
1838
1839 // CHECK-LABEL: @test_vstrwq_scatter_base_s32(
1840 // CHECK-NEXT:  entry:
1841 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4i32(<4 x i32> [[ADDR:%.*]], i32 156, <4 x i32> [[VALUE:%.*]])
1842 // CHECK-NEXT:    ret void
1843 //
1844 void test_vstrwq_scatter_base_s32(uint32x4_t addr, int32x4_t value)
1845 {
1846 #ifdef POLYMORPHIC
1847     vstrwq_scatter_base(addr, 0x9c, value);
1848 #else /* POLYMORPHIC */
1849     vstrwq_scatter_base_s32(addr, 0x9c, value);
1850 #endif /* POLYMORPHIC */
1851 }
1852
1853 // CHECK-LABEL: @test_vstrwq_scatter_base_u32(
1854 // CHECK-NEXT:  entry:
1855 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.base.v4i32.v4i32(<4 x i32> [[ADDR:%.*]], i32 212, <4 x i32> [[VALUE:%.*]])
1856 // CHECK-NEXT:    ret void
1857 //
1858 void test_vstrwq_scatter_base_u32(uint32x4_t addr, uint32x4_t value)
1859 {
1860 #ifdef POLYMORPHIC
1861     vstrwq_scatter_base(addr, 0xd4, value);
1862 #else /* POLYMORPHIC */
1863     vstrwq_scatter_base_u32(addr, 0xd4, value);
1864 #endif /* POLYMORPHIC */
1865 }
1866
1867 // CHECK-LABEL: @test_vstrwq_scatter_base_wb_f32(
1868 // CHECK-NEXT:  entry:
1869 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
1870 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4f32(<4 x i32> [[TMP0]], i32 -412, <4 x float> [[VALUE:%.*]])
1871 // CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[ADDR]], align 8
1872 // CHECK-NEXT:    ret void
1873 //
1874 void test_vstrwq_scatter_base_wb_f32(uint32x4_t *addr, float32x4_t value)
1875 {
1876 #ifdef POLYMORPHIC
1877     vstrwq_scatter_base_wb(addr, -0x19c, value);
1878 #else /* POLYMORPHIC */
1879     vstrwq_scatter_base_wb_f32(addr, -0x19c, value);
1880 #endif /* POLYMORPHIC */
1881 }
1882
1883 // CHECK-LABEL: @test_vstrwq_scatter_base_wb_p_f32(
1884 // CHECK-NEXT:  entry:
1885 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
1886 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1887 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
1888 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4f32.v4i1(<4 x i32> [[TMP0]], i32 236, <4 x float> [[VALUE:%.*]], <4 x i1> [[TMP2]])
1889 // CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr [[ADDR]], align 8
1890 // CHECK-NEXT:    ret void
1891 //
1892 void test_vstrwq_scatter_base_wb_p_f32(uint32x4_t *addr, float32x4_t value, mve_pred16_t p)
1893 {
1894 #ifdef POLYMORPHIC
1895     vstrwq_scatter_base_wb_p(addr, 0xec, value, p);
1896 #else /* POLYMORPHIC */
1897     vstrwq_scatter_base_wb_p_f32(addr, 0xec, value, p);
1898 #endif /* POLYMORPHIC */
1899 }
1900
1901 // CHECK-LABEL: @test_vstrwq_scatter_base_wb_p_s32(
1902 // CHECK-NEXT:  entry:
1903 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
1904 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1905 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
1906 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> [[TMP0]], i32 328, <4 x i32> [[VALUE:%.*]], <4 x i1> [[TMP2]])
1907 // CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr [[ADDR]], align 8
1908 // CHECK-NEXT:    ret void
1909 //
1910 void test_vstrwq_scatter_base_wb_p_s32(uint32x4_t *addr, int32x4_t value, mve_pred16_t p)
1911 {
1912 #ifdef POLYMORPHIC
1913     vstrwq_scatter_base_wb_p(addr, 0x148, value, p);
1914 #else /* POLYMORPHIC */
1915     vstrwq_scatter_base_wb_p_s32(addr, 0x148, value, p);
1916 #endif /* POLYMORPHIC */
1917 }
1918
1919 // CHECK-LABEL: @test_vstrwq_scatter_base_wb_p_u32(
1920 // CHECK-NEXT:  entry:
1921 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
1922 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1923 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
1924 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.predicated.v4i32.v4i32.v4i1(<4 x i32> [[TMP0]], i32 412, <4 x i32> [[VALUE:%.*]], <4 x i1> [[TMP2]])
1925 // CHECK-NEXT:    store <4 x i32> [[TMP3]], ptr [[ADDR]], align 8
1926 // CHECK-NEXT:    ret void
1927 //
1928 void test_vstrwq_scatter_base_wb_p_u32(uint32x4_t *addr, uint32x4_t value, mve_pred16_t p)
1929 {
1930 #ifdef POLYMORPHIC
1931     vstrwq_scatter_base_wb_p(addr, 0x19c, value, p);
1932 #else /* POLYMORPHIC */
1933     vstrwq_scatter_base_wb_p_u32(addr, 0x19c, value, p);
1934 #endif /* POLYMORPHIC */
1935 }
1936
1937 // CHECK-LABEL: @test_vstrwq_scatter_base_wb_s32(
1938 // CHECK-NEXT:  entry:
1939 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
1940 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4i32(<4 x i32> [[TMP0]], i32 -152, <4 x i32> [[VALUE:%.*]])
1941 // CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[ADDR]], align 8
1942 // CHECK-NEXT:    ret void
1943 //
1944 void test_vstrwq_scatter_base_wb_s32(uint32x4_t *addr, int32x4_t value)
1945 {
1946 #ifdef POLYMORPHIC
1947     vstrwq_scatter_base_wb(addr, -0x98, value);
1948 #else /* POLYMORPHIC */
1949     vstrwq_scatter_base_wb_s32(addr, -0x98, value);
1950 #endif /* POLYMORPHIC */
1951 }
1952
1953 // CHECK-LABEL: @test_vstrwq_scatter_base_wb_u32(
1954 // CHECK-NEXT:  entry:
1955 // CHECK-NEXT:    [[TMP0:%.*]] = load <4 x i32>, ptr [[ADDR:%.*]], align 8
1956 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i32> @llvm.arm.mve.vstr.scatter.base.wb.v4i32.v4i32(<4 x i32> [[TMP0]], i32 64, <4 x i32> [[VALUE:%.*]])
1957 // CHECK-NEXT:    store <4 x i32> [[TMP1]], ptr [[ADDR]], align 8
1958 // CHECK-NEXT:    ret void
1959 //
1960 void test_vstrwq_scatter_base_wb_u32(uint32x4_t *addr, uint32x4_t value)
1961 {
1962 #ifdef POLYMORPHIC
1963     vstrwq_scatter_base_wb(addr, 0x40, value);
1964 #else /* POLYMORPHIC */
1965     vstrwq_scatter_base_wb_u32(addr, 0x40, value);
1966 #endif /* POLYMORPHIC */
1967 }
1968
1969 // CHECK-LABEL: @test_vstrwq_scatter_offset_f32(
1970 // CHECK-NEXT:  entry:
1971 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4f32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x float> [[VALUE:%.*]], i32 32, i32 0)
1972 // CHECK-NEXT:    ret void
1973 //
1974 void test_vstrwq_scatter_offset_f32(float32_t *base, uint32x4_t offset, float32x4_t value)
1975 {
1976 #ifdef POLYMORPHIC
1977     vstrwq_scatter_offset(base, offset, value);
1978 #else /* POLYMORPHIC */
1979     vstrwq_scatter_offset_f32(base, offset, value);
1980 #endif /* POLYMORPHIC */
1981 }
1982
1983 // CHECK-LABEL: @test_vstrwq_scatter_offset_p_f32(
1984 // CHECK-NEXT:  entry:
1985 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
1986 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
1987 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4f32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x float> [[VALUE:%.*]], i32 32, i32 0, <4 x i1> [[TMP1]])
1988 // CHECK-NEXT:    ret void
1989 //
1990 void test_vstrwq_scatter_offset_p_f32(float32_t *base, uint32x4_t offset, float32x4_t value, mve_pred16_t p)
1991 {
1992 #ifdef POLYMORPHIC
1993     vstrwq_scatter_offset_p(base, offset, value, p);
1994 #else /* POLYMORPHIC */
1995     vstrwq_scatter_offset_p_f32(base, offset, value, p);
1996 #endif /* POLYMORPHIC */
1997 }
1998
1999 // CHECK-LABEL: @test_vstrwq_scatter_offset_p_s32(
2000 // CHECK-NEXT:  entry:
2001 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
2002 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
2003 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 0, <4 x i1> [[TMP1]])
2004 // CHECK-NEXT:    ret void
2005 //
2006 void test_vstrwq_scatter_offset_p_s32(int32_t *base, uint32x4_t offset, int32x4_t value, mve_pred16_t p)
2007 {
2008 #ifdef POLYMORPHIC
2009     vstrwq_scatter_offset_p(base, offset, value, p);
2010 #else /* POLYMORPHIC */
2011     vstrwq_scatter_offset_p_s32(base, offset, value, p);
2012 #endif /* POLYMORPHIC */
2013 }
2014
2015 // CHECK-LABEL: @test_vstrwq_scatter_offset_p_u32(
2016 // CHECK-NEXT:  entry:
2017 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
2018 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
2019 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 0, <4 x i1> [[TMP1]])
2020 // CHECK-NEXT:    ret void
2021 //
2022 void test_vstrwq_scatter_offset_p_u32(uint32_t *base, uint32x4_t offset, uint32x4_t value, mve_pred16_t p)
2023 {
2024 #ifdef POLYMORPHIC
2025     vstrwq_scatter_offset_p(base, offset, value, p);
2026 #else /* POLYMORPHIC */
2027     vstrwq_scatter_offset_p_u32(base, offset, value, p);
2028 #endif /* POLYMORPHIC */
2029 }
2030
2031 // CHECK-LABEL: @test_vstrwq_scatter_offset_s32(
2032 // CHECK-NEXT:  entry:
2033 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 0)
2034 // CHECK-NEXT:    ret void
2035 //
2036 void test_vstrwq_scatter_offset_s32(int32_t *base, uint32x4_t offset, int32x4_t value)
2037 {
2038 #ifdef POLYMORPHIC
2039     vstrwq_scatter_offset(base, offset, value);
2040 #else /* POLYMORPHIC */
2041     vstrwq_scatter_offset_s32(base, offset, value);
2042 #endif /* POLYMORPHIC */
2043 }
2044
2045 // CHECK-LABEL: @test_vstrwq_scatter_offset_u32(
2046 // CHECK-NEXT:  entry:
2047 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 0)
2048 // CHECK-NEXT:    ret void
2049 //
2050 void test_vstrwq_scatter_offset_u32(uint32_t *base, uint32x4_t offset, uint32x4_t value)
2051 {
2052 #ifdef POLYMORPHIC
2053     vstrwq_scatter_offset(base, offset, value);
2054 #else /* POLYMORPHIC */
2055     vstrwq_scatter_offset_u32(base, offset, value);
2056 #endif /* POLYMORPHIC */
2057 }
2058
2059 // CHECK-LABEL: @test_vstrwq_scatter_shifted_offset_f32(
2060 // CHECK-NEXT:  entry:
2061 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4f32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x float> [[VALUE:%.*]], i32 32, i32 2)
2062 // CHECK-NEXT:    ret void
2063 //
2064 void test_vstrwq_scatter_shifted_offset_f32(float32_t *base, uint32x4_t offset, float32x4_t value)
2065 {
2066 #ifdef POLYMORPHIC
2067     vstrwq_scatter_shifted_offset(base, offset, value);
2068 #else /* POLYMORPHIC */
2069     vstrwq_scatter_shifted_offset_f32(base, offset, value);
2070 #endif /* POLYMORPHIC */
2071 }
2072
2073 // CHECK-LABEL: @test_vstrwq_scatter_shifted_offset_p_f32(
2074 // CHECK-NEXT:  entry:
2075 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
2076 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
2077 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4f32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x float> [[VALUE:%.*]], i32 32, i32 2, <4 x i1> [[TMP1]])
2078 // CHECK-NEXT:    ret void
2079 //
2080 void test_vstrwq_scatter_shifted_offset_p_f32(float32_t *base, uint32x4_t offset, float32x4_t value, mve_pred16_t p)
2081 {
2082 #ifdef POLYMORPHIC
2083     vstrwq_scatter_shifted_offset_p(base, offset, value, p);
2084 #else /* POLYMORPHIC */
2085     vstrwq_scatter_shifted_offset_p_f32(base, offset, value, p);
2086 #endif /* POLYMORPHIC */
2087 }
2088
2089 // CHECK-LABEL: @test_vstrwq_scatter_shifted_offset_p_s32(
2090 // CHECK-NEXT:  entry:
2091 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
2092 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
2093 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 2, <4 x i1> [[TMP1]])
2094 // CHECK-NEXT:    ret void
2095 //
2096 void test_vstrwq_scatter_shifted_offset_p_s32(int32_t *base, uint32x4_t offset, int32x4_t value, mve_pred16_t p)
2097 {
2098 #ifdef POLYMORPHIC
2099     vstrwq_scatter_shifted_offset_p(base, offset, value, p);
2100 #else /* POLYMORPHIC */
2101     vstrwq_scatter_shifted_offset_p_s32(base, offset, value, p);
2102 #endif /* POLYMORPHIC */
2103 }
2104
2105 // CHECK-LABEL: @test_vstrwq_scatter_shifted_offset_p_u32(
2106 // CHECK-NEXT:  entry:
2107 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
2108 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
2109 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.predicated.p0.v4i32.v4i32.v4i1(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 2, <4 x i1> [[TMP1]])
2110 // CHECK-NEXT:    ret void
2111 //
2112 void test_vstrwq_scatter_shifted_offset_p_u32(uint32_t *base, uint32x4_t offset, uint32x4_t value, mve_pred16_t p)
2113 {
2114 #ifdef POLYMORPHIC
2115     vstrwq_scatter_shifted_offset_p(base, offset, value, p);
2116 #else /* POLYMORPHIC */
2117     vstrwq_scatter_shifted_offset_p_u32(base, offset, value, p);
2118 #endif /* POLYMORPHIC */
2119 }
2120
2121 // CHECK-LABEL: @test_vstrwq_scatter_shifted_offset_s32(
2122 // CHECK-NEXT:  entry:
2123 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 2)
2124 // CHECK-NEXT:    ret void
2125 //
2126 void test_vstrwq_scatter_shifted_offset_s32(int32_t *base, uint32x4_t offset, int32x4_t value)
2127 {
2128 #ifdef POLYMORPHIC
2129     vstrwq_scatter_shifted_offset(base, offset, value);
2130 #else /* POLYMORPHIC */
2131     vstrwq_scatter_shifted_offset_s32(base, offset, value);
2132 #endif /* POLYMORPHIC */
2133 }
2134
2135 // CHECK-LABEL: @test_vstrwq_scatter_shifted_offset_u32(
2136 // CHECK-NEXT:  entry:
2137 // CHECK-NEXT:    call void @llvm.arm.mve.vstr.scatter.offset.p0.v4i32.v4i32(ptr [[BASE:%.*]], <4 x i32> [[OFFSET:%.*]], <4 x i32> [[VALUE:%.*]], i32 32, i32 2)
2138 // CHECK-NEXT:    ret void
2139 //
2140 void test_vstrwq_scatter_shifted_offset_u32(uint32_t *base, uint32x4_t offset, uint32x4_t value)
2141 {
2142 #ifdef POLYMORPHIC
2143     vstrwq_scatter_shifted_offset(base, offset, value);
2144 #else /* POLYMORPHIC */
2145     vstrwq_scatter_shifted_offset_u32(base, offset, value);
2146 #endif /* POLYMORPHIC */
2147 }
2148