clang/test/CodeGen/arm-mve-intrinsics/load-store.c

   1 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
   2 // RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
   3 // RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -S -emit-llvm -o - %s | opt -S -passes=mem2reg | FileCheck %s
   4
   5 // REQUIRES: aarch64-registered-target || arm-registered-target
   6
   7 #include <arm_mve.h>
   8
   9 // CHECK-LABEL: @test_vld1q_f16(
  10 // CHECK-NEXT:  entry:
  11 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x half>, ptr [[BASE:%.*]], align 2
  12 // CHECK-NEXT:    ret <8 x half> [[TMP1]]
  13 //
  14 float16x8_t test_vld1q_f16(const float16_t *base)
  15 {
  16 #ifdef POLYMORPHIC
  17     return vld1q(base);
  18 #else /* POLYMORPHIC */
  19     return vld1q_f16(base);
  20 #endif /* POLYMORPHIC */
  21 }
  22
  23 // CHECK-LABEL: @test_vld1q_f32(
  24 // CHECK-NEXT:  entry:
  25 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[BASE:%.*]], align 4
  26 // CHECK-NEXT:    ret <4 x float> [[TMP1]]
  27 //
  28 float32x4_t test_vld1q_f32(const float32_t *base)
  29 {
  30 #ifdef POLYMORPHIC
  31     return vld1q(base);
  32 #else /* POLYMORPHIC */
  33     return vld1q_f32(base);
  34 #endif /* POLYMORPHIC */
  35 }
  36
  37 // CHECK-LABEL: @test_vld1q_s8(
  38 // CHECK-NEXT:  entry:
  39 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[BASE:%.*]], align 1
  40 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
  41 //
  42 int8x16_t test_vld1q_s8(const int8_t *base)
  43 {
  44 #ifdef POLYMORPHIC
  45     return vld1q(base);
  46 #else /* POLYMORPHIC */
  47     return vld1q_s8(base);
  48 #endif /* POLYMORPHIC */
  49 }
  50
  51 // CHECK-LABEL: @test_vld1q_s16(
  52 // CHECK-NEXT:  entry:
  53 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[BASE:%.*]], align 2
  54 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
  55 //
  56 int16x8_t test_vld1q_s16(const int16_t *base)
  57 {
  58 #ifdef POLYMORPHIC
  59     return vld1q(base);
  60 #else /* POLYMORPHIC */
  61     return vld1q_s16(base);
  62 #endif /* POLYMORPHIC */
  63 }
  64
  65 // CHECK-LABEL: @test_vld1q_s32(
  66 // CHECK-NEXT:  entry:
  67 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[BASE:%.*]], align 4
  68 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
  69 //
  70 int32x4_t test_vld1q_s32(const int32_t *base)
  71 {
  72 #ifdef POLYMORPHIC
  73     return vld1q(base);
  74 #else /* POLYMORPHIC */
  75     return vld1q_s32(base);
  76 #endif /* POLYMORPHIC */
  77 }
  78
  79 // CHECK-LABEL: @test_vld1q_u8(
  80 // CHECK-NEXT:  entry:
  81 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[BASE:%.*]], align 1
  82 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
  83 //
  84 uint8x16_t test_vld1q_u8(const uint8_t *base)
  85 {
  86 #ifdef POLYMORPHIC
  87     return vld1q(base);
  88 #else /* POLYMORPHIC */
  89     return vld1q_u8(base);
  90 #endif /* POLYMORPHIC */
  91 }
  92
  93 // CHECK-LABEL: @test_vld1q_u16(
  94 // CHECK-NEXT:  entry:
  95 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[BASE:%.*]], align 2
  96 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
  97 //
  98 uint16x8_t test_vld1q_u16(const uint16_t *base)
  99 {
 100 #ifdef POLYMORPHIC
 101     return vld1q(base);
 102 #else /* POLYMORPHIC */
 103     return vld1q_u16(base);
 104 #endif /* POLYMORPHIC */
 105 }
 106
 107 // CHECK-LABEL: @test_vld1q_u32(
 108 // CHECK-NEXT:  entry:
 109 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[BASE:%.*]], align 4
 110 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 111 //
 112 uint32x4_t test_vld1q_u32(const uint32_t *base)
 113 {
 114 #ifdef POLYMORPHIC
 115     return vld1q(base);
 116 #else /* POLYMORPHIC */
 117     return vld1q_u32(base);
 118 #endif /* POLYMORPHIC */
 119 }
 120
 121 // CHECK-LABEL: @test_vld1q_z_f16(
 122 // CHECK-NEXT:  entry:
 123 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 124 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 125 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x half> @llvm.masked.load.v8f16.p0(ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]], <8 x half> zeroinitializer)
 126 // CHECK-NEXT:    ret <8 x half> [[TMP3]]
 127 //
 128 float16x8_t test_vld1q_z_f16(const float16_t *base, mve_pred16_t p)
 129 {
 130 #ifdef POLYMORPHIC
 131     return vld1q_z(base, p);
 132 #else /* POLYMORPHIC */
 133     return vld1q_z_f16(base, p);
 134 #endif /* POLYMORPHIC */
 135 }
 136
 137 // CHECK-LABEL: @test_vld1q_z_f32(
 138 // CHECK-NEXT:  entry:
 139 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 140 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 141 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]], <4 x float> zeroinitializer)
 142 // CHECK-NEXT:    ret <4 x float> [[TMP3]]
 143 //
 144 float32x4_t test_vld1q_z_f32(const float32_t *base, mve_pred16_t p)
 145 {
 146 #ifdef POLYMORPHIC
 147     return vld1q_z(base, p);
 148 #else /* POLYMORPHIC */
 149     return vld1q_z_f32(base, p);
 150 #endif /* POLYMORPHIC */
 151 }
 152
 153 // CHECK-LABEL: @test_vld1q_z_s8(
 154 // CHECK-NEXT:  entry:
 155 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 156 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 157 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]], <16 x i8> zeroinitializer)
 158 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 159 //
 160 int8x16_t test_vld1q_z_s8(const int8_t *base, mve_pred16_t p)
 161 {
 162 #ifdef POLYMORPHIC
 163     return vld1q_z(base, p);
 164 #else /* POLYMORPHIC */
 165     return vld1q_z_s8(base, p);
 166 #endif /* POLYMORPHIC */
 167 }
 168
 169 // CHECK-LABEL: @test_vld1q_z_s16(
 170 // CHECK-NEXT:  entry:
 171 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 172 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 173 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]], <8 x i16> zeroinitializer)
 174 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 175 //
 176 int16x8_t test_vld1q_z_s16(const int16_t *base, mve_pred16_t p)
 177 {
 178 #ifdef POLYMORPHIC
 179     return vld1q_z(base, p);
 180 #else /* POLYMORPHIC */
 181     return vld1q_z_s16(base, p);
 182 #endif /* POLYMORPHIC */
 183 }
 184
 185 // CHECK-LABEL: @test_vld1q_z_s32(
 186 // CHECK-NEXT:  entry:
 187 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 188 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 189 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]], <4 x i32> zeroinitializer)
 190 // CHECK-NEXT:    ret <4 x i32> [[TMP3]]
 191 //
 192 int32x4_t test_vld1q_z_s32(const int32_t *base, mve_pred16_t p)
 193 {
 194 #ifdef POLYMORPHIC
 195     return vld1q_z(base, p);
 196 #else /* POLYMORPHIC */
 197     return vld1q_z_s32(base, p);
 198 #endif /* POLYMORPHIC */
 199 }
 200
 201 // CHECK-LABEL: @test_vld1q_z_u8(
 202 // CHECK-NEXT:  entry:
 203 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 204 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 205 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]], <16 x i8> zeroinitializer)
 206 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 207 //
 208 uint8x16_t test_vld1q_z_u8(const uint8_t *base, mve_pred16_t p)
 209 {
 210 #ifdef POLYMORPHIC
 211     return vld1q_z(base, p);
 212 #else /* POLYMORPHIC */
 213     return vld1q_z_u8(base, p);
 214 #endif /* POLYMORPHIC */
 215 }
 216
 217 // CHECK-LABEL: @test_vld1q_z_u16(
 218 // CHECK-NEXT:  entry:
 219 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 220 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 221 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]], <8 x i16> zeroinitializer)
 222 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 223 //
 224 uint16x8_t test_vld1q_z_u16(const uint16_t *base, mve_pred16_t p)
 225 {
 226 #ifdef POLYMORPHIC
 227     return vld1q_z(base, p);
 228 #else /* POLYMORPHIC */
 229     return vld1q_z_u16(base, p);
 230 #endif /* POLYMORPHIC */
 231 }
 232
 233 // CHECK-LABEL: @test_vld1q_z_u32(
 234 // CHECK-NEXT:  entry:
 235 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 236 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 237 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]], <4 x i32> zeroinitializer)
 238 // CHECK-NEXT:    ret <4 x i32> [[TMP3]]
 239 //
 240 uint32x4_t test_vld1q_z_u32(const uint32_t *base, mve_pred16_t p)
 241 {
 242 #ifdef POLYMORPHIC
 243     return vld1q_z(base, p);
 244 #else /* POLYMORPHIC */
 245     return vld1q_z_u32(base, p);
 246 #endif /* POLYMORPHIC */
 247 }
 248
 249 // CHECK-LABEL: @test_vldrbq_s8(
 250 // CHECK-NEXT:  entry:
 251 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[BASE:%.*]], align 1
 252 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 253 //
 254 int8x16_t test_vldrbq_s8(const int8_t *base)
 255 {
 256     return vldrbq_s8(base);
 257 }
 258
 259 // CHECK-LABEL: @test_vldrbq_s16(
 260 // CHECK-NEXT:  entry:
 261 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[BASE:%.*]], align 1
 262 // CHECK-NEXT:    [[TMP2:%.*]] = sext <8 x i8> [[TMP1]] to <8 x i16>
 263 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 264 //
 265 int16x8_t test_vldrbq_s16(const int8_t *base)
 266 {
 267     return vldrbq_s16(base);
 268 }
 269
 270 // CHECK-LABEL: @test_vldrbq_s32(
 271 // CHECK-NEXT:  entry:
 272 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[BASE:%.*]], align 1
 273 // CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i8> [[TMP1]] to <4 x i32>
 274 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 275 //
 276 int32x4_t test_vldrbq_s32(const int8_t *base)
 277 {
 278     return vldrbq_s32(base);
 279 }
 280
 281 // CHECK-LABEL: @test_vldrbq_u8(
 282 // CHECK-NEXT:  entry:
 283 // CHECK-NEXT:    [[TMP1:%.*]] = load <16 x i8>, ptr [[BASE:%.*]], align 1
 284 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 285 //
 286 uint8x16_t test_vldrbq_u8(const uint8_t *base)
 287 {
 288     return vldrbq_u8(base);
 289 }
 290
 291 // CHECK-LABEL: @test_vldrbq_u16(
 292 // CHECK-NEXT:  entry:
 293 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i8>, ptr [[BASE:%.*]], align 1
 294 // CHECK-NEXT:    [[TMP2:%.*]] = zext <8 x i8> [[TMP1]] to <8 x i16>
 295 // CHECK-NEXT:    ret <8 x i16> [[TMP2]]
 296 //
 297 uint16x8_t test_vldrbq_u16(const uint8_t *base)
 298 {
 299     return vldrbq_u16(base);
 300 }
 301
 302 // CHECK-LABEL: @test_vldrbq_u32(
 303 // CHECK-NEXT:  entry:
 304 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i8>, ptr [[BASE:%.*]], align 1
 305 // CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i8> [[TMP1]] to <4 x i32>
 306 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 307 //
 308 uint32x4_t test_vldrbq_u32(const uint8_t *base)
 309 {
 310     return vldrbq_u32(base);
 311 }
 312
 313 // CHECK-LABEL: @test_vldrbq_z_s8(
 314 // CHECK-NEXT:  entry:
 315 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 316 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 317 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]], <16 x i8> zeroinitializer)
 318 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 319 //
 320 int8x16_t test_vldrbq_z_s8(const int8_t *base, mve_pred16_t p)
 321 {
 322     return vldrbq_z_s8(base, p);
 323 }
 324
 325 // CHECK-LABEL: @test_vldrbq_z_s16(
 326 // CHECK-NEXT:  entry:
 327 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 328 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 329 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr [[BASE:%.*]], i32 1, <8 x i1> [[TMP2]], <8 x i8> zeroinitializer)
 330 // CHECK-NEXT:    [[TMP4:%.*]] = sext <8 x i8> [[TMP3]] to <8 x i16>
 331 // CHECK-NEXT:    ret <8 x i16> [[TMP4]]
 332 //
 333 int16x8_t test_vldrbq_z_s16(const int8_t *base, mve_pred16_t p)
 334 {
 335     return vldrbq_z_s16(base, p);
 336 }
 337
 338 // CHECK-LABEL: @test_vldrbq_z_s32(
 339 // CHECK-NEXT:  entry:
 340 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 341 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 342 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr [[BASE:%.*]], i32 1, <4 x i1> [[TMP2]], <4 x i8> zeroinitializer)
 343 // CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i8> [[TMP3]] to <4 x i32>
 344 // CHECK-NEXT:    ret <4 x i32> [[TMP4]]
 345 //
 346 int32x4_t test_vldrbq_z_s32(const int8_t *base, mve_pred16_t p)
 347 {
 348     return vldrbq_z_s32(base, p);
 349 }
 350
 351 // CHECK-LABEL: @test_vldrbq_z_u8(
 352 // CHECK-NEXT:  entry:
 353 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 354 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 355 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.masked.load.v16i8.p0(ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]], <16 x i8> zeroinitializer)
 356 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 357 //
 358 uint8x16_t test_vldrbq_z_u8(const uint8_t *base, mve_pred16_t p)
 359 {
 360     return vldrbq_z_u8(base, p);
 361 }
 362
 363 // CHECK-LABEL: @test_vldrbq_z_u16(
 364 // CHECK-NEXT:  entry:
 365 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 366 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 367 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr [[BASE:%.*]], i32 1, <8 x i1> [[TMP2]], <8 x i8> zeroinitializer)
 368 // CHECK-NEXT:    [[TMP4:%.*]] = zext <8 x i8> [[TMP3]] to <8 x i16>
 369 // CHECK-NEXT:    ret <8 x i16> [[TMP4]]
 370 //
 371 uint16x8_t test_vldrbq_z_u16(const uint8_t *base, mve_pred16_t p)
 372 {
 373     return vldrbq_z_u16(base, p);
 374 }
 375
 376 // CHECK-LABEL: @test_vldrbq_z_u32(
 377 // CHECK-NEXT:  entry:
 378 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 379 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 380 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr [[BASE:%.*]], i32 1, <4 x i1> [[TMP2]], <4 x i8> zeroinitializer)
 381 // CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i8> [[TMP3]] to <4 x i32>
 382 // CHECK-NEXT:    ret <4 x i32> [[TMP4]]
 383 //
 384 uint32x4_t test_vldrbq_z_u32(const uint8_t *base, mve_pred16_t p)
 385 {
 386     return vldrbq_z_u32(base, p);
 387 }
 388
 389 // CHECK-LABEL: @test_vldrhq_f16(
 390 // CHECK-NEXT:  entry:
 391 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x half>, ptr [[BASE:%.*]], align 2
 392 // CHECK-NEXT:    ret <8 x half> [[TMP1]]
 393 //
 394 float16x8_t test_vldrhq_f16(const float16_t *base)
 395 {
 396     return vldrhq_f16(base);
 397 }
 398
 399 // CHECK-LABEL: @test_vldrhq_s16(
 400 // CHECK-NEXT:  entry:
 401 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[BASE:%.*]], align 2
 402 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 403 //
 404 int16x8_t test_vldrhq_s16(const int16_t *base)
 405 {
 406     return vldrhq_s16(base);
 407 }
 408
 409 // CHECK-LABEL: @test_vldrhq_s32(
 410 // CHECK-NEXT:  entry:
 411 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[BASE:%.*]], align 2
 412 // CHECK-NEXT:    [[TMP2:%.*]] = sext <4 x i16> [[TMP1]] to <4 x i32>
 413 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 414 //
 415 int32x4_t test_vldrhq_s32(const int16_t *base)
 416 {
 417     return vldrhq_s32(base);
 418 }
 419
 420 // CHECK-LABEL: @test_vldrhq_u16(
 421 // CHECK-NEXT:  entry:
 422 // CHECK-NEXT:    [[TMP1:%.*]] = load <8 x i16>, ptr [[BASE:%.*]], align 2
 423 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 424 //
 425 uint16x8_t test_vldrhq_u16(const uint16_t *base)
 426 {
 427     return vldrhq_u16(base);
 428 }
 429
 430 // CHECK-LABEL: @test_vldrhq_u32(
 431 // CHECK-NEXT:  entry:
 432 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i16>, ptr [[BASE:%.*]], align 2
 433 // CHECK-NEXT:    [[TMP2:%.*]] = zext <4 x i16> [[TMP1]] to <4 x i32>
 434 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 435 //
 436 uint32x4_t test_vldrhq_u32(const uint16_t *base)
 437 {
 438     return vldrhq_u32(base);
 439 }
 440
 441 // CHECK-LABEL: @test_vldrhq_z_f16(
 442 // CHECK-NEXT:  entry:
 443 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 444 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 445 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x half> @llvm.masked.load.v8f16.p0(ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]], <8 x half> zeroinitializer)
 446 // CHECK-NEXT:    ret <8 x half> [[TMP3]]
 447 //
 448 float16x8_t test_vldrhq_z_f16(const float16_t *base, mve_pred16_t p)
 449 {
 450     return vldrhq_z_f16(base, p);
 451 }
 452
 453 // CHECK-LABEL: @test_vldrhq_z_s16(
 454 // CHECK-NEXT:  entry:
 455 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 456 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 457 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]], <8 x i16> zeroinitializer)
 458 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 459 //
 460 int16x8_t test_vldrhq_z_s16(const int16_t *base, mve_pred16_t p)
 461 {
 462     return vldrhq_z_s16(base, p);
 463 }
 464
 465 // CHECK-LABEL: @test_vldrhq_z_s32(
 466 // CHECK-NEXT:  entry:
 467 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 468 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 469 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[BASE:%.*]], i32 2, <4 x i1> [[TMP2]], <4 x i16> zeroinitializer)
 470 // CHECK-NEXT:    [[TMP4:%.*]] = sext <4 x i16> [[TMP3]] to <4 x i32>
 471 // CHECK-NEXT:    ret <4 x i32> [[TMP4]]
 472 //
 473 int32x4_t test_vldrhq_z_s32(const int16_t *base, mve_pred16_t p)
 474 {
 475     return vldrhq_z_s32(base, p);
 476 }
 477
 478 // CHECK-LABEL: @test_vldrhq_z_u16(
 479 // CHECK-NEXT:  entry:
 480 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 481 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 482 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]], <8 x i16> zeroinitializer)
 483 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 484 //
 485 uint16x8_t test_vldrhq_z_u16(const uint16_t *base, mve_pred16_t p)
 486 {
 487     return vldrhq_z_u16(base, p);
 488 }
 489
 490 // CHECK-LABEL: @test_vldrhq_z_u32(
 491 // CHECK-NEXT:  entry:
 492 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 493 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 494 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i16> @llvm.masked.load.v4i16.p0(ptr [[BASE:%.*]], i32 2, <4 x i1> [[TMP2]], <4 x i16> zeroinitializer)
 495 // CHECK-NEXT:    [[TMP4:%.*]] = zext <4 x i16> [[TMP3]] to <4 x i32>
 496 // CHECK-NEXT:    ret <4 x i32> [[TMP4]]
 497 //
 498 uint32x4_t test_vldrhq_z_u32(const uint16_t *base, mve_pred16_t p)
 499 {
 500     return vldrhq_z_u32(base, p);
 501 }
 502
 503 // CHECK-LABEL: @test_vldrwq_f32(
 504 // CHECK-NEXT:  entry:
 505 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x float>, ptr [[BASE:%.*]], align 4
 506 // CHECK-NEXT:    ret <4 x float> [[TMP1]]
 507 //
 508 float32x4_t test_vldrwq_f32(const float32_t *base)
 509 {
 510     return vldrwq_f32(base);
 511 }
 512
 513 // CHECK-LABEL: @test_vldrwq_s32(
 514 // CHECK-NEXT:  entry:
 515 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[BASE:%.*]], align 4
 516 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 517 //
 518 int32x4_t test_vldrwq_s32(const int32_t *base)
 519 {
 520     return vldrwq_s32(base);
 521 }
 522
 523 // CHECK-LABEL: @test_vldrwq_u32(
 524 // CHECK-NEXT:  entry:
 525 // CHECK-NEXT:    [[TMP1:%.*]] = load <4 x i32>, ptr [[BASE:%.*]], align 4
 526 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 527 //
 528 uint32x4_t test_vldrwq_u32(const uint32_t *base)
 529 {
 530     return vldrwq_u32(base);
 531 }
 532
 533 // CHECK-LABEL: @test_vldrwq_z_f32(
 534 // CHECK-NEXT:  entry:
 535 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 536 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 537 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.masked.load.v4f32.p0(ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]], <4 x float> zeroinitializer)
 538 // CHECK-NEXT:    ret <4 x float> [[TMP3]]
 539 //
 540 float32x4_t test_vldrwq_z_f32(const float32_t *base, mve_pred16_t p)
 541 {
 542     return vldrwq_z_f32(base, p);
 543 }
 544
 545 // CHECK-LABEL: @test_vldrwq_z_s32(
 546 // CHECK-NEXT:  entry:
 547 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 548 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 549 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]], <4 x i32> zeroinitializer)
 550 // CHECK-NEXT:    ret <4 x i32> [[TMP3]]
 551 //
 552 int32x4_t test_vldrwq_z_s32(const int32_t *base, mve_pred16_t p)
 553 {
 554     return vldrwq_z_s32(base, p);
 555 }
 556
 557 // CHECK-LABEL: @test_vldrwq_z_u32(
 558 // CHECK-NEXT:  entry:
 559 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 560 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 561 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]], <4 x i32> zeroinitializer)
 562 // CHECK-NEXT:    ret <4 x i32> [[TMP3]]
 563 //
 564 uint32x4_t test_vldrwq_z_u32(const uint32_t *base, mve_pred16_t p)
 565 {
 566     return vldrwq_z_u32(base, p);
 567 }
 568
 569 // CHECK-LABEL: @test_vst1q_f16(
 570 // CHECK-NEXT:  entry:
 571 // CHECK-NEXT:    store <8 x half> [[VALUE:%.*]], ptr [[BASE:%.*]], align 2
 572 // CHECK-NEXT:    ret void
 573 //
 574 void test_vst1q_f16(float16_t *base, float16x8_t value)
 575 {
 576 #ifdef POLYMORPHIC
 577     vst1q(base, value);
 578 #else /* POLYMORPHIC */
 579     vst1q_f16(base, value);
 580 #endif /* POLYMORPHIC */
 581 }
 582
 583 // CHECK-LABEL: @test_vst1q_f32(
 584 // CHECK-NEXT:  entry:
 585 // CHECK-NEXT:    store <4 x float> [[VALUE:%.*]], ptr [[BASE:%.*]], align 4
 586 // CHECK-NEXT:    ret void
 587 //
 588 void test_vst1q_f32(float32_t *base, float32x4_t value)
 589 {
 590 #ifdef POLYMORPHIC
 591     vst1q(base, value);
 592 #else /* POLYMORPHIC */
 593     vst1q_f32(base, value);
 594 #endif /* POLYMORPHIC */
 595 }
 596
 597 // CHECK-LABEL: @test_vst1q_s8(
 598 // CHECK-NEXT:  entry:
 599 // CHECK-NEXT:    store <16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], align 1
 600 // CHECK-NEXT:    ret void
 601 //
 602 void test_vst1q_s8(int8_t *base, int8x16_t value)
 603 {
 604 #ifdef POLYMORPHIC
 605     vst1q(base, value);
 606 #else /* POLYMORPHIC */
 607     vst1q_s8(base, value);
 608 #endif /* POLYMORPHIC */
 609 }
 610
 611 // CHECK-LABEL: @test_vst1q_s16(
 612 // CHECK-NEXT:  entry:
 613 // CHECK-NEXT:    store <8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], align 2
 614 // CHECK-NEXT:    ret void
 615 //
 616 void test_vst1q_s16(int16_t *base, int16x8_t value)
 617 {
 618 #ifdef POLYMORPHIC
 619     vst1q(base, value);
 620 #else /* POLYMORPHIC */
 621     vst1q_s16(base, value);
 622 #endif /* POLYMORPHIC */
 623 }
 624
 625 // CHECK-LABEL: @test_vst1q_s32(
 626 // CHECK-NEXT:  entry:
 627 // CHECK-NEXT:    store <4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], align 4
 628 // CHECK-NEXT:    ret void
 629 //
 630 void test_vst1q_s32(int32_t *base, int32x4_t value)
 631 {
 632 #ifdef POLYMORPHIC
 633     vst1q(base, value);
 634 #else /* POLYMORPHIC */
 635     vst1q_s32(base, value);
 636 #endif /* POLYMORPHIC */
 637 }
 638
 639 // CHECK-LABEL: @test_vst1q_u8(
 640 // CHECK-NEXT:  entry:
 641 // CHECK-NEXT:    store <16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], align 1
 642 // CHECK-NEXT:    ret void
 643 //
 644 void test_vst1q_u8(uint8_t *base, uint8x16_t value)
 645 {
 646 #ifdef POLYMORPHIC
 647     vst1q(base, value);
 648 #else /* POLYMORPHIC */
 649     vst1q_u8(base, value);
 650 #endif /* POLYMORPHIC */
 651 }
 652
 653 // CHECK-LABEL: @test_vst1q_u16(
 654 // CHECK-NEXT:  entry:
 655 // CHECK-NEXT:    store <8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], align 2
 656 // CHECK-NEXT:    ret void
 657 //
 658 void test_vst1q_u16(uint16_t *base, uint16x8_t value)
 659 {
 660 #ifdef POLYMORPHIC
 661     vst1q(base, value);
 662 #else /* POLYMORPHIC */
 663     vst1q_u16(base, value);
 664 #endif /* POLYMORPHIC */
 665 }
 666
 667 // CHECK-LABEL: @test_vst1q_u32(
 668 // CHECK-NEXT:  entry:
 669 // CHECK-NEXT:    store <4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], align 4
 670 // CHECK-NEXT:    ret void
 671 //
 672 void test_vst1q_u32(uint32_t *base, uint32x4_t value)
 673 {
 674 #ifdef POLYMORPHIC
 675     vst1q(base, value);
 676 #else /* POLYMORPHIC */
 677     vst1q_u32(base, value);
 678 #endif /* POLYMORPHIC */
 679 }
 680
 681 // CHECK-LABEL: @test_vst1q_p_f16(
 682 // CHECK-NEXT:  entry:
 683 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 684 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 685 // CHECK-NEXT:    call void @llvm.masked.store.v8f16.p0(<8 x half> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]])
 686 // CHECK-NEXT:    ret void
 687 //
 688 void test_vst1q_p_f16(float16_t *base, float16x8_t value, mve_pred16_t p)
 689 {
 690 #ifdef POLYMORPHIC
 691     vst1q_p(base, value, p);
 692 #else /* POLYMORPHIC */
 693     vst1q_p_f16(base, value, p);
 694 #endif /* POLYMORPHIC */
 695 }
 696
 697 // CHECK-LABEL: @test_vst1q_p_f32(
 698 // CHECK-NEXT:  entry:
 699 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 700 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 701 // CHECK-NEXT:    call void @llvm.masked.store.v4f32.p0(<4 x float> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]])
 702 // CHECK-NEXT:    ret void
 703 //
 704 void test_vst1q_p_f32(float32_t *base, float32x4_t value, mve_pred16_t p)
 705 {
 706 #ifdef POLYMORPHIC
 707     vst1q_p(base, value, p);
 708 #else /* POLYMORPHIC */
 709     vst1q_p_f32(base, value, p);
 710 #endif /* POLYMORPHIC */
 711 }
 712
 713 // CHECK-LABEL: @test_vst1q_p_s8(
 714 // CHECK-NEXT:  entry:
 715 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 716 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 717 // CHECK-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]])
 718 // CHECK-NEXT:    ret void
 719 //
 720 void test_vst1q_p_s8(int8_t *base, int8x16_t value, mve_pred16_t p)
 721 {
 722 #ifdef POLYMORPHIC
 723     vst1q_p(base, value, p);
 724 #else /* POLYMORPHIC */
 725     vst1q_p_s8(base, value, p);
 726 #endif /* POLYMORPHIC */
 727 }
 728
 729 // CHECK-LABEL: @test_vst1q_p_s16(
 730 // CHECK-NEXT:  entry:
 731 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 732 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 733 // CHECK-NEXT:    call void @llvm.masked.store.v8i16.p0(<8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]])
 734 // CHECK-NEXT:    ret void
 735 //
 736 void test_vst1q_p_s16(int16_t *base, int16x8_t value, mve_pred16_t p)
 737 {
 738 #ifdef POLYMORPHIC
 739     vst1q_p(base, value, p);
 740 #else /* POLYMORPHIC */
 741     vst1q_p_s16(base, value, p);
 742 #endif /* POLYMORPHIC */
 743 }
 744
 745 // CHECK-LABEL: @test_vst1q_p_s32(
 746 // CHECK-NEXT:  entry:
 747 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 748 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 749 // CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]])
 750 // CHECK-NEXT:    ret void
 751 //
 752 void test_vst1q_p_s32(int32_t *base, int32x4_t value, mve_pred16_t p)
 753 {
 754 #ifdef POLYMORPHIC
 755     vst1q_p(base, value, p);
 756 #else /* POLYMORPHIC */
 757     vst1q_p_s32(base, value, p);
 758 #endif /* POLYMORPHIC */
 759 }
 760
 761 // CHECK-LABEL: @test_vst1q_p_u8(
 762 // CHECK-NEXT:  entry:
 763 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 764 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 765 // CHECK-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]])
 766 // CHECK-NEXT:    ret void
 767 //
 768 void test_vst1q_p_u8(uint8_t *base, uint8x16_t value, mve_pred16_t p)
 769 {
 770 #ifdef POLYMORPHIC
 771     vst1q_p(base, value, p);
 772 #else /* POLYMORPHIC */
 773     vst1q_p_u8(base, value, p);
 774 #endif /* POLYMORPHIC */
 775 }
 776
 777 // CHECK-LABEL: @test_vst1q_p_u16(
 778 // CHECK-NEXT:  entry:
 779 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 780 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 781 // CHECK-NEXT:    call void @llvm.masked.store.v8i16.p0(<8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]])
 782 // CHECK-NEXT:    ret void
 783 //
 784 void test_vst1q_p_u16(uint16_t *base, uint16x8_t value, mve_pred16_t p)
 785 {
 786 #ifdef POLYMORPHIC
 787     vst1q_p(base, value, p);
 788 #else /* POLYMORPHIC */
 789     vst1q_p_u16(base, value, p);
 790 #endif /* POLYMORPHIC */
 791 }
 792
 793 // CHECK-LABEL: @test_vst1q_p_u32(
 794 // CHECK-NEXT:  entry:
 795 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 796 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 797 // CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]])
 798 // CHECK-NEXT:    ret void
 799 //
 800 void test_vst1q_p_u32(uint32_t *base, uint32x4_t value, mve_pred16_t p)
 801 {
 802 #ifdef POLYMORPHIC
 803     vst1q_p(base, value, p);
 804 #else /* POLYMORPHIC */
 805     vst1q_p_u32(base, value, p);
 806 #endif /* POLYMORPHIC */
 807 }
 808
 809 // CHECK-LABEL: @test_vstrbq_s8(
 810 // CHECK-NEXT:  entry:
 811 // CHECK-NEXT:    store <16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], align 1
 812 // CHECK-NEXT:    ret void
 813 //
 814 void test_vstrbq_s8(int8_t *base, int8x16_t value)
 815 {
 816 #ifdef POLYMORPHIC
 817     vstrbq(base, value);
 818 #else /* POLYMORPHIC */
 819     vstrbq_s8(base, value);
 820 #endif /* POLYMORPHIC */
 821 }
 822
 823 // CHECK-LABEL: @test_vstrbq_s16(
 824 // CHECK-NEXT:  entry:
 825 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <8 x i16> [[VALUE:%.*]] to <8 x i8>
 826 // CHECK-NEXT:    store <8 x i8> [[TMP0]], ptr [[BASE:%.*]], align 1
 827 // CHECK-NEXT:    ret void
 828 //
 829 void test_vstrbq_s16(int8_t *base, int16x8_t value)
 830 {
 831 #ifdef POLYMORPHIC
 832     vstrbq(base, value);
 833 #else /* POLYMORPHIC */
 834     vstrbq_s16(base, value);
 835 #endif /* POLYMORPHIC */
 836 }
 837
 838 // CHECK-LABEL: @test_vstrbq_s32(
 839 // CHECK-NEXT:  entry:
 840 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i8>
 841 // CHECK-NEXT:    store <4 x i8> [[TMP0]], ptr [[BASE:%.*]], align 1
 842 // CHECK-NEXT:    ret void
 843 //
 844 void test_vstrbq_s32(int8_t *base, int32x4_t value)
 845 {
 846 #ifdef POLYMORPHIC
 847     vstrbq(base, value);
 848 #else /* POLYMORPHIC */
 849     vstrbq_s32(base, value);
 850 #endif /* POLYMORPHIC */
 851 }
 852
 853 // CHECK-LABEL: @test_vstrbq_u8(
 854 // CHECK-NEXT:  entry:
 855 // CHECK-NEXT:    store <16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], align 1
 856 // CHECK-NEXT:    ret void
 857 //
 858 void test_vstrbq_u8(uint8_t *base, uint8x16_t value)
 859 {
 860 #ifdef POLYMORPHIC
 861     vstrbq(base, value);
 862 #else /* POLYMORPHIC */
 863     vstrbq_u8(base, value);
 864 #endif /* POLYMORPHIC */
 865 }
 866
 867 // CHECK-LABEL: @test_vstrbq_u16(
 868 // CHECK-NEXT:  entry:
 869 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <8 x i16> [[VALUE:%.*]] to <8 x i8>
 870 // CHECK-NEXT:    store <8 x i8> [[TMP0]], ptr [[BASE:%.*]], align 1
 871 // CHECK-NEXT:    ret void
 872 //
 873 void test_vstrbq_u16(uint8_t *base, uint16x8_t value)
 874 {
 875 #ifdef POLYMORPHIC
 876     vstrbq(base, value);
 877 #else /* POLYMORPHIC */
 878     vstrbq_u16(base, value);
 879 #endif /* POLYMORPHIC */
 880 }
 881
 882 // CHECK-LABEL: @test_vstrbq_u32(
 883 // CHECK-NEXT:  entry:
 884 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i8>
 885 // CHECK-NEXT:    store <4 x i8> [[TMP0]], ptr [[BASE:%.*]], align 1
 886 // CHECK-NEXT:    ret void
 887 //
 888 void test_vstrbq_u32(uint8_t *base, uint32x4_t value)
 889 {
 890 #ifdef POLYMORPHIC
 891     vstrbq(base, value);
 892 #else /* POLYMORPHIC */
 893     vstrbq_u32(base, value);
 894 #endif /* POLYMORPHIC */
 895 }
 896
 897 // CHECK-LABEL: @test_vstrbq_p_s8(
 898 // CHECK-NEXT:  entry:
 899 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 900 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 901 // CHECK-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]])
 902 // CHECK-NEXT:    ret void
 903 //
 904 void test_vstrbq_p_s8(int8_t *base, int8x16_t value, mve_pred16_t p)
 905 {
 906 #ifdef POLYMORPHIC
 907     vstrbq_p(base, value, p);
 908 #else /* POLYMORPHIC */
 909     vstrbq_p_s8(base, value, p);
 910 #endif /* POLYMORPHIC */
 911 }
 912
 913 // CHECK-LABEL: @test_vstrbq_p_s16(
 914 // CHECK-NEXT:  entry:
 915 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <8 x i16> [[VALUE:%.*]] to <8 x i8>
 916 // CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
 917 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP2]])
 918 // CHECK-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP0]], ptr [[BASE:%.*]], i32 1, <8 x i1> [[TMP3]])
 919 // CHECK-NEXT:    ret void
 920 //
 921 void test_vstrbq_p_s16(int8_t *base, int16x8_t value, mve_pred16_t p)
 922 {
 923 #ifdef POLYMORPHIC
 924     vstrbq_p(base, value, p);
 925 #else /* POLYMORPHIC */
 926     vstrbq_p_s16(base, value, p);
 927 #endif /* POLYMORPHIC */
 928 }
 929
 930 // CHECK-LABEL: @test_vstrbq_p_s32(
 931 // CHECK-NEXT:  entry:
 932 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i8>
 933 // CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
 934 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP2]])
 935 // CHECK-NEXT:    call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP0]], ptr [[BASE:%.*]], i32 1, <4 x i1> [[TMP3]])
 936 // CHECK-NEXT:    ret void
 937 //
 938 void test_vstrbq_p_s32(int8_t *base, int32x4_t value, mve_pred16_t p)
 939 {
 940 #ifdef POLYMORPHIC
 941     vstrbq_p(base, value, p);
 942 #else /* POLYMORPHIC */
 943     vstrbq_p_s32(base, value, p);
 944 #endif /* POLYMORPHIC */
 945 }
 946
 947 // CHECK-LABEL: @test_vstrbq_p_u8(
 948 // CHECK-NEXT:  entry:
 949 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 950 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 951 // CHECK-NEXT:    call void @llvm.masked.store.v16i8.p0(<16 x i8> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 1, <16 x i1> [[TMP2]])
 952 // CHECK-NEXT:    ret void
 953 //
 954 void test_vstrbq_p_u8(uint8_t *base, uint8x16_t value, mve_pred16_t p)
 955 {
 956 #ifdef POLYMORPHIC
 957     vstrbq_p(base, value, p);
 958 #else /* POLYMORPHIC */
 959     vstrbq_p_u8(base, value, p);
 960 #endif /* POLYMORPHIC */
 961 }
 962
 963 // CHECK-LABEL: @test_vstrbq_p_u16(
 964 // CHECK-NEXT:  entry:
 965 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <8 x i16> [[VALUE:%.*]] to <8 x i8>
 966 // CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
 967 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP2]])
 968 // CHECK-NEXT:    call void @llvm.masked.store.v8i8.p0(<8 x i8> [[TMP0]], ptr [[BASE:%.*]], i32 1, <8 x i1> [[TMP3]])
 969 // CHECK-NEXT:    ret void
 970 //
 971 void test_vstrbq_p_u16(uint8_t *base, uint16x8_t value, mve_pred16_t p)
 972 {
 973 #ifdef POLYMORPHIC
 974     vstrbq_p(base, value, p);
 975 #else /* POLYMORPHIC */
 976     vstrbq_p_u16(base, value, p);
 977 #endif /* POLYMORPHIC */
 978 }
 979
 980 // CHECK-LABEL: @test_vstrbq_p_u32(
 981 // CHECK-NEXT:  entry:
 982 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i8>
 983 // CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
 984 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP2]])
 985 // CHECK-NEXT:    call void @llvm.masked.store.v4i8.p0(<4 x i8> [[TMP0]], ptr [[BASE:%.*]], i32 1, <4 x i1> [[TMP3]])
 986 // CHECK-NEXT:    ret void
 987 //
 988 void test_vstrbq_p_u32(uint8_t *base, uint32x4_t value, mve_pred16_t p)
 989 {
 990 #ifdef POLYMORPHIC
 991     vstrbq_p(base, value, p);
 992 #else /* POLYMORPHIC */
 993     vstrbq_p_u32(base, value, p);
 994 #endif /* POLYMORPHIC */
 995 }
 996
 997 // CHECK-LABEL: @test_vstrhq_f16(
 998 // CHECK-NEXT:  entry:
 999 // CHECK-NEXT:    store <8 x half> [[VALUE:%.*]], ptr [[BASE:%.*]], align 2
1000 // CHECK-NEXT:    ret void
1001 //
1002 void test_vstrhq_f16(float16_t *base, float16x8_t value)
1003 {
1004 #ifdef POLYMORPHIC
1005     vstrhq(base, value);
1006 #else /* POLYMORPHIC */
1007     vstrhq_f16(base, value);
1008 #endif /* POLYMORPHIC */
1009 }
1010
1011 // CHECK-LABEL: @test_vstrhq_s16(
1012 // CHECK-NEXT:  entry:
1013 // CHECK-NEXT:    store <8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], align 2
1014 // CHECK-NEXT:    ret void
1015 //
1016 void test_vstrhq_s16(int16_t *base, int16x8_t value)
1017 {
1018 #ifdef POLYMORPHIC
1019     vstrhq(base, value);
1020 #else /* POLYMORPHIC */
1021     vstrhq_s16(base, value);
1022 #endif /* POLYMORPHIC */
1023 }
1024
1025 // CHECK-LABEL: @test_vstrhq_s32(
1026 // CHECK-NEXT:  entry:
1027 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i16>
1028 // CHECK-NEXT:    store <4 x i16> [[TMP0]], ptr [[BASE:%.*]], align 2
1029 // CHECK-NEXT:    ret void
1030 //
1031 void test_vstrhq_s32(int16_t *base, int32x4_t value)
1032 {
1033 #ifdef POLYMORPHIC
1034     vstrhq(base, value);
1035 #else /* POLYMORPHIC */
1036     vstrhq_s32(base, value);
1037 #endif /* POLYMORPHIC */
1038 }
1039
1040 // CHECK-LABEL: @test_vstrhq_u16(
1041 // CHECK-NEXT:  entry:
1042 // CHECK-NEXT:    store <8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], align 2
1043 // CHECK-NEXT:    ret void
1044 //
1045 void test_vstrhq_u16(uint16_t *base, uint16x8_t value)
1046 {
1047 #ifdef POLYMORPHIC
1048     vstrhq(base, value);
1049 #else /* POLYMORPHIC */
1050     vstrhq_u16(base, value);
1051 #endif /* POLYMORPHIC */
1052 }
1053
1054 // CHECK-LABEL: @test_vstrhq_u32(
1055 // CHECK-NEXT:  entry:
1056 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i16>
1057 // CHECK-NEXT:    store <4 x i16> [[TMP0]], ptr [[BASE:%.*]], align 2
1058 // CHECK-NEXT:    ret void
1059 //
1060 void test_vstrhq_u32(uint16_t *base, uint32x4_t value)
1061 {
1062 #ifdef POLYMORPHIC
1063     vstrhq(base, value);
1064 #else /* POLYMORPHIC */
1065     vstrhq_u32(base, value);
1066 #endif /* POLYMORPHIC */
1067 }
1068
1069 // CHECK-LABEL: @test_vstrhq_p_f16(
1070 // CHECK-NEXT:  entry:
1071 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1072 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
1073 // CHECK-NEXT:    call void @llvm.masked.store.v8f16.p0(<8 x half> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]])
1074 // CHECK-NEXT:    ret void
1075 //
1076 void test_vstrhq_p_f16(float16_t *base, float16x8_t value, mve_pred16_t p)
1077 {
1078 #ifdef POLYMORPHIC
1079     vstrhq_p(base, value, p);
1080 #else /* POLYMORPHIC */
1081     vstrhq_p_f16(base, value, p);
1082 #endif /* POLYMORPHIC */
1083 }
1084
1085 // CHECK-LABEL: @test_vstrhq_p_s16(
1086 // CHECK-NEXT:  entry:
1087 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1088 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
1089 // CHECK-NEXT:    call void @llvm.masked.store.v8i16.p0(<8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]])
1090 // CHECK-NEXT:    ret void
1091 //
1092 void test_vstrhq_p_s16(int16_t *base, int16x8_t value, mve_pred16_t p)
1093 {
1094 #ifdef POLYMORPHIC
1095     vstrhq_p(base, value, p);
1096 #else /* POLYMORPHIC */
1097     vstrhq_p_s16(base, value, p);
1098 #endif /* POLYMORPHIC */
1099 }
1100
1101 // CHECK-LABEL: @test_vstrhq_p_s32(
1102 // CHECK-NEXT:  entry:
1103 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i16>
1104 // CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
1105 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP2]])
1106 // CHECK-NEXT:    call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP0]], ptr [[BASE:%.*]], i32 2, <4 x i1> [[TMP3]])
1107 // CHECK-NEXT:    ret void
1108 //
1109 void test_vstrhq_p_s32(int16_t *base, int32x4_t value, mve_pred16_t p)
1110 {
1111 #ifdef POLYMORPHIC
1112     vstrhq_p(base, value, p);
1113 #else /* POLYMORPHIC */
1114     vstrhq_p_s32(base, value, p);
1115 #endif /* POLYMORPHIC */
1116 }
1117
1118 // CHECK-LABEL: @test_vstrhq_p_u16(
1119 // CHECK-NEXT:  entry:
1120 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1121 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
1122 // CHECK-NEXT:    call void @llvm.masked.store.v8i16.p0(<8 x i16> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 2, <8 x i1> [[TMP2]])
1123 // CHECK-NEXT:    ret void
1124 //
1125 void test_vstrhq_p_u16(uint16_t *base, uint16x8_t value, mve_pred16_t p)
1126 {
1127 #ifdef POLYMORPHIC
1128     vstrhq_p(base, value, p);
1129 #else /* POLYMORPHIC */
1130     vstrhq_p_u16(base, value, p);
1131 #endif /* POLYMORPHIC */
1132 }
1133
1134 // CHECK-LABEL: @test_vstrhq_p_u32(
1135 // CHECK-NEXT:  entry:
1136 // CHECK-NEXT:    [[TMP0:%.*]] = trunc <4 x i32> [[VALUE:%.*]] to <4 x i16>
1137 // CHECK-NEXT:    [[TMP2:%.*]] = zext i16 [[P:%.*]] to i32
1138 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP2]])
1139 // CHECK-NEXT:    call void @llvm.masked.store.v4i16.p0(<4 x i16> [[TMP0]], ptr [[BASE:%.*]], i32 2, <4 x i1> [[TMP3]])
1140 // CHECK-NEXT:    ret void
1141 //
1142 void test_vstrhq_p_u32(uint16_t *base, uint32x4_t value, mve_pred16_t p)
1143 {
1144 #ifdef POLYMORPHIC
1145     vstrhq_p(base, value, p);
1146 #else /* POLYMORPHIC */
1147     vstrhq_p_u32(base, value, p);
1148 #endif /* POLYMORPHIC */
1149 }
1150
1151 // CHECK-LABEL: @test_vstrwq_f32(
1152 // CHECK-NEXT:  entry:
1153 // CHECK-NEXT:    store <4 x float> [[VALUE:%.*]], ptr [[BASE:%.*]], align 4
1154 // CHECK-NEXT:    ret void
1155 //
1156 void test_vstrwq_f32(float32_t *base, float32x4_t value)
1157 {
1158 #ifdef POLYMORPHIC
1159     vstrwq(base, value);
1160 #else /* POLYMORPHIC */
1161     vstrwq_f32(base, value);
1162 #endif /* POLYMORPHIC */
1163 }
1164
1165 // CHECK-LABEL: @test_vstrwq_s32(
1166 // CHECK-NEXT:  entry:
1167 // CHECK-NEXT:    store <4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], align 4
1168 // CHECK-NEXT:    ret void
1169 //
1170 void test_vstrwq_s32(int32_t *base, int32x4_t value)
1171 {
1172 #ifdef POLYMORPHIC
1173     vstrwq(base, value);
1174 #else /* POLYMORPHIC */
1175     vstrwq_s32(base, value);
1176 #endif /* POLYMORPHIC */
1177 }
1178
1179 // CHECK-LABEL: @test_vstrwq_u32(
1180 // CHECK-NEXT:  entry:
1181 // CHECK-NEXT:    store <4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], align 4
1182 // CHECK-NEXT:    ret void
1183 //
1184 void test_vstrwq_u32(uint32_t *base, uint32x4_t value)
1185 {
1186 #ifdef POLYMORPHIC
1187     vstrwq(base, value);
1188 #else /* POLYMORPHIC */
1189     vstrwq_u32(base, value);
1190 #endif /* POLYMORPHIC */
1191 }
1192
1193 // CHECK-LABEL: @test_vstrwq_p_f32(
1194 // CHECK-NEXT:  entry:
1195 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1196 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
1197 // CHECK-NEXT:    call void @llvm.masked.store.v4f32.p0(<4 x float> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]])
1198 // CHECK-NEXT:    ret void
1199 //
1200 void test_vstrwq_p_f32(float32_t *base, float32x4_t value, mve_pred16_t p)
1201 {
1202 #ifdef POLYMORPHIC
1203     vstrwq_p(base, value, p);
1204 #else /* POLYMORPHIC */
1205     vstrwq_p_f32(base, value, p);
1206 #endif /* POLYMORPHIC */
1207 }
1208
1209 // CHECK-LABEL: @test_vstrwq_p_s32(
1210 // CHECK-NEXT:  entry:
1211 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1212 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
1213 // CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]])
1214 // CHECK-NEXT:    ret void
1215 //
1216 void test_vstrwq_p_s32(int32_t *base, int32x4_t value, mve_pred16_t p)
1217 {
1218 #ifdef POLYMORPHIC
1219     vstrwq_p(base, value, p);
1220 #else /* POLYMORPHIC */
1221     vstrwq_p_s32(base, value, p);
1222 #endif /* POLYMORPHIC */
1223 }
1224
1225 // CHECK-LABEL: @test_vstrwq_p_u32(
1226 // CHECK-NEXT:  entry:
1227 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
1228 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
1229 // CHECK-NEXT:    call void @llvm.masked.store.v4i32.p0(<4 x i32> [[VALUE:%.*]], ptr [[BASE:%.*]], i32 4, <4 x i1> [[TMP2]])
1230 // CHECK-NEXT:    ret void
1231 //
1232 void test_vstrwq_p_u32(uint32_t *base, uint32x4_t value, mve_pred16_t p)
1233 {
1234 #ifdef POLYMORPHIC
1235     vstrwq_p(base, value, p);
1236 #else /* POLYMORPHIC */
1237     vstrwq_p_u32(base, value, p);
1238 #endif /* POLYMORPHIC */
1239 }