clang/test/CodeGen/arm-mve-intrinsics/ternary.c

   1 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
   2 // RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=sroa | FileCheck %s
   3 // RUN: %clang_cc1 -triple thumbv8.1m.main-none-none-eabi -target-feature +mve.fp -mfloat-abi hard -O0 -disable-O0-optnone -DPOLYMORPHIC -emit-llvm -o - %s | opt -S -passes=sroa | FileCheck %s
   4
   5 // REQUIRES: aarch64-registered-target || arm-registered-target
   6
   7 #include <arm_mve.h>
   8
   9 // CHECK-LABEL: @test_vfmaq_f16(
  10 // CHECK-NEXT:  entry:
  11 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B:%.*]], <8 x half> [[C:%.*]], <8 x half> [[A:%.*]])
  12 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
  13 //
  14 float16x8_t test_vfmaq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
  15 #ifdef POLYMORPHIC
  16   return vfmaq(a, b, c);
  17 #else /* POLYMORPHIC */
  18   return vfmaq_f16(a, b, c);
  19 #endif /* POLYMORPHIC */
  20 }
  21
  22 // CHECK-LABEL: @test_vfmaq_f32(
  23 // CHECK-NEXT:  entry:
  24 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[B:%.*]], <4 x float> [[C:%.*]], <4 x float> [[A:%.*]])
  25 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
  26 //
  27 float32x4_t test_vfmaq_f32(float32x4_t a, float32x4_t b, float32x4_t c) {
  28 #ifdef POLYMORPHIC
  29   return vfmaq(a, b, c);
  30 #else /* POLYMORPHIC */
  31   return vfmaq_f32(a, b, c);
  32 #endif /* POLYMORPHIC */
  33 }
  34
  35 // CHECK-LABEL: @test_vfmaq_n_f16(
  36 // CHECK-NEXT:  entry:
  37 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[C:%.*]], i64 0
  38 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x half> [[DOTSPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
  39 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B:%.*]], <8 x half> [[DOTSPLAT]], <8 x half> [[A:%.*]])
  40 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
  41 //
  42 float16x8_t test_vfmaq_n_f16(float16x8_t a, float16x8_t b, float16_t c) {
  43 #ifdef POLYMORPHIC
  44   return vfmaq(a, b, c);
  45 #else /* POLYMORPHIC */
  46   return vfmaq_n_f16(a, b, c);
  47 #endif /* POLYMORPHIC */
  48 }
  49
  50 // CHECK-LABEL: @test_vfmaq_n_f32(
  51 // CHECK-NEXT:  entry:
  52 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[C:%.*]], i64 0
  53 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
  54 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[B:%.*]], <4 x float> [[DOTSPLAT]], <4 x float> [[A:%.*]])
  55 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
  56 //
  57 float32x4_t test_vfmaq_n_f32(float32x4_t a, float32x4_t b, float32_t c) {
  58 #ifdef POLYMORPHIC
  59   return vfmaq(a, b, c);
  60 #else /* POLYMORPHIC */
  61   return vfmaq_n_f32(a, b, c);
  62 #endif /* POLYMORPHIC */
  63 }
  64
  65 // CHECK-LABEL: @test_vfmasq_n_f16(
  66 // CHECK-NEXT:  entry:
  67 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[C:%.*]], i64 0
  68 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x half> [[DOTSPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
  69 // CHECK-NEXT:    [[TMP0:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x half> [[DOTSPLAT]])
  70 // CHECK-NEXT:    ret <8 x half> [[TMP0]]
  71 //
  72 float16x8_t test_vfmasq_n_f16(float16x8_t a, float16x8_t b, float16_t c) {
  73 #ifdef POLYMORPHIC
  74   return vfmasq(a, b, c);
  75 #else /* POLYMORPHIC */
  76   return vfmasq_n_f16(a, b, c);
  77 #endif /* POLYMORPHIC */
  78 }
  79
  80 // CHECK-LABEL: @test_vfmasq_n_f32(
  81 // CHECK-NEXT:  entry:
  82 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[C:%.*]], i64 0
  83 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
  84 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[DOTSPLAT]])
  85 // CHECK-NEXT:    ret <4 x float> [[TMP0]]
  86 //
  87 float32x4_t test_vfmasq_n_f32(float32x4_t a, float32x4_t b, float32_t c) {
  88 #ifdef POLYMORPHIC
  89   return vfmasq(a, b, c);
  90 #else /* POLYMORPHIC */
  91   return vfmasq_n_f32(a, b, c);
  92 #endif /* POLYMORPHIC */
  93 }
  94
  95 // CHECK-LABEL: @test_vfmsq_f16(
  96 // CHECK-NEXT:  entry:
  97 // CHECK-NEXT:    [[TMP0:%.*]] = fneg <8 x half> [[C:%.*]]
  98 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B:%.*]], <8 x half> [[TMP0]], <8 x half> [[A:%.*]])
  99 // CHECK-NEXT:    ret <8 x half> [[TMP1]]
 100 //
 101 float16x8_t test_vfmsq_f16(float16x8_t a, float16x8_t b, float16x8_t c) {
 102 #ifdef POLYMORPHIC
 103   return vfmsq(a, b, c);
 104 #else /* POLYMORPHIC */
 105   return vfmsq_f16(a, b, c);
 106 #endif /* POLYMORPHIC */
 107 }
 108
 109 // CHECK-LABEL: @test_vfmsq_f32(
 110 // CHECK-NEXT:  entry:
 111 // CHECK-NEXT:    [[TMP0:%.*]] = fneg <4 x float> [[C:%.*]]
 112 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[B:%.*]], <4 x float> [[TMP0]], <4 x float> [[A:%.*]])
 113 // CHECK-NEXT:    ret <4 x float> [[TMP1]]
 114 //
 115 float32x4_t test_vfmsq_f32(float32x4_t a, float32x4_t b, float32x4_t c) {
 116 #ifdef POLYMORPHIC
 117   return vfmsq(a, b, c);
 118 #else /* POLYMORPHIC */
 119   return vfmsq_f32(a, b, c);
 120 #endif /* POLYMORPHIC */
 121 }
 122
 123 // CHECK-LABEL: @test_vmlaq_n_s8(
 124 // CHECK-NEXT:  entry:
 125 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[C:%.*]], i64 0
 126 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i8> [[DOTSPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
 127 // CHECK-NEXT:    [[TMP0:%.*]] = mul <16 x i8> [[B:%.*]], [[DOTSPLAT]]
 128 // CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[TMP0]], [[A:%.*]]
 129 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 130 //
 131 int8x16_t test_vmlaq_n_s8(int8x16_t a, int8x16_t b, int8_t c) {
 132 #ifdef POLYMORPHIC
 133   return vmlaq(a, b, c);
 134 #else  /* POLYMORPHIC */
 135   return vmlaq_n_s8(a, b, c);
 136 #endif /* POLYMORPHIC */
 137 }
 138
 139 // CHECK-LABEL: @test_vmlaq_n_s16(
 140 // CHECK-NEXT:  entry:
 141 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[C:%.*]], i64 0
 142 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
 143 // CHECK-NEXT:    [[TMP0:%.*]] = mul <8 x i16> [[B:%.*]], [[DOTSPLAT]]
 144 // CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i16> [[TMP0]], [[A:%.*]]
 145 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 146 //
 147 int16x8_t test_vmlaq_n_s16(int16x8_t a, int16x8_t b, int16_t c) {
 148 #ifdef POLYMORPHIC
 149   return vmlaq(a, b, c);
 150 #else  /* POLYMORPHIC */
 151   return vmlaq_n_s16(a, b, c);
 152 #endif /* POLYMORPHIC */
 153 }
 154
 155 // CHECK-LABEL: @test_vmlaq_n_s32(
 156 // CHECK-NEXT:  entry:
 157 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
 158 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 159 // CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i32> [[B:%.*]], [[DOTSPLAT]]
 160 // CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[A:%.*]]
 161 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 162 //
 163 int32x4_t test_vmlaq_n_s32(int32x4_t a, int32x4_t b, int32_t c) {
 164 #ifdef POLYMORPHIC
 165   return vmlaq(a, b, c);
 166 #else  /* POLYMORPHIC */
 167   return vmlaq_n_s32(a, b, c);
 168 #endif /* POLYMORPHIC */
 169 }
 170
 171 // CHECK-LABEL: @test_vmlaq_n_u8(
 172 // CHECK-NEXT:  entry:
 173 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[C:%.*]], i64 0
 174 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i8> [[DOTSPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
 175 // CHECK-NEXT:    [[TMP0:%.*]] = mul <16 x i8> [[B:%.*]], [[DOTSPLAT]]
 176 // CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[TMP0]], [[A:%.*]]
 177 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 178 //
 179 uint8x16_t test_vmlaq_n_u8(uint8x16_t a, uint8x16_t b, uint8_t c) {
 180 #ifdef POLYMORPHIC
 181   return vmlaq(a, b, c);
 182 #else  /* POLYMORPHIC */
 183   return vmlaq_n_u8(a, b, c);
 184 #endif /* POLYMORPHIC */
 185 }
 186
 187 // CHECK-LABEL: @test_vmlaq_n_u16(
 188 // CHECK-NEXT:  entry:
 189 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[C:%.*]], i64 0
 190 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
 191 // CHECK-NEXT:    [[TMP0:%.*]] = mul <8 x i16> [[B:%.*]], [[DOTSPLAT]]
 192 // CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i16> [[TMP0]], [[A:%.*]]
 193 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 194 //
 195 uint16x8_t test_vmlaq_n_u16(uint16x8_t a, uint16x8_t b, uint16_t c) {
 196 #ifdef POLYMORPHIC
 197   return vmlaq(a, b, c);
 198 #else  /* POLYMORPHIC */
 199   return vmlaq_n_u16(a, b, c);
 200 #endif /* POLYMORPHIC */
 201 }
 202
 203 // CHECK-LABEL: @test_vmlaq_n_u32(
 204 // CHECK-NEXT:  entry:
 205 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
 206 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 207 // CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i32> [[B:%.*]], [[DOTSPLAT]]
 208 // CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[A:%.*]]
 209 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 210 //
 211 uint32x4_t test_vmlaq_n_u32(uint32x4_t a, uint32x4_t b, uint32_t c) {
 212 #ifdef POLYMORPHIC
 213   return vmlaq(a, b, c);
 214 #else  /* POLYMORPHIC */
 215   return vmlaq_n_u32(a, b, c);
 216 #endif /* POLYMORPHIC */
 217 }
 218
 219 // CHECK-LABEL: @test_vmlasq_n_s8(
 220 // CHECK-NEXT:  entry:
 221 // CHECK-NEXT:    [[TMP0:%.*]] = mul <16 x i8> [[A:%.*]], [[B:%.*]]
 222 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[C:%.*]], i64 0
 223 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i8> [[DOTSPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
 224 // CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[TMP0]], [[DOTSPLAT]]
 225 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 226 //
 227 int8x16_t test_vmlasq_n_s8(int8x16_t a, int8x16_t b, int8_t c) {
 228 #ifdef POLYMORPHIC
 229   return vmlasq(a, b, c);
 230 #else  /* POLYMORPHIC */
 231   return vmlasq_n_s8(a, b, c);
 232 #endif /* POLYMORPHIC */
 233 }
 234
 235 // CHECK-LABEL: @test_vmlasq_n_s16(
 236 // CHECK-NEXT:  entry:
 237 // CHECK-NEXT:    [[TMP0:%.*]] = mul <8 x i16> [[A:%.*]], [[B:%.*]]
 238 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[C:%.*]], i64 0
 239 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
 240 // CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i16> [[TMP0]], [[DOTSPLAT]]
 241 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 242 //
 243 int16x8_t test_vmlasq_n_s16(int16x8_t a, int16x8_t b, int16_t c) {
 244 #ifdef POLYMORPHIC
 245   return vmlasq(a, b, c);
 246 #else  /* POLYMORPHIC */
 247   return vmlasq_n_s16(a, b, c);
 248 #endif /* POLYMORPHIC */
 249 }
 250
 251 // CHECK-LABEL: @test_vmlasq_n_s32(
 252 // CHECK-NEXT:  entry:
 253 // CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
 254 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
 255 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 256 // CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[DOTSPLAT]]
 257 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 258 //
 259 int32x4_t test_vmlasq_n_s32(int32x4_t a, int32x4_t b, int32_t c) {
 260 #ifdef POLYMORPHIC
 261   return vmlasq(a, b, c);
 262 #else  /* POLYMORPHIC */
 263   return vmlasq_n_s32(a, b, c);
 264 #endif /* POLYMORPHIC */
 265 }
 266
 267 // CHECK-LABEL: @test_vmlasq_n_u8(
 268 // CHECK-NEXT:  entry:
 269 // CHECK-NEXT:    [[TMP0:%.*]] = mul <16 x i8> [[A:%.*]], [[B:%.*]]
 270 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <16 x i8> poison, i8 [[C:%.*]], i64 0
 271 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <16 x i8> [[DOTSPLATINSERT]], <16 x i8> poison, <16 x i32> zeroinitializer
 272 // CHECK-NEXT:    [[TMP1:%.*]] = add <16 x i8> [[TMP0]], [[DOTSPLAT]]
 273 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 274 //
 275 uint8x16_t test_vmlasq_n_u8(uint8x16_t a, uint8x16_t b, uint8_t c) {
 276 #ifdef POLYMORPHIC
 277   return vmlasq(a, b, c);
 278 #else  /* POLYMORPHIC */
 279   return vmlasq_n_u8(a, b, c);
 280 #endif /* POLYMORPHIC */
 281 }
 282
 283 // CHECK-LABEL: @test_vmlasq_n_u16(
 284 // CHECK-NEXT:  entry:
 285 // CHECK-NEXT:    [[TMP0:%.*]] = mul <8 x i16> [[A:%.*]], [[B:%.*]]
 286 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[C:%.*]], i64 0
 287 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x i16> [[DOTSPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer
 288 // CHECK-NEXT:    [[TMP1:%.*]] = add <8 x i16> [[TMP0]], [[DOTSPLAT]]
 289 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 290 //
 291 uint16x8_t test_vmlasq_n_u16(uint16x8_t a, uint16x8_t b, uint16_t c) {
 292 #ifdef POLYMORPHIC
 293   return vmlasq(a, b, c);
 294 #else  /* POLYMORPHIC */
 295   return vmlasq_n_u16(a, b, c);
 296 #endif /* POLYMORPHIC */
 297 }
 298
 299 // CHECK-LABEL: @test_vmlasq_n_u32(
 300 // CHECK-NEXT:  entry:
 301 // CHECK-NEXT:    [[TMP0:%.*]] = mul <4 x i32> [[A:%.*]], [[B:%.*]]
 302 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[C:%.*]], i64 0
 303 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x i32> [[DOTSPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer
 304 // CHECK-NEXT:    [[TMP1:%.*]] = add <4 x i32> [[TMP0]], [[DOTSPLAT]]
 305 // CHECK-NEXT:    ret <4 x i32> [[TMP1]]
 306 //
 307 uint32x4_t test_vmlasq_n_u32(uint32x4_t a, uint32x4_t b, uint32_t c) {
 308 #ifdef POLYMORPHIC
 309   return vmlasq(a, b, c);
 310 #else  /* POLYMORPHIC */
 311   return vmlasq_n_u32(a, b, c);
 312 #endif /* POLYMORPHIC */
 313 }
 314
 315 // CHECK-LABEL: @test_vqdmlahq_n_s8(
 316 // CHECK-NEXT:  entry:
 317 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 318 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.arm.mve.vqdmlah.v16i8(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]])
 319 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 320 //
 321 int8x16_t test_vqdmlahq_n_s8(int8x16_t a, int8x16_t b, int8_t c) {
 322 #ifdef POLYMORPHIC
 323   return vqdmlahq(a, b, c);
 324 #else  /* POLYMORPHIC */
 325   return vqdmlahq_n_s8(a, b, c);
 326 #endif /* POLYMORPHIC */
 327 }
 328
 329 // CHECK-LABEL: @test_vqdmlahq_n_s16(
 330 // CHECK-NEXT:  entry:
 331 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 332 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.arm.mve.vqdmlah.v8i16(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]])
 333 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 334 //
 335 int16x8_t test_vqdmlahq_n_s16(int16x8_t a, int16x8_t b, int16_t c) {
 336 #ifdef POLYMORPHIC
 337   return vqdmlahq(a, b, c);
 338 #else  /* POLYMORPHIC */
 339   return vqdmlahq_n_s16(a, b, c);
 340 #endif /* POLYMORPHIC */
 341 }
 342
 343 // CHECK-LABEL: @test_vqdmlahq_n_s32(
 344 // CHECK-NEXT:  entry:
 345 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vqdmlah.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]])
 346 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 347 //
 348 int32x4_t test_vqdmlahq_n_s32(int32x4_t a, int32x4_t b, int32_t c) {
 349 #ifdef POLYMORPHIC
 350   return vqdmlahq(a, b, c);
 351 #else  /* POLYMORPHIC */
 352   return vqdmlahq_n_s32(a, b, c);
 353 #endif /* POLYMORPHIC */
 354 }
 355
 356 // CHECK-LABEL: @test_vqdmlashq_n_s8(
 357 // CHECK-NEXT:  entry:
 358 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ADD:%.*]] to i32
 359 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.arm.mve.vqdmlash.v16i8(<16 x i8> [[M1:%.*]], <16 x i8> [[M2:%.*]], i32 [[TMP0]])
 360 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 361 //
 362 int8x16_t test_vqdmlashq_n_s8(int8x16_t m1, int8x16_t m2, int8_t add) {
 363 #ifdef POLYMORPHIC
 364   return vqdmlashq(m1, m2, add);
 365 #else  /* POLYMORPHIC */
 366   return vqdmlashq_n_s8(m1, m2, add);
 367 #endif /* POLYMORPHIC */
 368 }
 369
 370 // CHECK-LABEL: @test_vqdmlashq_n_s16(
 371 // CHECK-NEXT:  entry:
 372 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[ADD:%.*]] to i32
 373 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.arm.mve.vqdmlash.v8i16(<8 x i16> [[M1:%.*]], <8 x i16> [[M2:%.*]], i32 [[TMP0]])
 374 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 375 //
 376 int16x8_t test_vqdmlashq_n_s16(int16x8_t m1, int16x8_t m2, int16_t add) {
 377 #ifdef POLYMORPHIC
 378   return vqdmlashq(m1, m2, add);
 379 #else  /* POLYMORPHIC */
 380   return vqdmlashq_n_s16(m1, m2, add);
 381 #endif /* POLYMORPHIC */
 382 }
 383
 384 // CHECK-LABEL: @test_vqdmlashq_n_s32(
 385 // CHECK-NEXT:  entry:
 386 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vqdmlash.v4i32(<4 x i32> [[M1:%.*]], <4 x i32> [[M2:%.*]], i32 [[ADD:%.*]])
 387 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 388 //
 389 int32x4_t test_vqdmlashq_n_s32(int32x4_t m1, int32x4_t m2, int32_t add) {
 390 #ifdef POLYMORPHIC
 391   return vqdmlashq(m1, m2, add);
 392 #else  /* POLYMORPHIC */
 393   return vqdmlashq_n_s32(m1, m2, add);
 394 #endif /* POLYMORPHIC */
 395 }
 396
 397 // CHECK-LABEL: @test_vqrdmlahq_n_s8(
 398 // CHECK-NEXT:  entry:
 399 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 400 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.arm.mve.vqrdmlah.v16i8(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]])
 401 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 402 //
 403 int8x16_t test_vqrdmlahq_n_s8(int8x16_t a, int8x16_t b, int8_t c) {
 404 #ifdef POLYMORPHIC
 405   return vqrdmlahq(a, b, c);
 406 #else  /* POLYMORPHIC */
 407   return vqrdmlahq_n_s8(a, b, c);
 408 #endif /* POLYMORPHIC */
 409 }
 410
 411 // CHECK-LABEL: @test_vqrdmlahq_n_s16(
 412 // CHECK-NEXT:  entry:
 413 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 414 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.arm.mve.vqrdmlah.v8i16(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]])
 415 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 416 //
 417 int16x8_t test_vqrdmlahq_n_s16(int16x8_t a, int16x8_t b, int16_t c) {
 418 #ifdef POLYMORPHIC
 419   return vqrdmlahq(a, b, c);
 420 #else  /* POLYMORPHIC */
 421   return vqrdmlahq_n_s16(a, b, c);
 422 #endif /* POLYMORPHIC */
 423 }
 424
 425 // CHECK-LABEL: @test_vqrdmlahq_n_s32(
 426 // CHECK-NEXT:  entry:
 427 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vqrdmlah.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]])
 428 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 429 //
 430 int32x4_t test_vqrdmlahq_n_s32(int32x4_t a, int32x4_t b, int32_t c) {
 431 #ifdef POLYMORPHIC
 432   return vqrdmlahq(a, b, c);
 433 #else  /* POLYMORPHIC */
 434   return vqrdmlahq_n_s32(a, b, c);
 435 #endif /* POLYMORPHIC */
 436 }
 437
 438 // CHECK-LABEL: @test_vqrdmlashq_n_s8(
 439 // CHECK-NEXT:  entry:
 440 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 441 // CHECK-NEXT:    [[TMP1:%.*]] = call <16 x i8> @llvm.arm.mve.vqrdmlash.v16i8(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]])
 442 // CHECK-NEXT:    ret <16 x i8> [[TMP1]]
 443 //
 444 int8x16_t test_vqrdmlashq_n_s8(int8x16_t a, int8x16_t b, int8_t c) {
 445 #ifdef POLYMORPHIC
 446   return vqrdmlashq(a, b, c);
 447 #else  /* POLYMORPHIC */
 448   return vqrdmlashq_n_s8(a, b, c);
 449 #endif /* POLYMORPHIC */
 450 }
 451
 452 // CHECK-LABEL: @test_vqrdmlashq_n_s16(
 453 // CHECK-NEXT:  entry:
 454 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 455 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i16> @llvm.arm.mve.vqrdmlash.v8i16(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]])
 456 // CHECK-NEXT:    ret <8 x i16> [[TMP1]]
 457 //
 458 int16x8_t test_vqrdmlashq_n_s16(int16x8_t a, int16x8_t b, int16_t c) {
 459 #ifdef POLYMORPHIC
 460   return vqrdmlashq(a, b, c);
 461 #else  /* POLYMORPHIC */
 462   return vqrdmlashq_n_s16(a, b, c);
 463 #endif /* POLYMORPHIC */
 464 }
 465
 466 // CHECK-LABEL: @test_vqrdmlashq_n_s32(
 467 // CHECK-NEXT:  entry:
 468 // CHECK-NEXT:    [[TMP0:%.*]] = call <4 x i32> @llvm.arm.mve.vqrdmlash.v4i32(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]])
 469 // CHECK-NEXT:    ret <4 x i32> [[TMP0]]
 470 //
 471 int32x4_t test_vqrdmlashq_n_s32(int32x4_t a, int32x4_t b, int32_t c) {
 472 #ifdef POLYMORPHIC
 473   return vqrdmlashq(a, b, c);
 474 #else  /* POLYMORPHIC */
 475   return vqrdmlashq_n_s32(a, b, c);
 476 #endif /* POLYMORPHIC */
 477 }
 478
 479 // CHECK-LABEL: @test_vfmaq_m_f16(
 480 // CHECK-NEXT:  entry:
 481 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 482 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 483 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.fma.predicated.v8f16.v8i1(<8 x half> [[B:%.*]], <8 x half> [[C:%.*]], <8 x half> [[A:%.*]], <8 x i1> [[TMP1]])
 484 // CHECK-NEXT:    ret <8 x half> [[TMP2]]
 485 //
 486 float16x8_t test_vfmaq_m_f16(float16x8_t a, float16x8_t b, float16x8_t c, mve_pred16_t p) {
 487 #ifdef POLYMORPHIC
 488   return vfmaq_m(a, b, c, p);
 489 #else /* POLYMORPHIC */
 490   return vfmaq_m_f16(a, b, c, p);
 491 #endif /* POLYMORPHIC */
 492 }
 493
 494 // CHECK-LABEL: @test_vfmaq_m_f32(
 495 // CHECK-NEXT:  entry:
 496 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 497 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 498 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.fma.predicated.v4f32.v4i1(<4 x float> [[B:%.*]], <4 x float> [[C:%.*]], <4 x float> [[A:%.*]], <4 x i1> [[TMP1]])
 499 // CHECK-NEXT:    ret <4 x float> [[TMP2]]
 500 //
 501 float32x4_t test_vfmaq_m_f32(float32x4_t a, float32x4_t b, float32x4_t c, mve_pred16_t p) {
 502 #ifdef POLYMORPHIC
 503   return vfmaq_m(a, b, c, p);
 504 #else /* POLYMORPHIC */
 505   return vfmaq_m_f32(a, b, c, p);
 506 #endif /* POLYMORPHIC */
 507 }
 508
 509 // CHECK-LABEL: @test_vfmaq_m_n_f16(
 510 // CHECK-NEXT:  entry:
 511 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[C:%.*]], i64 0
 512 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x half> [[DOTSPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
 513 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 514 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 515 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.arm.mve.fma.predicated.v8f16.v8i1(<8 x half> [[B:%.*]], <8 x half> [[DOTSPLAT]], <8 x half> [[A:%.*]], <8 x i1> [[TMP1]])
 516 // CHECK-NEXT:    ret <8 x half> [[TMP2]]
 517 //
 518 float16x8_t test_vfmaq_m_n_f16(float16x8_t a, float16x8_t b, float16_t c, mve_pred16_t p) {
 519 #ifdef POLYMORPHIC
 520   return vfmaq_m(a, b, c, p);
 521 #else /* POLYMORPHIC */
 522   return vfmaq_m_n_f16(a, b, c, p);
 523 #endif /* POLYMORPHIC */
 524 }
 525
 526 // CHECK-LABEL: @test_vfmaq_m_n_f32(
 527 // CHECK-NEXT:  entry:
 528 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[C:%.*]], i64 0
 529 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
 530 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 531 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 532 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.arm.mve.fma.predicated.v4f32.v4i1(<4 x float> [[B:%.*]], <4 x float> [[DOTSPLAT]], <4 x float> [[A:%.*]], <4 x i1> [[TMP1]])
 533 // CHECK-NEXT:    ret <4 x float> [[TMP2]]
 534 //
 535 float32x4_t test_vfmaq_m_n_f32(float32x4_t a, float32x4_t b, float32_t c, mve_pred16_t p) {
 536 #ifdef POLYMORPHIC
 537   return vfmaq_m(a, b, c, p);
 538 #else /* POLYMORPHIC */
 539   return vfmaq_m_n_f32(a, b, c, p);
 540 #endif /* POLYMORPHIC */
 541 }
 542
 543 // CHECK-LABEL: @test_vfmasq_m_n_f16(
 544 // CHECK-NEXT:  entry:
 545 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 546 // CHECK-NEXT:    [[TMP1:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP0]])
 547 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <8 x half> poison, half [[C:%.*]], i64 0
 548 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <8 x half> [[DOTSPLATINSERT]], <8 x half> poison, <8 x i32> zeroinitializer
 549 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[A:%.*]], <8 x half> [[B:%.*]], <8 x half> [[DOTSPLAT]])
 550 // CHECK-NEXT:    [[TMP3:%.*]] = select <8 x i1> [[TMP1]], <8 x half> [[TMP2]], <8 x half> [[A]]
 551 // CHECK-NEXT:    ret <8 x half> [[TMP3]]
 552 //
 553 float16x8_t test_vfmasq_m_n_f16(float16x8_t a, float16x8_t b, float16_t c, mve_pred16_t p) {
 554 #ifdef POLYMORPHIC
 555   return vfmasq_m(a, b, c, p);
 556 #else /* POLYMORPHIC */
 557   return vfmasq_m_n_f16(a, b, c, p);
 558 #endif /* POLYMORPHIC */
 559 }
 560
 561 // CHECK-LABEL: @test_vfmasq_m_n_f32(
 562 // CHECK-NEXT:  entry:
 563 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 564 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 565 // CHECK-NEXT:    [[DOTSPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[C:%.*]], i64 0
 566 // CHECK-NEXT:    [[DOTSPLAT:%.*]] = shufflevector <4 x float> [[DOTSPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
 567 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x float> @llvm.fma.v4f32(<4 x float> [[A:%.*]], <4 x float> [[B:%.*]], <4 x float> [[DOTSPLAT]])
 568 // CHECK-NEXT:    [[TMP3:%.*]] = select <4 x i1> [[TMP1]], <4 x float> [[TMP2]], <4 x float> [[A]]
 569 // CHECK-NEXT:    ret <4 x float> [[TMP3]]
 570 //
 571 float32x4_t test_vfmasq_m_n_f32(float32x4_t a, float32x4_t b, float32_t c, mve_pred16_t p) {
 572 #ifdef POLYMORPHIC
 573   return vfmasq_m(a, b, c, p);
 574 #else /* POLYMORPHIC */
 575   return vfmasq_m_n_f32(a, b, c, p);
 576 #endif /* POLYMORPHIC */
 577 }
 578
 579 // CHECK-LABEL: @test_vfmsq_m_f16(
 580 // CHECK-NEXT:  entry:
 581 // CHECK-NEXT:    [[TMP0:%.*]] = fneg <8 x half> [[C:%.*]]
 582 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 583 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 584 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x half> @llvm.arm.mve.fma.predicated.v8f16.v8i1(<8 x half> [[B:%.*]], <8 x half> [[TMP0]], <8 x half> [[A:%.*]], <8 x i1> [[TMP2]])
 585 // CHECK-NEXT:    ret <8 x half> [[TMP3]]
 586 //
 587 float16x8_t test_vfmsq_m_f16(float16x8_t a, float16x8_t b, float16x8_t c, mve_pred16_t p) {
 588 #ifdef POLYMORPHIC
 589   return vfmsq_m(a, b, c, p);
 590 #else /* POLYMORPHIC */
 591   return vfmsq_m_f16(a, b, c, p);
 592 #endif /* POLYMORPHIC */
 593 }
 594
 595 // CHECK-LABEL: @test_vfmsq_m_f32(
 596 // CHECK-NEXT:  entry:
 597 // CHECK-NEXT:    [[TMP0:%.*]] = fneg <4 x float> [[C:%.*]]
 598 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 599 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP1]])
 600 // CHECK-NEXT:    [[TMP3:%.*]] = call <4 x float> @llvm.arm.mve.fma.predicated.v4f32.v4i1(<4 x float> [[B:%.*]], <4 x float> [[TMP0]], <4 x float> [[A:%.*]], <4 x i1> [[TMP2]])
 601 // CHECK-NEXT:    ret <4 x float> [[TMP3]]
 602 //
 603 float32x4_t test_vfmsq_m_f32(float32x4_t a, float32x4_t b, float32x4_t c, mve_pred16_t p) {
 604 #ifdef POLYMORPHIC
 605   return vfmsq_m(a, b, c, p);
 606 #else /* POLYMORPHIC */
 607   return vfmsq_m_f32(a, b, c, p);
 608 #endif /* POLYMORPHIC */
 609 }
 610
 611 // CHECK-LABEL: @test_vmlaq_m_n_s8(
 612 // CHECK-NEXT:  entry:
 613 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 614 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 615 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 616 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vmla.n.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 617 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 618 //
 619 int8x16_t test_vmlaq_m_n_s8(int8x16_t a, int8x16_t b, int8_t c, mve_pred16_t p) {
 620 #ifdef POLYMORPHIC
 621   return vmlaq_m(a, b, c, p);
 622 #else  /* POLYMORPHIC */
 623   return vmlaq_m_n_s8(a, b, c, p);
 624 #endif /* POLYMORPHIC */
 625 }
 626
 627 // CHECK-LABEL: @test_vmlaq_m_n_s16(
 628 // CHECK-NEXT:  entry:
 629 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 630 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 631 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 632 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vmla.n.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 633 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 634 //
 635 int16x8_t test_vmlaq_m_n_s16(int16x8_t a, int16x8_t b, int16_t c, mve_pred16_t p) {
 636 #ifdef POLYMORPHIC
 637   return vmlaq_m(a, b, c, p);
 638 #else  /* POLYMORPHIC */
 639   return vmlaq_m_n_s16(a, b, c, p);
 640 #endif /* POLYMORPHIC */
 641 }
 642
 643 // CHECK-LABEL: @test_vmlaq_m_n_s32(
 644 // CHECK-NEXT:  entry:
 645 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 646 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 647 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vmla.n.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 648 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 649 //
 650 int32x4_t test_vmlaq_m_n_s32(int32x4_t a, int32x4_t b, int32_t c, mve_pred16_t p) {
 651 #ifdef POLYMORPHIC
 652   return vmlaq_m(a, b, c, p);
 653 #else  /* POLYMORPHIC */
 654   return vmlaq_m_n_s32(a, b, c, p);
 655 #endif /* POLYMORPHIC */
 656 }
 657
 658 // CHECK-LABEL: @test_vmlaq_m_n_u8(
 659 // CHECK-NEXT:  entry:
 660 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 661 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 662 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 663 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vmla.n.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 664 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 665 //
 666 uint8x16_t test_vmlaq_m_n_u8(uint8x16_t a, uint8x16_t b, uint8_t c, mve_pred16_t p) {
 667 #ifdef POLYMORPHIC
 668   return vmlaq_m(a, b, c, p);
 669 #else  /* POLYMORPHIC */
 670   return vmlaq_m_n_u8(a, b, c, p);
 671 #endif /* POLYMORPHIC */
 672 }
 673
 674 // CHECK-LABEL: @test_vmlaq_m_n_u16(
 675 // CHECK-NEXT:  entry:
 676 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 677 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 678 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 679 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vmla.n.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 680 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 681 //
 682 uint16x8_t test_vmlaq_m_n_u16(uint16x8_t a, uint16x8_t b, uint16_t c, mve_pred16_t p) {
 683 #ifdef POLYMORPHIC
 684   return vmlaq_m(a, b, c, p);
 685 #else  /* POLYMORPHIC */
 686   return vmlaq_m_n_u16(a, b, c, p);
 687 #endif /* POLYMORPHIC */
 688 }
 689
 690 // CHECK-LABEL: @test_vmlaq_m_n_u32(
 691 // CHECK-NEXT:  entry:
 692 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 693 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 694 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vmla.n.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 695 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 696 //
 697 uint32x4_t test_vmlaq_m_n_u32(uint32x4_t a, uint32x4_t b, uint32_t c, mve_pred16_t p) {
 698 #ifdef POLYMORPHIC
 699   return vmlaq_m(a, b, c, p);
 700 #else  /* POLYMORPHIC */
 701   return vmlaq_m_n_u32(a, b, c, p);
 702 #endif /* POLYMORPHIC */
 703 }
 704
 705 // CHECK-LABEL: @test_vmlasq_m_n_s8(
 706 // CHECK-NEXT:  entry:
 707 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 708 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 709 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 710 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vmlas.n.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 711 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 712 //
 713 int8x16_t test_vmlasq_m_n_s8(int8x16_t a, int8x16_t b, int8_t c, mve_pred16_t p) {
 714 #ifdef POLYMORPHIC
 715   return vmlasq_m(a, b, c, p);
 716 #else  /* POLYMORPHIC */
 717   return vmlasq_m_n_s8(a, b, c, p);
 718 #endif /* POLYMORPHIC */
 719 }
 720
 721 // CHECK-LABEL: @test_vmlasq_m_n_s16(
 722 // CHECK-NEXT:  entry:
 723 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 724 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 725 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 726 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vmlas.n.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 727 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 728 //
 729 int16x8_t test_vmlasq_m_n_s16(int16x8_t a, int16x8_t b, int16_t c, mve_pred16_t p) {
 730 #ifdef POLYMORPHIC
 731   return vmlasq_m(a, b, c, p);
 732 #else  /* POLYMORPHIC */
 733   return vmlasq_m_n_s16(a, b, c, p);
 734 #endif /* POLYMORPHIC */
 735 }
 736
 737 // CHECK-LABEL: @test_vmlasq_m_n_s32(
 738 // CHECK-NEXT:  entry:
 739 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 740 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 741 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vmlas.n.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 742 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 743 //
 744 int32x4_t test_vmlasq_m_n_s32(int32x4_t a, int32x4_t b, int32_t c, mve_pred16_t p) {
 745 #ifdef POLYMORPHIC
 746   return vmlasq_m(a, b, c, p);
 747 #else  /* POLYMORPHIC */
 748   return vmlasq_m_n_s32(a, b, c, p);
 749 #endif /* POLYMORPHIC */
 750 }
 751
 752 // CHECK-LABEL: @test_vmlasq_m_n_u8(
 753 // CHECK-NEXT:  entry:
 754 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 755 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 756 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 757 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vmlas.n.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 758 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 759 //
 760 uint8x16_t test_vmlasq_m_n_u8(uint8x16_t a, uint8x16_t b, uint8_t c, mve_pred16_t p) {
 761 #ifdef POLYMORPHIC
 762   return vmlasq_m(a, b, c, p);
 763 #else  /* POLYMORPHIC */
 764   return vmlasq_m_n_u8(a, b, c, p);
 765 #endif /* POLYMORPHIC */
 766 }
 767
 768 // CHECK-LABEL: @test_vmlasq_m_n_u16(
 769 // CHECK-NEXT:  entry:
 770 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 771 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 772 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 773 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vmlas.n.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 774 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 775 //
 776 uint16x8_t test_vmlasq_m_n_u16(uint16x8_t a, uint16x8_t b, uint16_t c, mve_pred16_t p) {
 777 #ifdef POLYMORPHIC
 778   return vmlasq_m(a, b, c, p);
 779 #else  /* POLYMORPHIC */
 780   return vmlasq_m_n_u16(a, b, c, p);
 781 #endif /* POLYMORPHIC */
 782 }
 783
 784 // CHECK-LABEL: @test_vmlasq_m_n_u32(
 785 // CHECK-NEXT:  entry:
 786 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 787 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 788 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vmlas.n.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 789 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 790 //
 791 uint32x4_t test_vmlasq_m_n_u32(uint32x4_t a, uint32x4_t b, uint32_t c, mve_pred16_t p) {
 792 #ifdef POLYMORPHIC
 793   return vmlasq_m(a, b, c, p);
 794 #else  /* POLYMORPHIC */
 795   return vmlasq_m_n_u32(a, b, c, p);
 796 #endif /* POLYMORPHIC */
 797 }
 798
 799 // CHECK-LABEL: @test_vqdmlahq_m_n_s8(
 800 // CHECK-NEXT:  entry:
 801 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 802 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 803 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 804 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vqdmlah.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 805 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 806 //
 807 int8x16_t test_vqdmlahq_m_n_s8(int8x16_t a, int8x16_t b, int8_t c, mve_pred16_t p) {
 808 #ifdef POLYMORPHIC
 809   return vqdmlahq_m(a, b, c, p);
 810 #else  /* POLYMORPHIC */
 811   return vqdmlahq_m_n_s8(a, b, c, p);
 812 #endif /* POLYMORPHIC */
 813 }
 814
 815 // CHECK-LABEL: @test_vqdmlahq_m_n_s16(
 816 // CHECK-NEXT:  entry:
 817 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 818 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 819 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 820 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vqdmlah.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 821 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 822 //
 823 int16x8_t test_vqdmlahq_m_n_s16(int16x8_t a, int16x8_t b, int16_t c, mve_pred16_t p) {
 824 #ifdef POLYMORPHIC
 825   return vqdmlahq_m(a, b, c, p);
 826 #else  /* POLYMORPHIC */
 827   return vqdmlahq_m_n_s16(a, b, c, p);
 828 #endif /* POLYMORPHIC */
 829 }
 830
 831 // CHECK-LABEL: @test_vqdmlahq_m_n_s32(
 832 // CHECK-NEXT:  entry:
 833 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 834 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 835 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vqdmlah.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 836 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 837 //
 838 int32x4_t test_vqdmlahq_m_n_s32(int32x4_t a, int32x4_t b, int32_t c, mve_pred16_t p) {
 839 #ifdef POLYMORPHIC
 840   return vqdmlahq_m(a, b, c, p);
 841 #else  /* POLYMORPHIC */
 842   return vqdmlahq_m_n_s32(a, b, c, p);
 843 #endif /* POLYMORPHIC */
 844 }
 845
 846 // CHECK-LABEL: @test_vqdmlashq_m_n_s8(
 847 // CHECK-NEXT:  entry:
 848 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[ADD:%.*]] to i32
 849 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 850 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 851 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vqdmlash.predicated.v16i8.v16i1(<16 x i8> [[M1:%.*]], <16 x i8> [[M2:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 852 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 853 //
 854 int8x16_t test_vqdmlashq_m_n_s8(int8x16_t m1, int8x16_t m2, int8_t add, mve_pred16_t p) {
 855 #ifdef POLYMORPHIC
 856   return vqdmlashq_m(m1, m2, add, p);
 857 #else  /* POLYMORPHIC */
 858   return vqdmlashq_m_n_s8(m1, m2, add, p);
 859 #endif /* POLYMORPHIC */
 860 }
 861
 862 // CHECK-LABEL: @test_vqdmlashq_m_n_s16(
 863 // CHECK-NEXT:  entry:
 864 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[ADD:%.*]] to i32
 865 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 866 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 867 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vqdmlash.predicated.v8i16.v8i1(<8 x i16> [[M1:%.*]], <8 x i16> [[M2:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 868 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 869 //
 870 int16x8_t test_vqdmlashq_m_n_s16(int16x8_t m1, int16x8_t m2, int16_t add, mve_pred16_t p) {
 871 #ifdef POLYMORPHIC
 872   return vqdmlashq_m(m1, m2, add, p);
 873 #else  /* POLYMORPHIC */
 874   return vqdmlashq_m_n_s16(m1, m2, add, p);
 875 #endif /* POLYMORPHIC */
 876 }
 877
 878 // CHECK-LABEL: @test_vqdmlashq_m_n_s32(
 879 // CHECK-NEXT:  entry:
 880 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 881 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 882 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vqdmlash.predicated.v4i32.v4i1(<4 x i32> [[M1:%.*]], <4 x i32> [[M2:%.*]], i32 [[ADD:%.*]], <4 x i1> [[TMP1]])
 883 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 884 //
 885 int32x4_t test_vqdmlashq_m_n_s32(int32x4_t m1, int32x4_t m2, int32_t add, mve_pred16_t p) {
 886 #ifdef POLYMORPHIC
 887   return vqdmlashq_m(m1, m2, add, p);
 888 #else  /* POLYMORPHIC */
 889   return vqdmlashq_m_n_s32(m1, m2, add, p);
 890 #endif /* POLYMORPHIC */
 891 }
 892
 893 // CHECK-LABEL: @test_vqrdmlahq_m_n_s8(
 894 // CHECK-NEXT:  entry:
 895 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 896 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 897 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 898 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vqrdmlah.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 899 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 900 //
 901 int8x16_t test_vqrdmlahq_m_n_s8(int8x16_t a, int8x16_t b, int8_t c, mve_pred16_t p) {
 902 #ifdef POLYMORPHIC
 903   return vqrdmlahq_m(a, b, c, p);
 904 #else  /* POLYMORPHIC */
 905   return vqrdmlahq_m_n_s8(a, b, c, p);
 906 #endif /* POLYMORPHIC */
 907 }
 908
 909 // CHECK-LABEL: @test_vqrdmlahq_m_n_s16(
 910 // CHECK-NEXT:  entry:
 911 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 912 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 913 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 914 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vqrdmlah.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 915 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 916 //
 917 int16x8_t test_vqrdmlahq_m_n_s16(int16x8_t a, int16x8_t b, int16_t c, mve_pred16_t p) {
 918 #ifdef POLYMORPHIC
 919   return vqrdmlahq_m(a, b, c, p);
 920 #else  /* POLYMORPHIC */
 921   return vqrdmlahq_m_n_s16(a, b, c, p);
 922 #endif /* POLYMORPHIC */
 923 }
 924
 925 // CHECK-LABEL: @test_vqrdmlahq_m_n_s32(
 926 // CHECK-NEXT:  entry:
 927 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 928 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 929 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vqrdmlah.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 930 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 931 //
 932 int32x4_t test_vqrdmlahq_m_n_s32(int32x4_t a, int32x4_t b, int32_t c, mve_pred16_t p) {
 933 #ifdef POLYMORPHIC
 934   return vqrdmlahq_m(a, b, c, p);
 935 #else  /* POLYMORPHIC */
 936   return vqrdmlahq_m_n_s32(a, b, c, p);
 937 #endif /* POLYMORPHIC */
 938 }
 939
 940 // CHECK-LABEL: @test_vqrdmlashq_m_n_s8(
 941 // CHECK-NEXT:  entry:
 942 // CHECK-NEXT:    [[TMP0:%.*]] = zext i8 [[C:%.*]] to i32
 943 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 944 // CHECK-NEXT:    [[TMP2:%.*]] = call <16 x i1> @llvm.arm.mve.pred.i2v.v16i1(i32 [[TMP1]])
 945 // CHECK-NEXT:    [[TMP3:%.*]] = call <16 x i8> @llvm.arm.mve.vqrdmlash.predicated.v16i8.v16i1(<16 x i8> [[A:%.*]], <16 x i8> [[B:%.*]], i32 [[TMP0]], <16 x i1> [[TMP2]])
 946 // CHECK-NEXT:    ret <16 x i8> [[TMP3]]
 947 //
 948 int8x16_t test_vqrdmlashq_m_n_s8(int8x16_t a, int8x16_t b, int8_t c, mve_pred16_t p) {
 949 #ifdef POLYMORPHIC
 950   return vqrdmlashq_m(a, b, c, p);
 951 #else  /* POLYMORPHIC */
 952   return vqrdmlashq_m_n_s8(a, b, c, p);
 953 #endif /* POLYMORPHIC */
 954 }
 955
 956 // CHECK-LABEL: @test_vqrdmlashq_m_n_s16(
 957 // CHECK-NEXT:  entry:
 958 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[C:%.*]] to i32
 959 // CHECK-NEXT:    [[TMP1:%.*]] = zext i16 [[P:%.*]] to i32
 960 // CHECK-NEXT:    [[TMP2:%.*]] = call <8 x i1> @llvm.arm.mve.pred.i2v.v8i1(i32 [[TMP1]])
 961 // CHECK-NEXT:    [[TMP3:%.*]] = call <8 x i16> @llvm.arm.mve.vqrdmlash.predicated.v8i16.v8i1(<8 x i16> [[A:%.*]], <8 x i16> [[B:%.*]], i32 [[TMP0]], <8 x i1> [[TMP2]])
 962 // CHECK-NEXT:    ret <8 x i16> [[TMP3]]
 963 //
 964 int16x8_t test_vqrdmlashq_m_n_s16(int16x8_t a, int16x8_t b, int16_t c, mve_pred16_t p) {
 965 #ifdef POLYMORPHIC
 966   return vqrdmlashq_m(a, b, c, p);
 967 #else  /* POLYMORPHIC */
 968   return vqrdmlashq_m_n_s16(a, b, c, p);
 969 #endif /* POLYMORPHIC */
 970 }
 971
 972 // CHECK-LABEL: @test_vqrdmlashq_m_n_s32(
 973 // CHECK-NEXT:  entry:
 974 // CHECK-NEXT:    [[TMP0:%.*]] = zext i16 [[P:%.*]] to i32
 975 // CHECK-NEXT:    [[TMP1:%.*]] = call <4 x i1> @llvm.arm.mve.pred.i2v.v4i1(i32 [[TMP0]])
 976 // CHECK-NEXT:    [[TMP2:%.*]] = call <4 x i32> @llvm.arm.mve.vqrdmlash.predicated.v4i32.v4i1(<4 x i32> [[A:%.*]], <4 x i32> [[B:%.*]], i32 [[C:%.*]], <4 x i1> [[TMP1]])
 977 // CHECK-NEXT:    ret <4 x i32> [[TMP2]]
 978 //
 979 int32x4_t test_vqrdmlashq_m_n_s32(int32x4_t a, int32x4_t b, int32_t c, mve_pred16_t p) {
 980 #ifdef POLYMORPHIC
 981   return vqrdmlashq_m(a, b, c, p);
 982 #else  /* POLYMORPHIC */
 983   return vqrdmlashq_m_n_s32(a, b, c, p);
 984 #endif /* POLYMORPHIC */
 985 }