clang/test/CodeGen/aarch64-sve-vls-shift-ops.c

   1 // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
   2 // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve \
   3 // RUN: -disable-O0-optnone -mvscale-min=4 -mvscale-max=4 \
   4 // RUN:  -emit-llvm -o - %s | opt -S -passes=sroa | FileCheck %s
   5
   6 // REQUIRES: aarch64-registered-target
   7
   8 #include <arm_sve.h>
   9
  10 #define N 512
  11
  12 typedef svint8_t fixed_int8_t __attribute__((arm_sve_vector_bits(N)));
  13 typedef svint16_t fixed_int16_t __attribute__((arm_sve_vector_bits(N)));
  14 typedef svint32_t fixed_int32_t __attribute__((arm_sve_vector_bits(N)));
  15 typedef svint64_t fixed_int64_t __attribute__((arm_sve_vector_bits(N)));
  16
  17 typedef svuint8_t fixed_uint8_t __attribute__((arm_sve_vector_bits(N)));
  18 typedef svuint16_t fixed_uint16_t __attribute__((arm_sve_vector_bits(N)));
  19 typedef svuint32_t fixed_uint32_t __attribute__((arm_sve_vector_bits(N)));
  20 typedef svuint64_t fixed_uint64_t __attribute__((arm_sve_vector_bits(N)));
  21
  22 typedef svfloat16_t fixed_float16_t __attribute__((arm_sve_vector_bits(N)));
  23 typedef svfloat32_t fixed_float32_t __attribute__((arm_sve_vector_bits(N)));
  24 typedef svfloat64_t fixed_float64_t __attribute__((arm_sve_vector_bits(N)));
  25
  26 typedef svbool_t fixed_bool_t __attribute__((arm_sve_vector_bits(N)));
  27
  28 // CHECK-LABEL: @lshift_i8(
  29 // CHECK-NEXT:  entry:
  30 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
  31 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
  32 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[B]]
  33 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
  34 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
  35 //
  36 fixed_int8_t lshift_i8(fixed_int8_t a, fixed_int8_t b) {
  37   return a << b;
  38 }
  39
  40 // CHECK-LABEL: @rshift_i8(
  41 // CHECK-NEXT:  entry:
  42 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
  43 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
  44 // CHECK-NEXT:    [[SHR:%.*]] = ashr <64 x i8> [[A]], [[B]]
  45 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
  46 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
  47 //
  48 fixed_int8_t rshift_i8(fixed_int8_t a, fixed_int8_t b) {
  49   return a >> b;
  50 }
  51
  52 // CHECK-LABEL: @lshift_u8(
  53 // CHECK-NEXT:  entry:
  54 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
  55 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
  56 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[B]]
  57 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
  58 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
  59 //
  60 fixed_uint8_t lshift_u8(fixed_uint8_t a, fixed_uint8_t b) {
  61   return a << b;
  62 }
  63
  64 // CHECK-LABEL: @rshift_u8(
  65 // CHECK-NEXT:  entry:
  66 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
  67 // CHECK-NEXT:    [[B:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[B_COERCE:%.*]], i64 0)
  68 // CHECK-NEXT:    [[SHR:%.*]] = lshr <64 x i8> [[A]], [[B]]
  69 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
  70 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
  71 //
  72 fixed_uint8_t rshift_u8(fixed_uint8_t a, fixed_uint8_t b) {
  73   return a >> b;
  74 }
  75
  76 // CHECK-LABEL: @lshift_i16(
  77 // CHECK-NEXT:  entry:
  78 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
  79 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
  80 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[B]]
  81 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
  82 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
  83 //
  84 fixed_int16_t lshift_i16(fixed_int16_t a, fixed_int16_t b) {
  85   return a << b;
  86 }
  87
  88 // CHECK-LABEL: @rshift_i16(
  89 // CHECK-NEXT:  entry:
  90 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
  91 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
  92 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i16> [[A]], [[B]]
  93 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
  94 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
  95 //
  96 fixed_int16_t rshift_i16(fixed_int16_t a, fixed_int16_t b) {
  97   return a >> b;
  98 }
  99
 100 // CHECK-LABEL: @lshift_u16(
 101 // CHECK-NEXT:  entry:
 102 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 103 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 104 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[B]]
 105 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
 106 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 107 //
 108 fixed_uint16_t lshift_u16(fixed_uint16_t a, fixed_uint16_t b) {
 109   return a << b;
 110 }
 111
 112 // CHECK-LABEL: @rshift_u16(
 113 // CHECK-NEXT:  entry:
 114 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 115 // CHECK-NEXT:    [[B:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[B_COERCE:%.*]], i64 0)
 116 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i16> [[A]], [[B]]
 117 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
 118 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 119 //
 120 fixed_uint16_t rshift_u16(fixed_uint16_t a, fixed_uint16_t b) {
 121   return a >> b;
 122 }
 123
 124 // CHECK-LABEL: @lshift_i32(
 125 // CHECK-NEXT:  entry:
 126 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 127 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 128 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[B]]
 129 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
 130 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 131 //
 132 fixed_int32_t lshift_i32(fixed_int32_t a, fixed_int32_t b) {
 133   return a << b;
 134 }
 135
 136 // CHECK-LABEL: @rshift_i32(
 137 // CHECK-NEXT:  entry:
 138 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 139 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 140 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i32> [[A]], [[B]]
 141 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
 142 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 143 //
 144 fixed_int32_t rshift_i32(fixed_int32_t a, fixed_int32_t b) {
 145   return a >> b;
 146 }
 147
 148 // CHECK-LABEL: @lshift_u32(
 149 // CHECK-NEXT:  entry:
 150 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 151 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 152 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[B]]
 153 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
 154 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 155 //
 156 fixed_uint32_t lshift_u32(fixed_uint32_t a, fixed_uint32_t b) {
 157   return a << b;
 158 }
 159
 160 // CHECK-LABEL: @rshift_u32(
 161 // CHECK-NEXT:  entry:
 162 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 163 // CHECK-NEXT:    [[B:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[B_COERCE:%.*]], i64 0)
 164 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i32> [[A]], [[B]]
 165 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
 166 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 167 //
 168 fixed_uint32_t rshift_u32(fixed_uint32_t a, fixed_uint32_t b) {
 169   return a >> b;
 170 }
 171
 172 // CHECK-LABEL: @lshift_i64(
 173 // CHECK-NEXT:  entry:
 174 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 175 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 176 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[B]]
 177 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
 178 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 179 //
 180 fixed_int64_t lshift_i64(fixed_int64_t a, fixed_int64_t b) {
 181   return a << b;
 182 }
 183
 184 // CHECK-LABEL: @rshift_i64(
 185 // CHECK-NEXT:  entry:
 186 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 187 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 188 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i64> [[A]], [[B]]
 189 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
 190 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 191 //
 192 fixed_int64_t rshift_i64(fixed_int64_t a, fixed_int64_t b) {
 193   return a >> b;
 194 }
 195
 196 // CHECK-LABEL: @lshift_u64(
 197 // CHECK-NEXT:  entry:
 198 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 199 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 200 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[B]]
 201 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
 202 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 203 //
 204 fixed_uint64_t lshift_u64(fixed_uint64_t a, fixed_uint64_t b) {
 205   return a << b;
 206 }
 207
 208 // CHECK-LABEL: @rshift_u64(
 209 // CHECK-NEXT:  entry:
 210 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 211 // CHECK-NEXT:    [[B:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[B_COERCE:%.*]], i64 0)
 212 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i64> [[A]], [[B]]
 213 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
 214 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 215 //
 216 fixed_uint64_t rshift_u64(fixed_uint64_t a, fixed_uint64_t b) {
 217   return a >> b;
 218 }
 219
 220 // CHECK-LABEL: @lshift_i8_rsplat(
 221 // CHECK-NEXT:  entry:
 222 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 223 // CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[B:%.*]] to i32
 224 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i32> poison, i32 [[CONV]], i64 0
 225 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 226 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 227 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[SH_PROM]]
 228 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
 229 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 230 //
 231 fixed_int8_t lshift_i8_rsplat(fixed_int8_t a, int8_t b) {
 232   return a << b;
 233 }
 234
 235 // CHECK-LABEL: @lshift_i8_lsplat(
 236 // CHECK-NEXT:  entry:
 237 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 238 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 239 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 240 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[SPLAT_SPLAT]], [[A]]
 241 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
 242 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 243 //
 244 fixed_int8_t lshift_i8_lsplat(fixed_int8_t a, int8_t b) {
 245   return b << a;
 246 }
 247
 248 // CHECK-LABEL: @rshift_i8_rsplat(
 249 // CHECK-NEXT:  entry:
 250 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 251 // CHECK-NEXT:    [[CONV:%.*]] = sext i8 [[B:%.*]] to i32
 252 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i32> poison, i32 [[CONV]], i64 0
 253 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 254 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 255 // CHECK-NEXT:    [[SHR:%.*]] = ashr <64 x i8> [[A]], [[SH_PROM]]
 256 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
 257 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 258 //
 259 fixed_int8_t rshift_i8_rsplat(fixed_int8_t a, int8_t b) {
 260   return a >> b;
 261 }
 262
 263 // CHECK-LABEL: @rshift_i8_lsplat(
 264 // CHECK-NEXT:  entry:
 265 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 266 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 267 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 268 // CHECK-NEXT:    [[SHR:%.*]] = ashr <64 x i8> [[SPLAT_SPLAT]], [[A]]
 269 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
 270 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 271 //
 272 fixed_int8_t rshift_i8_lsplat(fixed_int8_t a, int8_t b) {
 273   return b >> a;
 274 }
 275
 276 // CHECK-LABEL: @lshift_u8_rsplat(
 277 // CHECK-NEXT:  entry:
 278 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 279 // CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[B:%.*]] to i32
 280 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i32> poison, i32 [[CONV]], i64 0
 281 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 282 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 283 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[A]], [[SH_PROM]]
 284 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
 285 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 286 //
 287 fixed_uint8_t lshift_u8_rsplat(fixed_uint8_t a, uint8_t b) {
 288   return a << b;
 289 }
 290
 291 // CHECK-LABEL: @lshift_u8_lsplat(
 292 // CHECK-NEXT:  entry:
 293 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 294 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 295 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 296 // CHECK-NEXT:    [[SHL:%.*]] = shl <64 x i8> [[SPLAT_SPLAT]], [[A]]
 297 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHL]], i64 0)
 298 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 299 //
 300 fixed_uint8_t lshift_u8_lsplat(fixed_uint8_t a, uint8_t b) {
 301   return b << a;
 302 }
 303
 304 // CHECK-LABEL: @rshift_u8_rsplat(
 305 // CHECK-NEXT:  entry:
 306 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 307 // CHECK-NEXT:    [[CONV:%.*]] = zext i8 [[B:%.*]] to i32
 308 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i32> poison, i32 [[CONV]], i64 0
 309 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i32> [[SPLAT_SPLATINSERT]], <64 x i32> poison, <64 x i32> zeroinitializer
 310 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <64 x i32> [[SPLAT_SPLAT]] to <64 x i8>
 311 // CHECK-NEXT:    [[SHR:%.*]] = lshr <64 x i8> [[A]], [[SH_PROM]]
 312 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
 313 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 314 //
 315 fixed_uint8_t rshift_u8_rsplat(fixed_uint8_t a, uint8_t b) {
 316   return a >> b;
 317 }
 318
 319 // CHECK-LABEL: @rshift_u8_lsplat(
 320 // CHECK-NEXT:  entry:
 321 // CHECK-NEXT:    [[A:%.*]] = call <64 x i8> @llvm.vector.extract.v64i8.nxv16i8(<vscale x 16 x i8> [[A_COERCE:%.*]], i64 0)
 322 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <64 x i8> poison, i8 [[B:%.*]], i64 0
 323 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <64 x i8> [[SPLAT_SPLATINSERT]], <64 x i8> poison, <64 x i32> zeroinitializer
 324 // CHECK-NEXT:    [[SHR:%.*]] = lshr <64 x i8> [[SPLAT_SPLAT]], [[A]]
 325 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v64i8(<vscale x 16 x i8> undef, <64 x i8> [[SHR]], i64 0)
 326 // CHECK-NEXT:    ret <vscale x 16 x i8> [[CASTSCALABLESVE]]
 327 //
 328 fixed_uint8_t rshift_u8_lsplat(fixed_uint8_t a, uint8_t b) {
 329   return b >> a;
 330 }
 331
 332 // CHECK-LABEL: @lshift_i16_rsplat(
 333 // CHECK-NEXT:  entry:
 334 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 335 // CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[B:%.*]] to i32
 336 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
 337 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 338 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 339 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[SH_PROM]]
 340 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
 341 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 342 //
 343 fixed_int16_t lshift_i16_rsplat(fixed_int16_t a, int16_t b) {
 344   return a << b;
 345 }
 346
 347 // CHECK-LABEL: @lshift_i16_lsplat(
 348 // CHECK-NEXT:  entry:
 349 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 350 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 351 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 352 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[SPLAT_SPLAT]], [[A]]
 353 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
 354 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 355 //
 356 fixed_int16_t lshift_i16_lsplat(fixed_int16_t a, int16_t b) {
 357   return b << a;
 358 }
 359
 360 // CHECK-LABEL: @rshift_i16_rsplat(
 361 // CHECK-NEXT:  entry:
 362 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 363 // CHECK-NEXT:    [[CONV:%.*]] = sext i16 [[B:%.*]] to i32
 364 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
 365 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 366 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 367 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i16> [[A]], [[SH_PROM]]
 368 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
 369 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 370 //
 371 fixed_int16_t rshift_i16_rsplat(fixed_int16_t a, int16_t b) {
 372   return a >> b;
 373 }
 374
 375 // CHECK-LABEL: @rshift_i16_lsplat(
 376 // CHECK-NEXT:  entry:
 377 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 378 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 379 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 380 // CHECK-NEXT:    [[SHR:%.*]] = ashr <32 x i16> [[SPLAT_SPLAT]], [[A]]
 381 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
 382 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 383 //
 384 fixed_int16_t rshift_i16_lsplat(fixed_int16_t a, int16_t b) {
 385   return b >> a;
 386 }
 387
 388 // CHECK-LABEL: @lshift_u16_rsplat(
 389 // CHECK-NEXT:  entry:
 390 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 391 // CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[B:%.*]] to i32
 392 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
 393 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 394 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 395 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[A]], [[SH_PROM]]
 396 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
 397 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 398 //
 399 fixed_uint16_t lshift_u16_rsplat(fixed_uint16_t a, uint16_t b) {
 400   return a << b;
 401 }
 402
 403 // CHECK-LABEL: @lshift_u16_lsplat(
 404 // CHECK-NEXT:  entry:
 405 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 406 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 407 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 408 // CHECK-NEXT:    [[SHL:%.*]] = shl <32 x i16> [[SPLAT_SPLAT]], [[A]]
 409 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHL]], i64 0)
 410 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 411 //
 412 fixed_uint16_t lshift_u16_lsplat(fixed_uint16_t a, uint16_t b) {
 413   return b << a;
 414 }
 415
 416 // CHECK-LABEL: @rshift_u16_rsplat(
 417 // CHECK-NEXT:  entry:
 418 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 419 // CHECK-NEXT:    [[CONV:%.*]] = zext i16 [[B:%.*]] to i32
 420 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i32> poison, i32 [[CONV]], i64 0
 421 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i32> [[SPLAT_SPLATINSERT]], <32 x i32> poison, <32 x i32> zeroinitializer
 422 // CHECK-NEXT:    [[SH_PROM:%.*]] = trunc <32 x i32> [[SPLAT_SPLAT]] to <32 x i16>
 423 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i16> [[A]], [[SH_PROM]]
 424 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
 425 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 426 //
 427 fixed_uint16_t rshift_u16_rsplat(fixed_uint16_t a, uint16_t b) {
 428   return a >> b;
 429 }
 430
 431 // CHECK-LABEL: @rshift_u16_lsplat(
 432 // CHECK-NEXT:  entry:
 433 // CHECK-NEXT:    [[A:%.*]] = call <32 x i16> @llvm.vector.extract.v32i16.nxv8i16(<vscale x 8 x i16> [[A_COERCE:%.*]], i64 0)
 434 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <32 x i16> poison, i16 [[B:%.*]], i64 0
 435 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <32 x i16> [[SPLAT_SPLATINSERT]], <32 x i16> poison, <32 x i32> zeroinitializer
 436 // CHECK-NEXT:    [[SHR:%.*]] = lshr <32 x i16> [[SPLAT_SPLAT]], [[A]]
 437 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 8 x i16> @llvm.vector.insert.nxv8i16.v32i16(<vscale x 8 x i16> undef, <32 x i16> [[SHR]], i64 0)
 438 // CHECK-NEXT:    ret <vscale x 8 x i16> [[CASTSCALABLESVE]]
 439 //
 440 fixed_uint16_t rshift_u16_lsplat(fixed_uint16_t a, uint16_t b) {
 441   return b >> a;
 442 }
 443
 444 // CHECK-LABEL: @lshift_i32_rsplat(
 445 // CHECK-NEXT:  entry:
 446 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 447 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 448 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 449 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[SPLAT_SPLAT]]
 450 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
 451 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 452 //
 453 fixed_int32_t lshift_i32_rsplat(fixed_int32_t a, int32_t b) {
 454   return a << b;
 455 }
 456
 457 // CHECK-LABEL: @lshift_i32_lsplat(
 458 // CHECK-NEXT:  entry:
 459 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 460 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 461 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 462 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[SPLAT_SPLAT]], [[A]]
 463 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
 464 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 465 //
 466 fixed_int32_t lshift_i32_lsplat(fixed_int32_t a, int32_t b) {
 467   return b << a;
 468 }
 469
 470 // CHECK-LABEL: @rshift_i32_rsplat(
 471 // CHECK-NEXT:  entry:
 472 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 473 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 474 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 475 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i32> [[A]], [[SPLAT_SPLAT]]
 476 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
 477 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 478 //
 479 fixed_int32_t rshift_i32_rsplat(fixed_int32_t a, int32_t b) {
 480   return a >> b;
 481 }
 482
 483 // CHECK-LABEL: @rshift_i32_lsplat(
 484 // CHECK-NEXT:  entry:
 485 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 486 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 487 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 488 // CHECK-NEXT:    [[SHR:%.*]] = ashr <16 x i32> [[SPLAT_SPLAT]], [[A]]
 489 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
 490 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 491 //
 492 fixed_int32_t rshift_i32_lsplat(fixed_int32_t a, int32_t b) {
 493   return b >> a;
 494 }
 495
 496 // CHECK-LABEL: @lshift_u32_rsplat(
 497 // CHECK-NEXT:  entry:
 498 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 499 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 500 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 501 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[A]], [[SPLAT_SPLAT]]
 502 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
 503 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 504 //
 505 fixed_uint32_t lshift_u32_rsplat(fixed_uint32_t a, uint32_t b) {
 506   return a << b;
 507 }
 508
 509 // CHECK-LABEL: @lshift_u32_lsplat(
 510 // CHECK-NEXT:  entry:
 511 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 512 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 513 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 514 // CHECK-NEXT:    [[SHL:%.*]] = shl <16 x i32> [[SPLAT_SPLAT]], [[A]]
 515 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHL]], i64 0)
 516 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 517 //
 518 fixed_uint32_t lshift_u32_lsplat(fixed_uint32_t a, uint32_t b) {
 519   return b << a;
 520 }
 521
 522 // CHECK-LABEL: @rshift_u32_rsplat(
 523 // CHECK-NEXT:  entry:
 524 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 525 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 526 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 527 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i32> [[A]], [[SPLAT_SPLAT]]
 528 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
 529 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 530 //
 531 fixed_uint32_t rshift_u32_rsplat(fixed_uint32_t a, uint32_t b) {
 532   return a >> b;
 533 }
 534
 535 // CHECK-LABEL: @rshift_u32_lsplat(
 536 // CHECK-NEXT:  entry:
 537 // CHECK-NEXT:    [[A:%.*]] = call <16 x i32> @llvm.vector.extract.v16i32.nxv4i32(<vscale x 4 x i32> [[A_COERCE:%.*]], i64 0)
 538 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <16 x i32> poison, i32 [[B:%.*]], i64 0
 539 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <16 x i32> [[SPLAT_SPLATINSERT]], <16 x i32> poison, <16 x i32> zeroinitializer
 540 // CHECK-NEXT:    [[SHR:%.*]] = lshr <16 x i32> [[SPLAT_SPLAT]], [[A]]
 541 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 4 x i32> @llvm.vector.insert.nxv4i32.v16i32(<vscale x 4 x i32> undef, <16 x i32> [[SHR]], i64 0)
 542 // CHECK-NEXT:    ret <vscale x 4 x i32> [[CASTSCALABLESVE]]
 543 //
 544 fixed_uint32_t rshift_u32_lsplat(fixed_uint32_t a, uint32_t b) {
 545   return b >> a;
 546 }
 547
 548 // CHECK-LABEL: @lshift_i64_rsplat(
 549 // CHECK-NEXT:  entry:
 550 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 551 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 552 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 553 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[SPLAT_SPLAT]]
 554 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
 555 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 556 //
 557 fixed_int64_t lshift_i64_rsplat(fixed_int64_t a, int64_t b) {
 558   return a << b;
 559 }
 560
 561 // CHECK-LABEL: @lshift_i64_lsplat(
 562 // CHECK-NEXT:  entry:
 563 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 564 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 565 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 566 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[SPLAT_SPLAT]], [[A]]
 567 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
 568 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 569 //
 570 fixed_int64_t lshift_i64_lsplat(fixed_int64_t a, int64_t b) {
 571   return b << a;
 572 }
 573
 574 // CHECK-LABEL: @rshift_i64_rsplat(
 575 // CHECK-NEXT:  entry:
 576 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 577 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 578 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 579 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i64> [[A]], [[SPLAT_SPLAT]]
 580 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
 581 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 582 //
 583 fixed_int64_t rshift_i64_rsplat(fixed_int64_t a, int64_t b) {
 584   return a >> b;
 585 }
 586
 587 // CHECK-LABEL: @rshift_i64_lsplat(
 588 // CHECK-NEXT:  entry:
 589 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 590 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 591 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 592 // CHECK-NEXT:    [[SHR:%.*]] = ashr <8 x i64> [[SPLAT_SPLAT]], [[A]]
 593 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
 594 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 595 //
 596 fixed_int64_t rshift_i64_lsplat(fixed_int64_t a, int64_t b) {
 597   return b >> a;
 598 }
 599
 600 // CHECK-LABEL: @lshift_u64_rsplat(
 601 // CHECK-NEXT:  entry:
 602 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 603 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 604 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 605 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[A]], [[SPLAT_SPLAT]]
 606 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
 607 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 608 //
 609 fixed_uint64_t lshift_u64_rsplat(fixed_uint64_t a, uint64_t b) {
 610   return a << b;
 611 }
 612
 613 // CHECK-LABEL: @lshift_u64_lsplat(
 614 // CHECK-NEXT:  entry:
 615 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 616 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 617 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 618 // CHECK-NEXT:    [[SHL:%.*]] = shl <8 x i64> [[SPLAT_SPLAT]], [[A]]
 619 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHL]], i64 0)
 620 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 621 //
 622 fixed_uint64_t lshift_u64_lsplat(fixed_uint64_t a, uint64_t b) {
 623   return b << a;
 624 }
 625
 626 // CHECK-LABEL: @rshift_u64_rsplat(
 627 // CHECK-NEXT:  entry:
 628 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 629 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 630 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 631 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i64> [[A]], [[SPLAT_SPLAT]]
 632 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
 633 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 634 //
 635 fixed_uint64_t rshift_u64_rsplat(fixed_uint64_t a, uint64_t b) {
 636   return a >> b;
 637 }
 638
 639 // CHECK-LABEL: @rshift_u64_lsplat(
 640 // CHECK-NEXT:  entry:
 641 // CHECK-NEXT:    [[A:%.*]] = call <8 x i64> @llvm.vector.extract.v8i64.nxv2i64(<vscale x 2 x i64> [[A_COERCE:%.*]], i64 0)
 642 // CHECK-NEXT:    [[SPLAT_SPLATINSERT:%.*]] = insertelement <8 x i64> poison, i64 [[B:%.*]], i64 0
 643 // CHECK-NEXT:    [[SPLAT_SPLAT:%.*]] = shufflevector <8 x i64> [[SPLAT_SPLATINSERT]], <8 x i64> poison, <8 x i32> zeroinitializer
 644 // CHECK-NEXT:    [[SHR:%.*]] = lshr <8 x i64> [[SPLAT_SPLAT]], [[A]]
 645 // CHECK-NEXT:    [[CASTSCALABLESVE:%.*]] = call <vscale x 2 x i64> @llvm.vector.insert.nxv2i64.v8i64(<vscale x 2 x i64> undef, <8 x i64> [[SHR]], i64 0)
 646 // CHECK-NEXT:    ret <vscale x 2 x i64> [[CASTSCALABLESVE]]
 647 //
 648 fixed_uint64_t rshift_u64_lsplat(fixed_uint64_t a, uint64_t b) {
 649   return b >> a;
 650 }