?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[ORC] Add std::tuple support to SimplePackedSerialization.
[llvm-project.git] / llvm / lib / Target / RISCV / RISCVISelLowering.cpp
blob6659d26a8f1d732d45c5562926fce49208cb218f
1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #include "RISCVISelLowering.h"
15 #include "MCTargetDesc/RISCVMatInt.h"
16 #include "RISCV.h"
17 #include "RISCVMachineFunctionInfo.h"
18 #include "RISCVRegisterInfo.h"
19 #include "RISCVSubtarget.h"
20 #include "RISCVTargetMachine.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/CodeGen/MachineFrameInfo.h"
24 #include "llvm/CodeGen/MachineFunction.h"
25 #include "llvm/CodeGen/MachineInstrBuilder.h"
26 #include "llvm/CodeGen/MachineRegisterInfo.h"
27 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
28 #include "llvm/CodeGen/ValueTypes.h"
29 #include "llvm/IR/DiagnosticInfo.h"
30 #include "llvm/IR/DiagnosticPrinter.h"
31 #include "llvm/IR/IntrinsicsRISCV.h"
32 #include "llvm/IR/IRBuilder.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/ErrorHandling.h"
35 #include "llvm/Support/KnownBits.h"
36 #include "llvm/Support/MathExtras.h"
37 #include "llvm/Support/raw_ostream.h"
38
39 using namespace llvm;
40
41 #define DEBUG_TYPE "riscv-lower"
42
43 STATISTIC(NumTailCalls, "Number of tail calls");
44
45 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
46 const RISCVSubtarget &STI)
47 : TargetLowering(TM), Subtarget(STI) {
48
49 if (Subtarget.isRV32E())
50 report_fatal_error("Codegen not yet implemented for RV32E");
51
52 RISCVABI::ABI ABI = Subtarget.getTargetABI();
53 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
54
55 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
56 !Subtarget.hasStdExtF()) {
57 errs() << "Hard-float 'f' ABI can't be used for a target that "
58 "doesn't support the F instruction set extension (ignoring "
59 "target-abi)\n";
60 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
61 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
62 !Subtarget.hasStdExtD()) {
63 errs() << "Hard-float 'd' ABI can't be used for a target that "
64 "doesn't support the D instruction set extension (ignoring "
65 "target-abi)\n";
66 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
67 }
68
69 switch (ABI) {
70 default:
71 report_fatal_error("Don't know how to lower this ABI");
72 case RISCVABI::ABI_ILP32:
73 case RISCVABI::ABI_ILP32F:
74 case RISCVABI::ABI_ILP32D:
75 case RISCVABI::ABI_LP64:
76 case RISCVABI::ABI_LP64F:
77 case RISCVABI::ABI_LP64D:
78 break;
79 }
80
81 MVT XLenVT = Subtarget.getXLenVT();
82
83 // Set up the register classes.
84 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
85
86 if (Subtarget.hasStdExtZfh())
87 addRegisterClass(MVT::f16, &RISCV::FPR16RegClass);
88 if (Subtarget.hasStdExtF())
89 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
90 if (Subtarget.hasStdExtD())
91 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
92
93 static const MVT::SimpleValueType BoolVecVTs[] = {
94 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
95 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
96 static const MVT::SimpleValueType IntVecVTs[] = {
97 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
98 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
99 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
100 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
101 MVT::nxv4i64, MVT::nxv8i64};
102 static const MVT::SimpleValueType F16VecVTs[] = {
103 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
104 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
105 static const MVT::SimpleValueType F32VecVTs[] = {
106 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
107 static const MVT::SimpleValueType F64VecVTs[] = {
108 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
109
110 if (Subtarget.hasStdExtV()) {
111 auto addRegClassForRVV = [this](MVT VT) {
112 unsigned Size = VT.getSizeInBits().getKnownMinValue();
113 assert(Size <= 512 && isPowerOf2_32(Size));
114 const TargetRegisterClass *RC;
115 if (Size <= 64)
116 RC = &RISCV::VRRegClass;
117 else if (Size == 128)
118 RC = &RISCV::VRM2RegClass;
119 else if (Size == 256)
120 RC = &RISCV::VRM4RegClass;
121 else
122 RC = &RISCV::VRM8RegClass;
123
124 addRegisterClass(VT, RC);
125 };
126
127 for (MVT VT : BoolVecVTs)
128 addRegClassForRVV(VT);
129 for (MVT VT : IntVecVTs)
130 addRegClassForRVV(VT);
131
132 if (Subtarget.hasStdExtZfh())
133 for (MVT VT : F16VecVTs)
134 addRegClassForRVV(VT);
135
136 if (Subtarget.hasStdExtF())
137 for (MVT VT : F32VecVTs)
138 addRegClassForRVV(VT);
139
140 if (Subtarget.hasStdExtD())
141 for (MVT VT : F64VecVTs)
142 addRegClassForRVV(VT);
143
144 if (Subtarget.useRVVForFixedLengthVectors()) {
145 auto addRegClassForFixedVectors = [this](MVT VT) {
146 MVT ContainerVT = getContainerForFixedLengthVector(VT);
147 unsigned RCID = getRegClassIDForVecVT(ContainerVT);
148 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
149 addRegisterClass(VT, TRI.getRegClass(RCID));
150 };
151 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
152 if (useRVVForFixedLengthVectorVT(VT))
153 addRegClassForFixedVectors(VT);
154
155 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
156 if (useRVVForFixedLengthVectorVT(VT))
157 addRegClassForFixedVectors(VT);
158 }
159 }
160
161 // Compute derived properties from the register classes.
162 computeRegisterProperties(STI.getRegisterInfo());
163
164 setStackPointerRegisterToSaveRestore(RISCV::X2);
165
166 for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
167 setLoadExtAction(N, XLenVT, MVT::i1, Promote);
168
169 // TODO: add all necessary setOperationAction calls.
170 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
171
172 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
173 setOperationAction(ISD::BR_CC, XLenVT, Expand);
174 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
175 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
176
177 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
178 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
179
180 setOperationAction(ISD::VASTART, MVT::Other, Custom);
181 setOperationAction(ISD::VAARG, MVT::Other, Expand);
182 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
183 setOperationAction(ISD::VAEND, MVT::Other, Expand);
184
185 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
186 if (!Subtarget.hasStdExtZbb()) {
187 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8, Expand);
188 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16, Expand);
189 }
190
191 if (Subtarget.is64Bit()) {
192 setOperationAction(ISD::ADD, MVT::i32, Custom);
193 setOperationAction(ISD::SUB, MVT::i32, Custom);
194 setOperationAction(ISD::SHL, MVT::i32, Custom);
195 setOperationAction(ISD::SRA, MVT::i32, Custom);
196 setOperationAction(ISD::SRL, MVT::i32, Custom);
197
198 setOperationAction(ISD::UADDO, MVT::i32, Custom);
199 setOperationAction(ISD::USUBO, MVT::i32, Custom);
200 setOperationAction(ISD::UADDSAT, MVT::i32, Custom);
201 setOperationAction(ISD::USUBSAT, MVT::i32, Custom);
202 }
203
204 if (!Subtarget.hasStdExtM()) {
205 setOperationAction(ISD::MUL, XLenVT, Expand);
206 setOperationAction(ISD::MULHS, XLenVT, Expand);
207 setOperationAction(ISD::MULHU, XLenVT, Expand);
208 setOperationAction(ISD::SDIV, XLenVT, Expand);
209 setOperationAction(ISD::UDIV, XLenVT, Expand);
210 setOperationAction(ISD::SREM, XLenVT, Expand);
211 setOperationAction(ISD::UREM, XLenVT, Expand);
212 } else {
213 if (Subtarget.is64Bit()) {
214 setOperationAction(ISD::MUL, MVT::i32, Custom);
215 setOperationAction(ISD::MUL, MVT::i128, Custom);
216
217 setOperationAction(ISD::SDIV, MVT::i8, Custom);
218 setOperationAction(ISD::UDIV, MVT::i8, Custom);
219 setOperationAction(ISD::UREM, MVT::i8, Custom);
220 setOperationAction(ISD::SDIV, MVT::i16, Custom);
221 setOperationAction(ISD::UDIV, MVT::i16, Custom);
222 setOperationAction(ISD::UREM, MVT::i16, Custom);
223 setOperationAction(ISD::SDIV, MVT::i32, Custom);
224 setOperationAction(ISD::UDIV, MVT::i32, Custom);
225 setOperationAction(ISD::UREM, MVT::i32, Custom);
226 } else {
227 setOperationAction(ISD::MUL, MVT::i64, Custom);
228 }
229 }
230
231 setOperationAction(ISD::SDIVREM, XLenVT, Expand);
232 setOperationAction(ISD::UDIVREM, XLenVT, Expand);
233 setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
234 setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
235
236 setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
237 setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
238 setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
239
240 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
241 if (Subtarget.is64Bit()) {
242 setOperationAction(ISD::ROTL, MVT::i32, Custom);
243 setOperationAction(ISD::ROTR, MVT::i32, Custom);
244 }
245 } else {
246 setOperationAction(ISD::ROTL, XLenVT, Expand);
247 setOperationAction(ISD::ROTR, XLenVT, Expand);
248 }
249
250 if (Subtarget.hasStdExtZbp()) {
251 // Custom lower bswap/bitreverse so we can convert them to GREVI to enable
252 // more combining.
253 setOperationAction(ISD::BITREVERSE, XLenVT, Custom);
254 setOperationAction(ISD::BSWAP, XLenVT, Custom);
255 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
256 // BSWAP i8 doesn't exist.
257 setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
258 setOperationAction(ISD::BSWAP, MVT::i16, Custom);
259
260 if (Subtarget.is64Bit()) {
261 setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
262 setOperationAction(ISD::BSWAP, MVT::i32, Custom);
263 }
264 } else {
265 // With Zbb we have an XLen rev8 instruction, but not GREVI. So we'll
266 // pattern match it directly in isel.
267 setOperationAction(ISD::BSWAP, XLenVT,
268 Subtarget.hasStdExtZbb() ? Legal : Expand);
269 }
270
271 if (Subtarget.hasStdExtZbb()) {
272 setOperationAction(ISD::SMIN, XLenVT, Legal);
273 setOperationAction(ISD::SMAX, XLenVT, Legal);
274 setOperationAction(ISD::UMIN, XLenVT, Legal);
275 setOperationAction(ISD::UMAX, XLenVT, Legal);
276
277 if (Subtarget.is64Bit()) {
278 setOperationAction(ISD::CTTZ, MVT::i32, Custom);
279 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32, Custom);
280 setOperationAction(ISD::CTLZ, MVT::i32, Custom);
281 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32, Custom);
282 }
283 } else {
284 setOperationAction(ISD::CTTZ, XLenVT, Expand);
285 setOperationAction(ISD::CTLZ, XLenVT, Expand);
286 setOperationAction(ISD::CTPOP, XLenVT, Expand);
287 }
288
289 if (Subtarget.hasStdExtZbt()) {
290 setOperationAction(ISD::FSHL, XLenVT, Custom);
291 setOperationAction(ISD::FSHR, XLenVT, Custom);
292 setOperationAction(ISD::SELECT, XLenVT, Legal);
293
294 if (Subtarget.is64Bit()) {
295 setOperationAction(ISD::FSHL, MVT::i32, Custom);
296 setOperationAction(ISD::FSHR, MVT::i32, Custom);
297 }
298 } else {
299 setOperationAction(ISD::SELECT, XLenVT, Custom);
300 }
301
302 ISD::CondCode FPCCToExpand[] = {
303 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
304 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
305 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
306
307 ISD::NodeType FPOpToExpand[] = {
308 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FP16_TO_FP,
309 ISD::FP_TO_FP16};
310
311 if (Subtarget.hasStdExtZfh())
312 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
313
314 if (Subtarget.hasStdExtZfh()) {
315 setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
316 setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
317 setOperationAction(ISD::LRINT, MVT::f16, Legal);
318 setOperationAction(ISD::LLRINT, MVT::f16, Legal);
319 setOperationAction(ISD::LROUND, MVT::f16, Legal);
320 setOperationAction(ISD::LLROUND, MVT::f16, Legal);
321 for (auto CC : FPCCToExpand)
322 setCondCodeAction(CC, MVT::f16, Expand);
323 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
324 setOperationAction(ISD::SELECT, MVT::f16, Custom);
325 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
326 for (auto Op : FPOpToExpand)
327 setOperationAction(Op, MVT::f16, Expand);
328 }
329
330 if (Subtarget.hasStdExtF()) {
331 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
332 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
333 setOperationAction(ISD::LRINT, MVT::f32, Legal);
334 setOperationAction(ISD::LLRINT, MVT::f32, Legal);
335 setOperationAction(ISD::LROUND, MVT::f32, Legal);
336 setOperationAction(ISD::LLROUND, MVT::f32, Legal);
337 for (auto CC : FPCCToExpand)
338 setCondCodeAction(CC, MVT::f32, Expand);
339 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
340 setOperationAction(ISD::SELECT, MVT::f32, Custom);
341 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
342 for (auto Op : FPOpToExpand)
343 setOperationAction(Op, MVT::f32, Expand);
344 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
345 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
346 }
347
348 if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
349 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
350
351 if (Subtarget.hasStdExtD()) {
352 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
353 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
354 setOperationAction(ISD::LRINT, MVT::f64, Legal);
355 setOperationAction(ISD::LLRINT, MVT::f64, Legal);
356 setOperationAction(ISD::LROUND, MVT::f64, Legal);
357 setOperationAction(ISD::LLROUND, MVT::f64, Legal);
358 for (auto CC : FPCCToExpand)
359 setCondCodeAction(CC, MVT::f64, Expand);
360 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
361 setOperationAction(ISD::SELECT, MVT::f64, Custom);
362 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
363 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
364 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
365 for (auto Op : FPOpToExpand)
366 setOperationAction(Op, MVT::f64, Expand);
367 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
368 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
369 }
370
371 if (Subtarget.is64Bit()) {
372 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
373 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
374 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
375 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
376 }
377
378 if (Subtarget.hasStdExtF()) {
379 setOperationAction(ISD::FP_TO_UINT_SAT, XLenVT, Custom);
380 setOperationAction(ISD::FP_TO_SINT_SAT, XLenVT, Custom);
381
382 setOperationAction(ISD::FLT_ROUNDS_, XLenVT, Custom);
383 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
384 }
385
386 setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
387 setOperationAction(ISD::BlockAddress, XLenVT, Custom);
388 setOperationAction(ISD::ConstantPool, XLenVT, Custom);
389 setOperationAction(ISD::JumpTable, XLenVT, Custom);
390
391 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
392
393 // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
394 // Unfortunately this can't be determined just from the ISA naming string.
395 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
396 Subtarget.is64Bit() ? Legal : Custom);
397
398 setOperationAction(ISD::TRAP, MVT::Other, Legal);
399 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
400 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
401 if (Subtarget.is64Bit())
402 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i32, Custom);
403
404 if (Subtarget.hasStdExtA()) {
405 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
406 setMinCmpXchgSizeInBits(32);
407 } else {
408 setMaxAtomicSizeInBitsSupported(0);
409 }
410
411 setBooleanContents(ZeroOrOneBooleanContent);
412
413 if (Subtarget.hasStdExtV()) {
414 setBooleanVectorContents(ZeroOrOneBooleanContent);
415
416 setOperationAction(ISD::VSCALE, XLenVT, Custom);
417
418 // RVV intrinsics may have illegal operands.
419 // We also need to custom legalize vmv.x.s.
420 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
421 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
422 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
423 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i16, Custom);
424 if (Subtarget.is64Bit()) {
425 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i32, Custom);
426 } else {
427 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
428 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
429 }
430
431 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
432
433 static unsigned IntegerVPOps[] = {
434 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL, ISD::VP_SDIV, ISD::VP_UDIV,
435 ISD::VP_SREM, ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR,
436 ISD::VP_ASHR, ISD::VP_LSHR, ISD::VP_SHL};
437
438 static unsigned FloatingPointVPOps[] = {ISD::VP_FADD, ISD::VP_FSUB,
439 ISD::VP_FMUL, ISD::VP_FDIV};
440
441 if (!Subtarget.is64Bit()) {
442 // We must custom-lower certain vXi64 operations on RV32 due to the vector
443 // element type being illegal.
444 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::i64, Custom);
445 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::i64, Custom);
446
447 setOperationAction(ISD::VECREDUCE_ADD, MVT::i64, Custom);
448 setOperationAction(ISD::VECREDUCE_AND, MVT::i64, Custom);
449 setOperationAction(ISD::VECREDUCE_OR, MVT::i64, Custom);
450 setOperationAction(ISD::VECREDUCE_XOR, MVT::i64, Custom);
451 setOperationAction(ISD::VECREDUCE_SMAX, MVT::i64, Custom);
452 setOperationAction(ISD::VECREDUCE_SMIN, MVT::i64, Custom);
453 setOperationAction(ISD::VECREDUCE_UMAX, MVT::i64, Custom);
454 setOperationAction(ISD::VECREDUCE_UMIN, MVT::i64, Custom);
455 }
456
457 for (MVT VT : BoolVecVTs) {
458 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
459
460 // Mask VTs are custom-expanded into a series of standard nodes
461 setOperationAction(ISD::TRUNCATE, VT, Custom);
462 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
463 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
464 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
465
466 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
467 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
468
469 setOperationAction(ISD::SELECT, VT, Custom);
470 setOperationAction(ISD::SELECT_CC, VT, Expand);
471 setOperationAction(ISD::VSELECT, VT, Expand);
472
473 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
474 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
475 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
476
477 // RVV has native int->float & float->int conversions where the
478 // element type sizes are within one power-of-two of each other. Any
479 // wider distances between type sizes have to be lowered as sequences
480 // which progressively narrow the gap in stages.
481 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
482 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
483 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
484 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
485
486 // Expand all extending loads to types larger than this, and truncating
487 // stores from types larger than this.
488 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
489 setTruncStoreAction(OtherVT, VT, Expand);
490 setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
491 setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
492 setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
493 }
494 }
495
496 for (MVT VT : IntVecVTs) {
497 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
498 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
499
500 setOperationAction(ISD::SMIN, VT, Legal);
501 setOperationAction(ISD::SMAX, VT, Legal);
502 setOperationAction(ISD::UMIN, VT, Legal);
503 setOperationAction(ISD::UMAX, VT, Legal);
504
505 setOperationAction(ISD::ROTL, VT, Expand);
506 setOperationAction(ISD::ROTR, VT, Expand);
507
508 // Custom-lower extensions and truncations from/to mask types.
509 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
510 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
511 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
512
513 // RVV has native int->float & float->int conversions where the
514 // element type sizes are within one power-of-two of each other. Any
515 // wider distances between type sizes have to be lowered as sequences
516 // which progressively narrow the gap in stages.
517 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
518 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
519 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
520 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
521
522 setOperationAction(ISD::SADDSAT, VT, Legal);
523 setOperationAction(ISD::UADDSAT, VT, Legal);
524 setOperationAction(ISD::SSUBSAT, VT, Legal);
525 setOperationAction(ISD::USUBSAT, VT, Legal);
526
527 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
528 // nodes which truncate by one power of two at a time.
529 setOperationAction(ISD::TRUNCATE, VT, Custom);
530
531 // Custom-lower insert/extract operations to simplify patterns.
532 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
533 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
534
535 // Custom-lower reduction operations to set up the corresponding custom
536 // nodes' operands.
537 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
538 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
539 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
540 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
541 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
542 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
543 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
544 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
545
546 for (unsigned VPOpc : IntegerVPOps)
547 setOperationAction(VPOpc, VT, Custom);
548
549 setOperationAction(ISD::LOAD, VT, Custom);
550 setOperationAction(ISD::STORE, VT, Custom);
551
552 setOperationAction(ISD::MLOAD, VT, Custom);
553 setOperationAction(ISD::MSTORE, VT, Custom);
554 setOperationAction(ISD::MGATHER, VT, Custom);
555 setOperationAction(ISD::MSCATTER, VT, Custom);
556
557 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
558 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
559 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
560
561 setOperationAction(ISD::SELECT, VT, Custom);
562 setOperationAction(ISD::SELECT_CC, VT, Expand);
563
564 setOperationAction(ISD::STEP_VECTOR, VT, Custom);
565 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
566
567 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
568 setTruncStoreAction(VT, OtherVT, Expand);
569 setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
570 setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
571 setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
572 }
573 }
574
575 // Expand various CCs to best match the RVV ISA, which natively supports UNE
576 // but no other unordered comparisons, and supports all ordered comparisons
577 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
578 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
579 // and we pattern-match those back to the "original", swapping operands once
580 // more. This way we catch both operations and both "vf" and "fv" forms with
581 // fewer patterns.
582 ISD::CondCode VFPCCToExpand[] = {
583 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
584 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
585 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
586 };
587
588 // Sets common operation actions on RVV floating-point vector types.
589 const auto SetCommonVFPActions = [&](MVT VT) {
590 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
591 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
592 // sizes are within one power-of-two of each other. Therefore conversions
593 // between vXf16 and vXf64 must be lowered as sequences which convert via
594 // vXf32.
595 setOperationAction(ISD::FP_ROUND, VT, Custom);
596 setOperationAction(ISD::FP_EXTEND, VT, Custom);
597 // Custom-lower insert/extract operations to simplify patterns.
598 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
599 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
600 // Expand various condition codes (explained above).
601 for (auto CC : VFPCCToExpand)
602 setCondCodeAction(CC, VT, Expand);
603
604 setOperationAction(ISD::FMINNUM, VT, Legal);
605 setOperationAction(ISD::FMAXNUM, VT, Legal);
606
607 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
608 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
609 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
610 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
611 setOperationAction(ISD::FCOPYSIGN, VT, Legal);
612
613 setOperationAction(ISD::LOAD, VT, Custom);
614 setOperationAction(ISD::STORE, VT, Custom);
615
616 setOperationAction(ISD::MLOAD, VT, Custom);
617 setOperationAction(ISD::MSTORE, VT, Custom);
618 setOperationAction(ISD::MGATHER, VT, Custom);
619 setOperationAction(ISD::MSCATTER, VT, Custom);
620
621 setOperationAction(ISD::SELECT, VT, Custom);
622 setOperationAction(ISD::SELECT_CC, VT, Expand);
623
624 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
625 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
626 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
627
628 setOperationAction(ISD::VECTOR_REVERSE, VT, Custom);
629
630 for (unsigned VPOpc : FloatingPointVPOps)
631 setOperationAction(VPOpc, VT, Custom);
632 };
633
634 // Sets common extload/truncstore actions on RVV floating-point vector
635 // types.
636 const auto SetCommonVFPExtLoadTruncStoreActions =
637 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
638 for (auto SmallVT : SmallerVTs) {
639 setTruncStoreAction(VT, SmallVT, Expand);
640 setLoadExtAction(ISD::EXTLOAD, VT, SmallVT, Expand);
641 }
642 };
643
644 if (Subtarget.hasStdExtZfh())
645 for (MVT VT : F16VecVTs)
646 SetCommonVFPActions(VT);
647
648 for (MVT VT : F32VecVTs) {
649 if (Subtarget.hasStdExtF())
650 SetCommonVFPActions(VT);
651 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
652 }
653
654 for (MVT VT : F64VecVTs) {
655 if (Subtarget.hasStdExtD())
656 SetCommonVFPActions(VT);
657 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
658 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
659 }
660
661 if (Subtarget.useRVVForFixedLengthVectors()) {
662 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
663 if (!useRVVForFixedLengthVectorVT(VT))
664 continue;
665
666 // By default everything must be expanded.
667 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
668 setOperationAction(Op, VT, Expand);
669 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
670 setTruncStoreAction(VT, OtherVT, Expand);
671 setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
672 setLoadExtAction(ISD::SEXTLOAD, OtherVT, VT, Expand);
673 setLoadExtAction(ISD::ZEXTLOAD, OtherVT, VT, Expand);
674 }
675
676 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
677 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
678 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
679
680 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
681 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
682
683 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
684 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
685
686 setOperationAction(ISD::LOAD, VT, Custom);
687 setOperationAction(ISD::STORE, VT, Custom);
688
689 setOperationAction(ISD::SETCC, VT, Custom);
690
691 setOperationAction(ISD::SELECT, VT, Custom);
692
693 setOperationAction(ISD::TRUNCATE, VT, Custom);
694
695 setOperationAction(ISD::BITCAST, VT, Custom);
696
697 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
698 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
699 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
700
701 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
702 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
703 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
704 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
705
706 // Operations below are different for between masks and other vectors.
707 if (VT.getVectorElementType() == MVT::i1) {
708 setOperationAction(ISD::AND, VT, Custom);
709 setOperationAction(ISD::OR, VT, Custom);
710 setOperationAction(ISD::XOR, VT, Custom);
711 continue;
712 }
713
714 // Use SPLAT_VECTOR to prevent type legalization from destroying the
715 // splats when type legalizing i64 scalar on RV32.
716 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
717 // improvements first.
718 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
719 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
720 setOperationAction(ISD::SPLAT_VECTOR_PARTS, VT, Custom);
721 }
722
723 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
724 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
725
726 setOperationAction(ISD::MLOAD, VT, Custom);
727 setOperationAction(ISD::MSTORE, VT, Custom);
728 setOperationAction(ISD::MGATHER, VT, Custom);
729 setOperationAction(ISD::MSCATTER, VT, Custom);
730 setOperationAction(ISD::ADD, VT, Custom);
731 setOperationAction(ISD::MUL, VT, Custom);
732 setOperationAction(ISD::SUB, VT, Custom);
733 setOperationAction(ISD::AND, VT, Custom);
734 setOperationAction(ISD::OR, VT, Custom);
735 setOperationAction(ISD::XOR, VT, Custom);
736 setOperationAction(ISD::SDIV, VT, Custom);
737 setOperationAction(ISD::SREM, VT, Custom);
738 setOperationAction(ISD::UDIV, VT, Custom);
739 setOperationAction(ISD::UREM, VT, Custom);
740 setOperationAction(ISD::SHL, VT, Custom);
741 setOperationAction(ISD::SRA, VT, Custom);
742 setOperationAction(ISD::SRL, VT, Custom);
743
744 setOperationAction(ISD::SMIN, VT, Custom);
745 setOperationAction(ISD::SMAX, VT, Custom);
746 setOperationAction(ISD::UMIN, VT, Custom);
747 setOperationAction(ISD::UMAX, VT, Custom);
748 setOperationAction(ISD::ABS, VT, Custom);
749
750 setOperationAction(ISD::MULHS, VT, Custom);
751 setOperationAction(ISD::MULHU, VT, Custom);
752
753 setOperationAction(ISD::SADDSAT, VT, Custom);
754 setOperationAction(ISD::UADDSAT, VT, Custom);
755 setOperationAction(ISD::SSUBSAT, VT, Custom);
756 setOperationAction(ISD::USUBSAT, VT, Custom);
757
758 setOperationAction(ISD::VSELECT, VT, Custom);
759 setOperationAction(ISD::SELECT_CC, VT, Expand);
760
761 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
762 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
763 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
764
765 // Custom-lower reduction operations to set up the corresponding custom
766 // nodes' operands.
767 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
768 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
769 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
770 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
771 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
772
773 for (unsigned VPOpc : IntegerVPOps)
774 setOperationAction(VPOpc, VT, Custom);
775 }
776
777 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
778 if (!useRVVForFixedLengthVectorVT(VT))
779 continue;
780
781 // By default everything must be expanded.
782 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
783 setOperationAction(Op, VT, Expand);
784 for (MVT OtherVT : MVT::fp_fixedlen_vector_valuetypes()) {
785 setLoadExtAction(ISD::EXTLOAD, OtherVT, VT, Expand);
786 setTruncStoreAction(VT, OtherVT, Expand);
787 }
788
789 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
790 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
791 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
792
793 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
794 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
795 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
796 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
797
798 setOperationAction(ISD::LOAD, VT, Custom);
799 setOperationAction(ISD::STORE, VT, Custom);
800 setOperationAction(ISD::MLOAD, VT, Custom);
801 setOperationAction(ISD::MSTORE, VT, Custom);
802 setOperationAction(ISD::MGATHER, VT, Custom);
803 setOperationAction(ISD::MSCATTER, VT, Custom);
804 setOperationAction(ISD::FADD, VT, Custom);
805 setOperationAction(ISD::FSUB, VT, Custom);
806 setOperationAction(ISD::FMUL, VT, Custom);
807 setOperationAction(ISD::FDIV, VT, Custom);
808 setOperationAction(ISD::FNEG, VT, Custom);
809 setOperationAction(ISD::FABS, VT, Custom);
810 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
811 setOperationAction(ISD::FSQRT, VT, Custom);
812 setOperationAction(ISD::FMA, VT, Custom);
813 setOperationAction(ISD::FMINNUM, VT, Custom);
814 setOperationAction(ISD::FMAXNUM, VT, Custom);
815
816 setOperationAction(ISD::FP_ROUND, VT, Custom);
817 setOperationAction(ISD::FP_EXTEND, VT, Custom);
818
819 for (auto CC : VFPCCToExpand)
820 setCondCodeAction(CC, VT, Expand);
821
822 setOperationAction(ISD::VSELECT, VT, Custom);
823 setOperationAction(ISD::SELECT, VT, Custom);
824 setOperationAction(ISD::SELECT_CC, VT, Expand);
825
826 setOperationAction(ISD::BITCAST, VT, Custom);
827
828 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
829 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
830 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
831 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
832
833 for (unsigned VPOpc : FloatingPointVPOps)
834 setOperationAction(VPOpc, VT, Custom);
835 }
836
837 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
838 setOperationAction(ISD::BITCAST, MVT::i8, Custom);
839 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
840 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
841 setOperationAction(ISD::BITCAST, MVT::i64, Custom);
842 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
843 setOperationAction(ISD::BITCAST, MVT::f32, Custom);
844 setOperationAction(ISD::BITCAST, MVT::f64, Custom);
845 }
846 }
847
848 // Function alignments.
849 const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
850 setMinFunctionAlignment(FunctionAlignment);
851 setPrefFunctionAlignment(FunctionAlignment);
852
853 setMinimumJumpTableEntries(5);
854
855 // Jumps are expensive, compared to logic
856 setJumpIsExpensive();
857
858 // We can use any register for comparisons
859 setHasMultipleConditionRegisters();
860
861 setTargetDAGCombine(ISD::ADD);
862 setTargetDAGCombine(ISD::SUB);
863 setTargetDAGCombine(ISD::AND);
864 setTargetDAGCombine(ISD::OR);
865 setTargetDAGCombine(ISD::XOR);
866 setTargetDAGCombine(ISD::ANY_EXTEND);
867 setTargetDAGCombine(ISD::ZERO_EXTEND);
868 if (Subtarget.hasStdExtV()) {
869 setTargetDAGCombine(ISD::FCOPYSIGN);
870 setTargetDAGCombine(ISD::MGATHER);
871 setTargetDAGCombine(ISD::MSCATTER);
872 setTargetDAGCombine(ISD::SRA);
873 setTargetDAGCombine(ISD::SRL);
874 setTargetDAGCombine(ISD::SHL);
875 }
876 }
877
878 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
879 LLVMContext &Context,
880 EVT VT) const {
881 if (!VT.isVector())
882 return getPointerTy(DL);
883 if (Subtarget.hasStdExtV() &&
884 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
885 return EVT::getVectorVT(Context, MVT::i1, VT.getVectorElementCount());
886 return VT.changeVectorElementTypeToInteger();
887 }
888
889 MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
890 return Subtarget.getXLenVT();
891 }
892
893 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
894 const CallInst &I,
895 MachineFunction &MF,
896 unsigned Intrinsic) const {
897 switch (Intrinsic) {
898 default:
899 return false;
900 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
901 case Intrinsic::riscv_masked_atomicrmw_add_i32:
902 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
903 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
904 case Intrinsic::riscv_masked_atomicrmw_max_i32:
905 case Intrinsic::riscv_masked_atomicrmw_min_i32:
906 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
907 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
908 case Intrinsic::riscv_masked_cmpxchg_i32: {
909 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
910 Info.opc = ISD::INTRINSIC_W_CHAIN;
911 Info.memVT = MVT::getVT(PtrTy->getElementType());
912 Info.ptrVal = I.getArgOperand(0);
913 Info.offset = 0;
914 Info.align = Align(4);
915 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
916 MachineMemOperand::MOVolatile;
917 return true;
918 }
919 }
920 }
921
922 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
923 const AddrMode &AM, Type *Ty,
924 unsigned AS,
925 Instruction *I) const {
926 // No global is ever allowed as a base.
927 if (AM.BaseGV)
928 return false;
929
930 // Require a 12-bit signed offset.
931 if (!isInt<12>(AM.BaseOffs))
932 return false;
933
934 switch (AM.Scale) {
935 case 0: // "r+i" or just "i", depending on HasBaseReg.
936 break;
937 case 1:
938 if (!AM.HasBaseReg) // allow "r+i".
939 break;
940 return false; // disallow "r+r" or "r+r+i".
941 default:
942 return false;
943 }
944
945 return true;
946 }
947
948 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
949 return isInt<12>(Imm);
950 }
951
952 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
953 return isInt<12>(Imm);
954 }
955
956 // On RV32, 64-bit integers are split into their high and low parts and held
957 // in two different registers, so the trunc is free since the low register can
958 // just be used.
959 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
960 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
961 return false;
962 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
963 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
964 return (SrcBits == 64 && DestBits == 32);
965 }
966
967 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
968 if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
969 !SrcVT.isInteger() || !DstVT.isInteger())
970 return false;
971 unsigned SrcBits = SrcVT.getSizeInBits();
972 unsigned DestBits = DstVT.getSizeInBits();
973 return (SrcBits == 64 && DestBits == 32);
974 }
975
976 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
977 // Zexts are free if they can be combined with a load.
978 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
979 EVT MemVT = LD->getMemoryVT();
980 if ((MemVT == MVT::i8 || MemVT == MVT::i16 ||
981 (Subtarget.is64Bit() && MemVT == MVT::i32)) &&
982 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
983 LD->getExtensionType() == ISD::ZEXTLOAD))
984 return true;
985 }
986
987 return TargetLowering::isZExtFree(Val, VT2);
988 }
989
990 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
991 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
992 }
993
994 bool RISCVTargetLowering::isCheapToSpeculateCttz() const {
995 return Subtarget.hasStdExtZbb();
996 }
997
998 bool RISCVTargetLowering::isCheapToSpeculateCtlz() const {
999 return Subtarget.hasStdExtZbb();
1000 }
1001
1002 bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
1003 bool ForCodeSize) const {
1004 if (VT == MVT::f16 && !Subtarget.hasStdExtZfh())
1005 return false;
1006 if (VT == MVT::f32 && !Subtarget.hasStdExtF())
1007 return false;
1008 if (VT == MVT::f64 && !Subtarget.hasStdExtD())
1009 return false;
1010 if (Imm.isNegZero())
1011 return false;
1012 return Imm.isZero();
1013 }
1014
1015 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
1016 return (VT == MVT::f16 && Subtarget.hasStdExtZfh()) ||
1017 (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
1018 (VT == MVT::f64 && Subtarget.hasStdExtD());
1019 }
1020
1021 MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1022 CallingConv::ID CC,
1023 EVT VT) const {
1024 // Use f32 to pass f16 if it is legal and Zfh is not enabled. We might still
1025 // end up using a GPR but that will be decided based on ABI.
1026 if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1027 return MVT::f32;
1028
1029 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1030 }
1031
1032 unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1033 CallingConv::ID CC,
1034 EVT VT) const {
1035 // Use f32 to pass f16 if it is legal and Zfh is not enabled. We might still
1036 // end up using a GPR but that will be decided based on ABI.
1037 if (VT == MVT::f16 && Subtarget.hasStdExtF() && !Subtarget.hasStdExtZfh())
1038 return 1;
1039
1040 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1041 }
1042
1043 // Changes the condition code and swaps operands if necessary, so the SetCC
1044 // operation matches one of the comparisons supported directly by branches
1045 // in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
1046 // with 1/-1.
1047 static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
1048 ISD::CondCode &CC, SelectionDAG &DAG) {
1049 // Convert X > -1 to X >= 0.
1050 if (CC == ISD::SETGT && isAllOnesConstant(RHS)) {
1051 RHS = DAG.getConstant(0, DL, RHS.getValueType());
1052 CC = ISD::SETGE;
1053 return;
1054 }
1055 // Convert X < 1 to 0 >= X.
1056 if (CC == ISD::SETLT && isOneConstant(RHS)) {
1057 RHS = LHS;
1058 LHS = DAG.getConstant(0, DL, RHS.getValueType());
1059 CC = ISD::SETGE;
1060 return;
1061 }
1062
1063 switch (CC) {
1064 default:
1065 break;
1066 case ISD::SETGT:
1067 case ISD::SETLE:
1068 case ISD::SETUGT:
1069 case ISD::SETULE:
1070 CC = ISD::getSetCCSwappedOperands(CC);
1071 std::swap(LHS, RHS);
1072 break;
1073 }
1074 }
1075
1076 RISCVII::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
1077 assert(VT.isScalableVector() && "Expecting a scalable vector type");
1078 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
1079 if (VT.getVectorElementType() == MVT::i1)
1080 KnownSize *= 8;
1081
1082 switch (KnownSize) {
1083 default:
1084 llvm_unreachable("Invalid LMUL.");
1085 case 8:
1086 return RISCVII::VLMUL::LMUL_F8;
1087 case 16:
1088 return RISCVII::VLMUL::LMUL_F4;
1089 case 32:
1090 return RISCVII::VLMUL::LMUL_F2;
1091 case 64:
1092 return RISCVII::VLMUL::LMUL_1;
1093 case 128:
1094 return RISCVII::VLMUL::LMUL_2;
1095 case 256:
1096 return RISCVII::VLMUL::LMUL_4;
1097 case 512:
1098 return RISCVII::VLMUL::LMUL_8;
1099 }
1100 }
1101
1102 unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVII::VLMUL LMul) {
1103 switch (LMul) {
1104 default:
1105 llvm_unreachable("Invalid LMUL.");
1106 case RISCVII::VLMUL::LMUL_F8:
1107 case RISCVII::VLMUL::LMUL_F4:
1108 case RISCVII::VLMUL::LMUL_F2:
1109 case RISCVII::VLMUL::LMUL_1:
1110 return RISCV::VRRegClassID;
1111 case RISCVII::VLMUL::LMUL_2:
1112 return RISCV::VRM2RegClassID;
1113 case RISCVII::VLMUL::LMUL_4:
1114 return RISCV::VRM4RegClassID;
1115 case RISCVII::VLMUL::LMUL_8:
1116 return RISCV::VRM8RegClassID;
1117 }
1118 }
1119
1120 unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
1121 RISCVII::VLMUL LMUL = getLMUL(VT);
1122 if (LMUL == RISCVII::VLMUL::LMUL_F8 ||
1123 LMUL == RISCVII::VLMUL::LMUL_F4 ||
1124 LMUL == RISCVII::VLMUL::LMUL_F2 ||
1125 LMUL == RISCVII::VLMUL::LMUL_1) {
1126 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
1127 "Unexpected subreg numbering");
1128 return RISCV::sub_vrm1_0 + Index;
1129 }
1130 if (LMUL == RISCVII::VLMUL::LMUL_2) {
1131 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
1132 "Unexpected subreg numbering");
1133 return RISCV::sub_vrm2_0 + Index;
1134 }
1135 if (LMUL == RISCVII::VLMUL::LMUL_4) {
1136 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
1137 "Unexpected subreg numbering");
1138 return RISCV::sub_vrm4_0 + Index;
1139 }
1140 llvm_unreachable("Invalid vector type.");
1141 }
1142
1143 unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
1144 if (VT.getVectorElementType() == MVT::i1)
1145 return RISCV::VRRegClassID;
1146 return getRegClassIDForLMUL(getLMUL(VT));
1147 }
1148
1149 // Attempt to decompose a subvector insert/extract between VecVT and
1150 // SubVecVT via subregister indices. Returns the subregister index that
1151 // can perform the subvector insert/extract with the given element index, as
1152 // well as the index corresponding to any leftover subvectors that must be
1153 // further inserted/extracted within the register class for SubVecVT.
1154 std::pair<unsigned, unsigned>
1155 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
1156 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
1157 const RISCVRegisterInfo *TRI) {
1158 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
1159 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
1160 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
1161 "Register classes not ordered");
1162 unsigned VecRegClassID = getRegClassIDForVecVT(VecVT);
1163 unsigned SubRegClassID = getRegClassIDForVecVT(SubVecVT);
1164 // Try to compose a subregister index that takes us from the incoming
1165 // LMUL>1 register class down to the outgoing one. At each step we half
1166 // the LMUL:
1167 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
1168 // Note that this is not guaranteed to find a subregister index, such as
1169 // when we are extracting from one VR type to another.
1170 unsigned SubRegIdx = RISCV::NoSubRegister;
1171 for (const unsigned RCID :
1172 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
1173 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
1174 VecVT = VecVT.getHalfNumVectorElementsVT();
1175 bool IsHi =
1176 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
1177 SubRegIdx = TRI->composeSubRegIndices(SubRegIdx,
1178 getSubregIndexByMVT(VecVT, IsHi));
1179 if (IsHi)
1180 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
1181 }
1182 return {SubRegIdx, InsertExtractIdx};
1183 }
1184
1185 // Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
1186 // stores for those types.
1187 bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
1188 return !Subtarget.useRVVForFixedLengthVectors() ||
1189 (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
1190 }
1191
1192 static bool useRVVForFixedLengthVectorVT(MVT VT,
1193 const RISCVSubtarget &Subtarget) {
1194 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
1195 if (!Subtarget.useRVVForFixedLengthVectors())
1196 return false;
1197
1198 // We only support a set of vector types with a consistent maximum fixed size
1199 // across all supported vector element types to avoid legalization issues.
1200 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
1201 // fixed-length vector type we support is 1024 bytes.
1202 if (VT.getFixedSizeInBits() > 1024 * 8)
1203 return false;
1204
1205 unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1206
1207 // Don't use RVV for vectors we cannot scalarize if required.
1208 switch (VT.getVectorElementType().SimpleTy) {
1209 // i1 is supported but has different rules.
1210 default:
1211 return false;
1212 case MVT::i1:
1213 // Masks can only use a single register.
1214 if (VT.getVectorNumElements() > MinVLen)
1215 return false;
1216 MinVLen /= 8;
1217 break;
1218 case MVT::i8:
1219 case MVT::i16:
1220 case MVT::i32:
1221 case MVT::i64:
1222 break;
1223 case MVT::f16:
1224 if (!Subtarget.hasStdExtZfh())
1225 return false;
1226 break;
1227 case MVT::f32:
1228 if (!Subtarget.hasStdExtF())
1229 return false;
1230 break;
1231 case MVT::f64:
1232 if (!Subtarget.hasStdExtD())
1233 return false;
1234 break;
1235 }
1236
1237 unsigned LMul = divideCeil(VT.getSizeInBits(), MinVLen);
1238 // Don't use RVV for types that don't fit.
1239 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
1240 return false;
1241
1242 // TODO: Perhaps an artificial restriction, but worth having whilst getting
1243 // the base fixed length RVV support in place.
1244 if (!VT.isPow2VectorType())
1245 return false;
1246
1247 return true;
1248 }
1249
1250 bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
1251 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
1252 }
1253
1254 // Return the largest legal scalable vector type that matches VT's element type.
1255 static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
1256 const RISCVSubtarget &Subtarget) {
1257 // This may be called before legal types are setup.
1258 assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
1259 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
1260 "Expected legal fixed length vector!");
1261
1262 unsigned MinVLen = Subtarget.getMinRVVVectorSizeInBits();
1263
1264 MVT EltVT = VT.getVectorElementType();
1265 switch (EltVT.SimpleTy) {
1266 default:
1267 llvm_unreachable("unexpected element type for RVV container");
1268 case MVT::i1:
1269 case MVT::i8:
1270 case MVT::i16:
1271 case MVT::i32:
1272 case MVT::i64:
1273 case MVT::f16:
1274 case MVT::f32:
1275 case MVT::f64: {
1276 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
1277 // narrower types, but we can't have a fractional LMUL with demoninator less
1278 // than 64/SEW.
1279 unsigned NumElts =
1280 divideCeil(VT.getVectorNumElements(), MinVLen / RISCV::RVVBitsPerBlock);
1281 return MVT::getScalableVectorVT(EltVT, NumElts);
1282 }
1283 }
1284 }
1285
1286 static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
1287 const RISCVSubtarget &Subtarget) {
1288 return getContainerForFixedLengthVector(DAG.getTargetLoweringInfo(), VT,
1289 Subtarget);
1290 }
1291
1292 MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
1293 return ::getContainerForFixedLengthVector(*this, VT, getSubtarget());
1294 }
1295
1296 // Grow V to consume an entire RVV register.
1297 static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1298 const RISCVSubtarget &Subtarget) {
1299 assert(VT.isScalableVector() &&
1300 "Expected to convert into a scalable vector!");
1301 assert(V.getValueType().isFixedLengthVector() &&
1302 "Expected a fixed length vector operand!");
1303 SDLoc DL(V);
1304 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1305 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V, Zero);
1306 }
1307
1308 // Shrink V so it's just big enough to maintain a VT's worth of data.
1309 static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
1310 const RISCVSubtarget &Subtarget) {
1311 assert(VT.isFixedLengthVector() &&
1312 "Expected to convert into a fixed length vector!");
1313 assert(V.getValueType().isScalableVector() &&
1314 "Expected a scalable vector operand!");
1315 SDLoc DL(V);
1316 SDValue Zero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
1317 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V, Zero);
1318 }
1319
1320 // Gets the two common "VL" operands: an all-ones mask and the vector length.
1321 // VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
1322 // the vector type that it is contained in.
1323 static std::pair<SDValue, SDValue>
1324 getDefaultVLOps(MVT VecVT, MVT ContainerVT, SDLoc DL, SelectionDAG &DAG,
1325 const RISCVSubtarget &Subtarget) {
1326 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
1327 MVT XLenVT = Subtarget.getXLenVT();
1328 SDValue VL = VecVT.isFixedLengthVector()
1329 ? DAG.getConstant(VecVT.getVectorNumElements(), DL, XLenVT)
1330 : DAG.getRegister(RISCV::X0, XLenVT);
1331 MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
1332 SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
1333 return {Mask, VL};
1334 }
1335
1336 // As above but assuming the given type is a scalable vector type.
1337 static std::pair<SDValue, SDValue>
1338 getDefaultScalableVLOps(MVT VecVT, SDLoc DL, SelectionDAG &DAG,
1339 const RISCVSubtarget &Subtarget) {
1340 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
1341 return getDefaultVLOps(VecVT, VecVT, DL, DAG, Subtarget);
1342 }
1343
1344 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
1345 // of either is (currently) supported. This can get us into an infinite loop
1346 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
1347 // as a ..., etc.
1348 // Until either (or both) of these can reliably lower any node, reporting that
1349 // we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
1350 // the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
1351 // which is not desirable.
1352 bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
1353 EVT VT, unsigned DefinedValues) const {
1354 return false;
1355 }
1356
1357 bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
1358 // Only splats are currently supported.
1359 if (ShuffleVectorSDNode::isSplatMask(M.data(), VT))
1360 return true;
1361
1362 return false;
1363 }
1364
1365 static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) {
1366 // RISCV FP-to-int conversions saturate to the destination register size, but
1367 // don't produce 0 for nan. We can use a conversion instruction and fix the
1368 // nan case with a compare and a select.
1369 SDValue Src = Op.getOperand(0);
1370
1371 EVT DstVT = Op.getValueType();
1372 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
1373
1374 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
1375 unsigned Opc;
1376 if (SatVT == DstVT)
1377 Opc = IsSigned ? RISCVISD::FCVT_X_RTZ : RISCVISD::FCVT_XU_RTZ;
1378 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
1379 Opc = IsSigned ? RISCVISD::FCVT_W_RTZ_RV64 : RISCVISD::FCVT_WU_RTZ_RV64;
1380 else
1381 return SDValue();
1382 // FIXME: Support other SatVTs by clamping before or after the conversion.
1383
1384 SDLoc DL(Op);
1385 SDValue FpToInt = DAG.getNode(Opc, DL, DstVT, Src);
1386
1387 SDValue ZeroInt = DAG.getConstant(0, DL, DstVT);
1388 return DAG.getSelectCC(DL, Src, Src, ZeroInt, FpToInt, ISD::CondCode::SETUO);
1389 }
1390
1391 static SDValue lowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG,
1392 const RISCVSubtarget &Subtarget) {
1393 MVT VT = Op.getSimpleValueType();
1394 assert(VT.isFixedLengthVector() && "Unexpected vector!");
1395
1396 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1397
1398 SDLoc DL(Op);
1399 SDValue Mask, VL;
1400 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1401
1402 unsigned Opc =
1403 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1404 SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, Op.getOperand(0), VL);
1405 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1406 }
1407
1408 struct VIDSequence {
1409 int64_t StepNumerator;
1410 unsigned StepDenominator;
1411 int64_t Addend;
1412 };
1413
1414 // Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
1415 // to the (non-zero) step S and start value X. This can be then lowered as the
1416 // RVV sequence (VID * S) + X, for example.
1417 // The step S is represented as an integer numerator divided by a positive
1418 // denominator. Note that the implementation currently only identifies
1419 // sequences in which either the numerator is +/- 1 or the denominator is 1. It
1420 // cannot detect 2/3, for example.
1421 // Note that this method will also match potentially unappealing index
1422 // sequences, like <i32 0, i32 50939494>, however it is left to the caller to
1423 // determine whether this is worth generating code for.
1424 static Optional<VIDSequence> isSimpleVIDSequence(SDValue Op) {
1425 unsigned NumElts = Op.getNumOperands();
1426 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
1427 if (!Op.getValueType().isInteger())
1428 return None;
1429
1430 Optional<unsigned> SeqStepDenom;
1431 Optional<int64_t> SeqStepNum, SeqAddend;
1432 Optional<std::pair<uint64_t, unsigned>> PrevElt;
1433 unsigned EltSizeInBits = Op.getValueType().getScalarSizeInBits();
1434 for (unsigned Idx = 0; Idx < NumElts; Idx++) {
1435 // Assume undef elements match the sequence; we just have to be careful
1436 // when interpolating across them.
1437 if (Op.getOperand(Idx).isUndef())
1438 continue;
1439 // The BUILD_VECTOR must be all constants.
1440 if (!isa<ConstantSDNode>(Op.getOperand(Idx)))
1441 return None;
1442
1443 uint64_t Val = Op.getConstantOperandVal(Idx) &
1444 maskTrailingOnes<uint64_t>(EltSizeInBits);
1445
1446 if (PrevElt) {
1447 // Calculate the step since the last non-undef element, and ensure
1448 // it's consistent across the entire sequence.
1449 unsigned IdxDiff = Idx - PrevElt->second;
1450 int64_t ValDiff = SignExtend64(Val - PrevElt->first, EltSizeInBits);
1451
1452 // A zero-value value difference means that we're somewhere in the middle
1453 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
1454 // step change before evaluating the sequence.
1455 if (ValDiff != 0) {
1456 int64_t Remainder = ValDiff % IdxDiff;
1457 // Normalize the step if it's greater than 1.
1458 if (Remainder != ValDiff) {
1459 // The difference must cleanly divide the element span.
1460 if (Remainder != 0)
1461 return None;
1462 ValDiff /= IdxDiff;
1463 IdxDiff = 1;
1464 }
1465
1466 if (!SeqStepNum)
1467 SeqStepNum = ValDiff;
1468 else if (ValDiff != SeqStepNum)
1469 return None;
1470
1471 if (!SeqStepDenom)
1472 SeqStepDenom = IdxDiff;
1473 else if (IdxDiff != *SeqStepDenom)
1474 return None;
1475 }
1476 }
1477
1478 // Record and/or check any addend.
1479 if (SeqStepNum && SeqStepDenom) {
1480 uint64_t ExpectedVal =
1481 (int64_t)(Idx * (uint64_t)*SeqStepNum) / *SeqStepDenom;
1482 int64_t Addend = SignExtend64(Val - ExpectedVal, EltSizeInBits);
1483 if (!SeqAddend)
1484 SeqAddend = Addend;
1485 else if (SeqAddend != Addend)
1486 return None;
1487 }
1488
1489 // Record this non-undef element for later.
1490 if (!PrevElt || PrevElt->first != Val)
1491 PrevElt = std::make_pair(Val, Idx);
1492 }
1493 // We need to have logged both a step and an addend for this to count as
1494 // a legal index sequence.
1495 if (!SeqStepNum || !SeqStepDenom || !SeqAddend)
1496 return None;
1497
1498 return VIDSequence{*SeqStepNum, *SeqStepDenom, *SeqAddend};
1499 }
1500
1501 static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
1502 const RISCVSubtarget &Subtarget) {
1503 MVT VT = Op.getSimpleValueType();
1504 assert(VT.isFixedLengthVector() && "Unexpected vector!");
1505
1506 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1507
1508 SDLoc DL(Op);
1509 SDValue Mask, VL;
1510 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1511
1512 MVT XLenVT = Subtarget.getXLenVT();
1513 unsigned NumElts = Op.getNumOperands();
1514
1515 if (VT.getVectorElementType() == MVT::i1) {
1516 if (ISD::isBuildVectorAllZeros(Op.getNode())) {
1517 SDValue VMClr = DAG.getNode(RISCVISD::VMCLR_VL, DL, ContainerVT, VL);
1518 return convertFromScalableVector(VT, VMClr, DAG, Subtarget);
1519 }
1520
1521 if (ISD::isBuildVectorAllOnes(Op.getNode())) {
1522 SDValue VMSet = DAG.getNode(RISCVISD::VMSET_VL, DL, ContainerVT, VL);
1523 return convertFromScalableVector(VT, VMSet, DAG, Subtarget);
1524 }
1525
1526 // Lower constant mask BUILD_VECTORs via an integer vector type, in
1527 // scalar integer chunks whose bit-width depends on the number of mask
1528 // bits and XLEN.
1529 // First, determine the most appropriate scalar integer type to use. This
1530 // is at most XLenVT, but may be shrunk to a smaller vector element type
1531 // according to the size of the final vector - use i8 chunks rather than
1532 // XLenVT if we're producing a v8i1. This results in more consistent
1533 // codegen across RV32 and RV64.
1534 unsigned NumViaIntegerBits =
1535 std::min(std::max(NumElts, 8u), Subtarget.getXLen());
1536 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
1537 // If we have to use more than one INSERT_VECTOR_ELT then this
1538 // optimization is likely to increase code size; avoid peforming it in
1539 // such a case. We can use a load from a constant pool in this case.
1540 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
1541 return SDValue();
1542 // Now we can create our integer vector type. Note that it may be larger
1543 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
1544 MVT IntegerViaVecVT =
1545 MVT::getVectorVT(MVT::getIntegerVT(NumViaIntegerBits),
1546 divideCeil(NumElts, NumViaIntegerBits));
1547
1548 uint64_t Bits = 0;
1549 unsigned BitPos = 0, IntegerEltIdx = 0;
1550 SDValue Vec = DAG.getUNDEF(IntegerViaVecVT);
1551
1552 for (unsigned I = 0; I < NumElts; I++, BitPos++) {
1553 // Once we accumulate enough bits to fill our scalar type, insert into
1554 // our vector and clear our accumulated data.
1555 if (I != 0 && I % NumViaIntegerBits == 0) {
1556 if (NumViaIntegerBits <= 32)
1557 Bits = SignExtend64(Bits, 32);
1558 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
1559 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec,
1560 Elt, DAG.getConstant(IntegerEltIdx, DL, XLenVT));
1561 Bits = 0;
1562 BitPos = 0;
1563 IntegerEltIdx++;
1564 }
1565 SDValue V = Op.getOperand(I);
1566 bool BitValue = !V.isUndef() && cast<ConstantSDNode>(V)->getZExtValue();
1567 Bits |= ((uint64_t)BitValue << BitPos);
1568 }
1569
1570 // Insert the (remaining) scalar value into position in our integer
1571 // vector type.
1572 if (NumViaIntegerBits <= 32)
1573 Bits = SignExtend64(Bits, 32);
1574 SDValue Elt = DAG.getConstant(Bits, DL, XLenVT);
1575 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, IntegerViaVecVT, Vec, Elt,
1576 DAG.getConstant(IntegerEltIdx, DL, XLenVT));
1577
1578 if (NumElts < NumViaIntegerBits) {
1579 // If we're producing a smaller vector than our minimum legal integer
1580 // type, bitcast to the equivalent (known-legal) mask type, and extract
1581 // our final mask.
1582 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
1583 Vec = DAG.getBitcast(MVT::v8i1, Vec);
1584 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Vec,
1585 DAG.getConstant(0, DL, XLenVT));
1586 } else {
1587 // Else we must have produced an integer type with the same size as the
1588 // mask type; bitcast for the final result.
1589 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
1590 Vec = DAG.getBitcast(VT, Vec);
1591 }
1592
1593 return Vec;
1594 }
1595
1596 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
1597 // vector type, we have a legal equivalently-sized i8 type, so we can use
1598 // that.
1599 MVT WideVecVT = VT.changeVectorElementType(MVT::i8);
1600 SDValue VecZero = DAG.getConstant(0, DL, WideVecVT);
1601
1602 SDValue WideVec;
1603 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
1604 // For a splat, perform a scalar truncate before creating the wider
1605 // vector.
1606 assert(Splat.getValueType() == XLenVT &&
1607 "Unexpected type for i1 splat value");
1608 Splat = DAG.getNode(ISD::AND, DL, XLenVT, Splat,
1609 DAG.getConstant(1, DL, XLenVT));
1610 WideVec = DAG.getSplatBuildVector(WideVecVT, DL, Splat);
1611 } else {
1612 SmallVector<SDValue, 8> Ops(Op->op_values());
1613 WideVec = DAG.getBuildVector(WideVecVT, DL, Ops);
1614 SDValue VecOne = DAG.getConstant(1, DL, WideVecVT);
1615 WideVec = DAG.getNode(ISD::AND, DL, WideVecVT, WideVec, VecOne);
1616 }
1617
1618 return DAG.getSetCC(DL, VT, WideVec, VecZero, ISD::SETNE);
1619 }
1620
1621 if (SDValue Splat = cast<BuildVectorSDNode>(Op)->getSplatValue()) {
1622 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
1623 : RISCVISD::VMV_V_X_VL;
1624 Splat = DAG.getNode(Opc, DL, ContainerVT, Splat, VL);
1625 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1626 }
1627
1628 // Try and match index sequences, which we can lower to the vid instruction
1629 // with optional modifications. An all-undef vector is matched by
1630 // getSplatValue, above.
1631 if (auto SimpleVID = isSimpleVIDSequence(Op)) {
1632 int64_t StepNumerator = SimpleVID->StepNumerator;
1633 unsigned StepDenominator = SimpleVID->StepDenominator;
1634 int64_t Addend = SimpleVID->Addend;
1635 // Only emit VIDs with suitably-small steps/addends. We use imm5 is a
1636 // threshold since it's the immediate value many RVV instructions accept.
1637 if (isInt<5>(StepNumerator) && isPowerOf2_32(StepDenominator) &&
1638 isInt<5>(Addend)) {
1639 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, ContainerVT, Mask, VL);
1640 // Convert right out of the scalable type so we can use standard ISD
1641 // nodes for the rest of the computation. If we used scalable types with
1642 // these, we'd lose the fixed-length vector info and generate worse
1643 // vsetvli code.
1644 VID = convertFromScalableVector(VT, VID, DAG, Subtarget);
1645 assert(StepNumerator != 0 && "Invalid step");
1646 bool Negate = false;
1647 if (StepNumerator != 1) {
1648 int64_t SplatStepVal = StepNumerator;
1649 unsigned Opcode = ISD::MUL;
1650 if (isPowerOf2_64(std::abs(StepNumerator))) {
1651 Negate = StepNumerator < 0;
1652 Opcode = ISD::SHL;
1653 SplatStepVal = Log2_64(std::abs(StepNumerator));
1654 }
1655 SDValue SplatStep = DAG.getSplatVector(
1656 VT, DL, DAG.getConstant(SplatStepVal, DL, XLenVT));
1657 VID = DAG.getNode(Opcode, DL, VT, VID, SplatStep);
1658 }
1659 if (StepDenominator != 1) {
1660 SDValue SplatStep = DAG.getSplatVector(
1661 VT, DL, DAG.getConstant(Log2_64(StepDenominator), DL, XLenVT));
1662 VID = DAG.getNode(ISD::SRL, DL, VT, VID, SplatStep);
1663 }
1664 if (Addend != 0 || Negate) {
1665 SDValue SplatAddend =
1666 DAG.getSplatVector(VT, DL, DAG.getConstant(Addend, DL, XLenVT));
1667 VID = DAG.getNode(Negate ? ISD::SUB : ISD::ADD, DL, VT, SplatAddend, VID);
1668 }
1669 return VID;
1670 }
1671 }
1672
1673 // Attempt to detect "hidden" splats, which only reveal themselves as splats
1674 // when re-interpreted as a vector with a larger element type. For example,
1675 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
1676 // could be instead splat as
1677 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
1678 // TODO: This optimization could also work on non-constant splats, but it
1679 // would require bit-manipulation instructions to construct the splat value.
1680 SmallVector<SDValue> Sequence;
1681 unsigned EltBitSize = VT.getScalarSizeInBits();
1682 const auto *BV = cast<BuildVectorSDNode>(Op);
1683 if (VT.isInteger() && EltBitSize < 64 &&
1684 ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
1685 BV->getRepeatedSequence(Sequence) &&
1686 (Sequence.size() * EltBitSize) <= 64) {
1687 unsigned SeqLen = Sequence.size();
1688 MVT ViaIntVT = MVT::getIntegerVT(EltBitSize * SeqLen);
1689 MVT ViaVecVT = MVT::getVectorVT(ViaIntVT, NumElts / SeqLen);
1690 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
1691 ViaIntVT == MVT::i64) &&
1692 "Unexpected sequence type");
1693
1694 unsigned EltIdx = 0;
1695 uint64_t EltMask = maskTrailingOnes<uint64_t>(EltBitSize);
1696 uint64_t SplatValue = 0;
1697 // Construct the amalgamated value which can be splatted as this larger
1698 // vector type.
1699 for (const auto &SeqV : Sequence) {
1700 if (!SeqV.isUndef())
1701 SplatValue |= ((cast<ConstantSDNode>(SeqV)->getZExtValue() & EltMask)
1702 << (EltIdx * EltBitSize));
1703 EltIdx++;
1704 }
1705
1706 // On RV64, sign-extend from 32 to 64 bits where possible in order to
1707 // achieve better constant materializion.
1708 if (Subtarget.is64Bit() && ViaIntVT == MVT::i32)
1709 SplatValue = SignExtend64(SplatValue, 32);
1710
1711 // Since we can't introduce illegal i64 types at this stage, we can only
1712 // perform an i64 splat on RV32 if it is its own sign-extended value. That
1713 // way we can use RVV instructions to splat.
1714 assert((ViaIntVT.bitsLE(XLenVT) ||
1715 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
1716 "Unexpected bitcast sequence");
1717 if (ViaIntVT.bitsLE(XLenVT) || isInt<32>(SplatValue)) {
1718 SDValue ViaVL =
1719 DAG.getConstant(ViaVecVT.getVectorNumElements(), DL, XLenVT);
1720 MVT ViaContainerVT =
1721 getContainerForFixedLengthVector(DAG, ViaVecVT, Subtarget);
1722 SDValue Splat =
1723 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ViaContainerVT,
1724 DAG.getConstant(SplatValue, DL, XLenVT), ViaVL);
1725 Splat = convertFromScalableVector(ViaVecVT, Splat, DAG, Subtarget);
1726 return DAG.getBitcast(VT, Splat);
1727 }
1728 }
1729
1730 // Try and optimize BUILD_VECTORs with "dominant values" - these are values
1731 // which constitute a large proportion of the elements. In such cases we can
1732 // splat a vector with the dominant element and make up the shortfall with
1733 // INSERT_VECTOR_ELTs.
1734 // Note that this includes vectors of 2 elements by association. The
1735 // upper-most element is the "dominant" one, allowing us to use a splat to
1736 // "insert" the upper element, and an insert of the lower element at position
1737 // 0, which improves codegen.
1738 SDValue DominantValue;
1739 unsigned MostCommonCount = 0;
1740 DenseMap<SDValue, unsigned> ValueCounts;
1741 unsigned NumUndefElts =
1742 count_if(Op->op_values(), [](const SDValue &V) { return V.isUndef(); });
1743
1744 // Track the number of scalar loads we know we'd be inserting, estimated as
1745 // any non-zero floating-point constant. Other kinds of element are either
1746 // already in registers or are materialized on demand. The threshold at which
1747 // a vector load is more desirable than several scalar materializion and
1748 // vector-insertion instructions is not known.
1749 unsigned NumScalarLoads = 0;
1750
1751 for (SDValue V : Op->op_values()) {
1752 if (V.isUndef())
1753 continue;
1754
1755 ValueCounts.insert(std::make_pair(V, 0));
1756 unsigned &Count = ValueCounts[V];
1757
1758 if (auto *CFP = dyn_cast<ConstantFPSDNode>(V))
1759 NumScalarLoads += !CFP->isExactlyValue(+0.0);
1760
1761 // Is this value dominant? In case of a tie, prefer the highest element as
1762 // it's cheaper to insert near the beginning of a vector than it is at the
1763 // end.
1764 if (++Count >= MostCommonCount) {
1765 DominantValue = V;
1766 MostCommonCount = Count;
1767 }
1768 }
1769
1770 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
1771 unsigned NumDefElts = NumElts - NumUndefElts;
1772 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
1773
1774 // Don't perform this optimization when optimizing for size, since
1775 // materializing elements and inserting them tends to cause code bloat.
1776 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
1777 ((MostCommonCount > DominantValueCountThreshold) ||
1778 (ValueCounts.size() <= Log2_32(NumDefElts)))) {
1779 // Start by splatting the most common element.
1780 SDValue Vec = DAG.getSplatBuildVector(VT, DL, DominantValue);
1781
1782 DenseSet<SDValue> Processed{DominantValue};
1783 MVT SelMaskTy = VT.changeVectorElementType(MVT::i1);
1784 for (const auto &OpIdx : enumerate(Op->ops())) {
1785 const SDValue &V = OpIdx.value();
1786 if (V.isUndef() || !Processed.insert(V).second)
1787 continue;
1788 if (ValueCounts[V] == 1) {
1789 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, Vec, V,
1790 DAG.getConstant(OpIdx.index(), DL, XLenVT));
1791 } else {
1792 // Blend in all instances of this value using a VSELECT, using a
1793 // mask where each bit signals whether that element is the one
1794 // we're after.
1795 SmallVector<SDValue> Ops;
1796 transform(Op->op_values(), std::back_inserter(Ops), [&](SDValue V1) {
1797 return DAG.getConstant(V == V1, DL, XLenVT);
1798 });
1799 Vec = DAG.getNode(ISD::VSELECT, DL, VT,
1800 DAG.getBuildVector(SelMaskTy, DL, Ops),
1801 DAG.getSplatBuildVector(VT, DL, V), Vec);
1802 }
1803 }
1804
1805 return Vec;
1806 }
1807
1808 return SDValue();
1809 }
1810
1811 static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Lo,
1812 SDValue Hi, SDValue VL, SelectionDAG &DAG) {
1813 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
1814 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
1815 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
1816 // If Hi constant is all the same sign bit as Lo, lower this as a custom
1817 // node in order to try and match RVV vector/scalar instructions.
1818 if ((LoC >> 31) == HiC)
1819 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Lo, VL);
1820 }
1821
1822 // Fall back to a stack store and stride x0 vector load.
1823 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, Lo, Hi, VL);
1824 }
1825
1826 // Called by type legalization to handle splat of i64 on RV32.
1827 // FIXME: We can optimize this when the type has sign or zero bits in one
1828 // of the halves.
1829 static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Scalar,
1830 SDValue VL, SelectionDAG &DAG) {
1831 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
1832 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
1833 DAG.getConstant(0, DL, MVT::i32));
1834 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
1835 DAG.getConstant(1, DL, MVT::i32));
1836 return splatPartsI64WithVL(DL, VT, Lo, Hi, VL, DAG);
1837 }
1838
1839 // This function lowers a splat of a scalar operand Splat with the vector
1840 // length VL. It ensures the final sequence is type legal, which is useful when
1841 // lowering a splat after type legalization.
1842 static SDValue lowerScalarSplat(SDValue Scalar, SDValue VL, MVT VT, SDLoc DL,
1843 SelectionDAG &DAG,
1844 const RISCVSubtarget &Subtarget) {
1845 if (VT.isFloatingPoint())
1846 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, VT, Scalar, VL);
1847
1848 MVT XLenVT = Subtarget.getXLenVT();
1849
1850 // Simplest case is that the operand needs to be promoted to XLenVT.
1851 if (Scalar.getValueType().bitsLE(XLenVT)) {
1852 // If the operand is a constant, sign extend to increase our chances
1853 // of being able to use a .vi instruction. ANY_EXTEND would become a
1854 // a zero extend and the simm5 check in isel would fail.
1855 // FIXME: Should we ignore the upper bits in isel instead?
1856 unsigned ExtOpc =
1857 isa<ConstantSDNode>(Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
1858 Scalar = DAG.getNode(ExtOpc, DL, XLenVT, Scalar);
1859 return DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, Scalar, VL);
1860 }
1861
1862 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
1863 "Unexpected scalar for splat lowering!");
1864
1865 // Otherwise use the more complicated splatting algorithm.
1866 return splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
1867 }
1868
1869 static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
1870 const RISCVSubtarget &Subtarget) {
1871 SDValue V1 = Op.getOperand(0);
1872 SDValue V2 = Op.getOperand(1);
1873 SDLoc DL(Op);
1874 MVT XLenVT = Subtarget.getXLenVT();
1875 MVT VT = Op.getSimpleValueType();
1876 unsigned NumElts = VT.getVectorNumElements();
1877 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op.getNode());
1878
1879 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
1880
1881 SDValue TrueMask, VL;
1882 std::tie(TrueMask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
1883
1884 if (SVN->isSplat()) {
1885 const int Lane = SVN->getSplatIndex();
1886 if (Lane >= 0) {
1887 MVT SVT = VT.getVectorElementType();
1888
1889 // Turn splatted vector load into a strided load with an X0 stride.
1890 SDValue V = V1;
1891 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
1892 // with undef.
1893 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
1894 int Offset = Lane;
1895 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
1896 int OpElements =
1897 V.getOperand(0).getSimpleValueType().getVectorNumElements();
1898 V = V.getOperand(Offset / OpElements);
1899 Offset %= OpElements;
1900 }
1901
1902 // We need to ensure the load isn't atomic or volatile.
1903 if (ISD::isNormalLoad(V.getNode()) && cast<LoadSDNode>(V)->isSimple()) {
1904 auto *Ld = cast<LoadSDNode>(V);
1905 Offset *= SVT.getStoreSize();
1906 SDValue NewAddr = DAG.getMemBasePlusOffset(Ld->getBasePtr(),
1907 TypeSize::Fixed(Offset), DL);
1908
1909 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
1910 if (SVT.isInteger() && SVT.bitsGT(XLenVT)) {
1911 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
1912 SDValue IntID =
1913 DAG.getTargetConstant(Intrinsic::riscv_vlse, DL, XLenVT);
1914 SDValue Ops[] = {Ld->getChain(), IntID, NewAddr,
1915 DAG.getRegister(RISCV::X0, XLenVT), VL};
1916 SDValue NewLoad = DAG.getMemIntrinsicNode(
1917 ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops, SVT,
1918 DAG.getMachineFunction().getMachineMemOperand(
1919 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
1920 DAG.makeEquivalentMemoryOrdering(Ld, NewLoad);
1921 return convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
1922 }
1923
1924 // Otherwise use a scalar load and splat. This will give the best
1925 // opportunity to fold a splat into the operation. ISel can turn it into
1926 // the x0 strided load if we aren't able to fold away the select.
1927 if (SVT.isFloatingPoint())
1928 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
1929 Ld->getPointerInfo().getWithOffset(Offset),
1930 Ld->getOriginalAlign(),
1931 Ld->getMemOperand()->getFlags());
1932 else
1933 V = DAG.getExtLoad(ISD::SEXTLOAD, DL, XLenVT, Ld->getChain(), NewAddr,
1934 Ld->getPointerInfo().getWithOffset(Offset), SVT,
1935 Ld->getOriginalAlign(),
1936 Ld->getMemOperand()->getFlags());
1937 DAG.makeEquivalentMemoryOrdering(Ld, V);
1938
1939 unsigned Opc =
1940 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
1941 SDValue Splat = DAG.getNode(Opc, DL, ContainerVT, V, VL);
1942 return convertFromScalableVector(VT, Splat, DAG, Subtarget);
1943 }
1944
1945 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
1946 assert(Lane < (int)NumElts && "Unexpected lane!");
1947 SDValue Gather =
1948 DAG.getNode(RISCVISD::VRGATHER_VX_VL, DL, ContainerVT, V1,
1949 DAG.getConstant(Lane, DL, XLenVT), TrueMask, VL);
1950 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
1951 }
1952 }
1953
1954 // Detect shuffles which can be re-expressed as vector selects; these are
1955 // shuffles in which each element in the destination is taken from an element
1956 // at the corresponding index in either source vectors.
1957 bool IsSelect = all_of(enumerate(SVN->getMask()), [&](const auto &MaskIdx) {
1958 int MaskIndex = MaskIdx.value();
1959 return MaskIndex < 0 || MaskIdx.index() == (unsigned)MaskIndex % NumElts;
1960 });
1961
1962 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
1963
1964 SmallVector<SDValue> MaskVals;
1965 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
1966 // merged with a second vrgather.
1967 SmallVector<SDValue> GatherIndicesLHS, GatherIndicesRHS;
1968
1969 // By default we preserve the original operand order, and use a mask to
1970 // select LHS as true and RHS as false. However, since RVV vector selects may
1971 // feature splats but only on the LHS, we may choose to invert our mask and
1972 // instead select between RHS and LHS.
1973 bool SwapOps = DAG.isSplatValue(V2) && !DAG.isSplatValue(V1);
1974 bool InvertMask = IsSelect == SwapOps;
1975
1976 // Keep a track of which non-undef indices are used by each LHS/RHS shuffle
1977 // half.
1978 DenseMap<int, unsigned> LHSIndexCounts, RHSIndexCounts;
1979
1980 // Now construct the mask that will be used by the vselect or blended
1981 // vrgather operation. For vrgathers, construct the appropriate indices into
1982 // each vector.
1983 for (int MaskIndex : SVN->getMask()) {
1984 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ InvertMask;
1985 MaskVals.push_back(DAG.getConstant(SelectMaskVal, DL, XLenVT));
1986 if (!IsSelect) {
1987 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
1988 GatherIndicesLHS.push_back(IsLHSOrUndefIndex && MaskIndex >= 0
1989 ? DAG.getConstant(MaskIndex, DL, XLenVT)
1990 : DAG.getUNDEF(XLenVT));
1991 GatherIndicesRHS.push_back(
1992 IsLHSOrUndefIndex ? DAG.getUNDEF(XLenVT)
1993 : DAG.getConstant(MaskIndex - NumElts, DL, XLenVT));
1994 if (IsLHSOrUndefIndex && MaskIndex >= 0)
1995 ++LHSIndexCounts[MaskIndex];
1996 if (!IsLHSOrUndefIndex)
1997 ++RHSIndexCounts[MaskIndex - NumElts];
1998 }
1999 }
2000
2001 if (SwapOps) {
2002 std::swap(V1, V2);
2003 std::swap(GatherIndicesLHS, GatherIndicesRHS);
2004 }
2005
2006 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
2007 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
2008 SDValue SelectMask = DAG.getBuildVector(MaskVT, DL, MaskVals);
2009
2010 if (IsSelect)
2011 return DAG.getNode(ISD::VSELECT, DL, VT, SelectMask, V1, V2);
2012
2013 if (VT.getScalarSizeInBits() == 8 && VT.getVectorNumElements() > 256) {
2014 // On such a large vector we're unable to use i8 as the index type.
2015 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
2016 // may involve vector splitting if we're already at LMUL=8, or our
2017 // user-supplied maximum fixed-length LMUL.
2018 return SDValue();
2019 }
2020
2021 unsigned GatherVXOpc = RISCVISD::VRGATHER_VX_VL;
2022 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
2023 MVT IndexVT = VT.changeTypeToInteger();
2024 // Since we can't introduce illegal index types at this stage, use i16 and
2025 // vrgatherei16 if the corresponding index type for plain vrgather is greater
2026 // than XLenVT.
2027 if (IndexVT.getScalarType().bitsGT(XLenVT)) {
2028 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
2029 IndexVT = IndexVT.changeVectorElementType(MVT::i16);
2030 }
2031
2032 MVT IndexContainerVT =
2033 ContainerVT.changeVectorElementType(IndexVT.getScalarType());
2034
2035 SDValue Gather;
2036 // TODO: This doesn't trigger for i64 vectors on RV32, since there we
2037 // encounter a bitcasted BUILD_VECTOR with low/high i32 values.
2038 if (SDValue SplatValue = DAG.getSplatValue(V1, /*LegalTypes*/ true)) {
2039 Gather = lowerScalarSplat(SplatValue, VL, ContainerVT, DL, DAG, Subtarget);
2040 } else {
2041 V1 = convertToScalableVector(ContainerVT, V1, DAG, Subtarget);
2042 // If only one index is used, we can use a "splat" vrgather.
2043 // TODO: We can splat the most-common index and fix-up any stragglers, if
2044 // that's beneficial.
2045 if (LHSIndexCounts.size() == 1) {
2046 int SplatIndex = LHSIndexCounts.begin()->getFirst();
2047 Gather =
2048 DAG.getNode(GatherVXOpc, DL, ContainerVT, V1,
2049 DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2050 } else {
2051 SDValue LHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesLHS);
2052 LHSIndices =
2053 convertToScalableVector(IndexContainerVT, LHSIndices, DAG, Subtarget);
2054
2055 Gather = DAG.getNode(GatherVVOpc, DL, ContainerVT, V1, LHSIndices,
2056 TrueMask, VL);
2057 }
2058 }
2059
2060 // If a second vector operand is used by this shuffle, blend it in with an
2061 // additional vrgather.
2062 if (!V2.isUndef()) {
2063 V2 = convertToScalableVector(ContainerVT, V2, DAG, Subtarget);
2064 // If only one index is used, we can use a "splat" vrgather.
2065 // TODO: We can splat the most-common index and fix-up any stragglers, if
2066 // that's beneficial.
2067 if (RHSIndexCounts.size() == 1) {
2068 int SplatIndex = RHSIndexCounts.begin()->getFirst();
2069 V2 = DAG.getNode(GatherVXOpc, DL, ContainerVT, V2,
2070 DAG.getConstant(SplatIndex, DL, XLenVT), TrueMask, VL);
2071 } else {
2072 SDValue RHSIndices = DAG.getBuildVector(IndexVT, DL, GatherIndicesRHS);
2073 RHSIndices =
2074 convertToScalableVector(IndexContainerVT, RHSIndices, DAG, Subtarget);
2075 V2 = DAG.getNode(GatherVVOpc, DL, ContainerVT, V2, RHSIndices, TrueMask,
2076 VL);
2077 }
2078
2079 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
2080 SelectMask =
2081 convertToScalableVector(MaskContainerVT, SelectMask, DAG, Subtarget);
2082
2083 Gather = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, SelectMask, V2,
2084 Gather, VL);
2085 }
2086
2087 return convertFromScalableVector(VT, Gather, DAG, Subtarget);
2088 }
2089
2090 static SDValue getRVVFPExtendOrRound(SDValue Op, MVT VT, MVT ContainerVT,
2091 SDLoc DL, SelectionDAG &DAG,
2092 const RISCVSubtarget &Subtarget) {
2093 if (VT.isScalableVector())
2094 return DAG.getFPExtendOrRound(Op, DL, VT);
2095 assert(VT.isFixedLengthVector() &&
2096 "Unexpected value type for RVV FP extend/round lowering");
2097 SDValue Mask, VL;
2098 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2099 unsigned RVVOpc = ContainerVT.bitsGT(Op.getSimpleValueType())
2100 ? RISCVISD::FP_EXTEND_VL
2101 : RISCVISD::FP_ROUND_VL;
2102 return DAG.getNode(RVVOpc, DL, ContainerVT, Op, Mask, VL);
2103 }
2104
2105 // While RVV has alignment restrictions, we should always be able to load as a
2106 // legal equivalently-sized byte-typed vector instead. This method is
2107 // responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
2108 // the load is already correctly-aligned, it returns SDValue().
2109 SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
2110 SelectionDAG &DAG) const {
2111 auto *Load = cast<LoadSDNode>(Op);
2112 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
2113
2114 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2115 Load->getMemoryVT(),
2116 *Load->getMemOperand()))
2117 return SDValue();
2118
2119 SDLoc DL(Op);
2120 MVT VT = Op.getSimpleValueType();
2121 unsigned EltSizeBits = VT.getScalarSizeInBits();
2122 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2123 "Unexpected unaligned RVV load type");
2124 MVT NewVT =
2125 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2126 assert(NewVT.isValid() &&
2127 "Expecting equally-sized RVV vector types to be legal");
2128 SDValue L = DAG.getLoad(NewVT, DL, Load->getChain(), Load->getBasePtr(),
2129 Load->getPointerInfo(), Load->getOriginalAlign(),
2130 Load->getMemOperand()->getFlags());
2131 return DAG.getMergeValues({DAG.getBitcast(VT, L), L.getValue(1)}, DL);
2132 }
2133
2134 // While RVV has alignment restrictions, we should always be able to store as a
2135 // legal equivalently-sized byte-typed vector instead. This method is
2136 // responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
2137 // returns SDValue() if the store is already correctly aligned.
2138 SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
2139 SelectionDAG &DAG) const {
2140 auto *Store = cast<StoreSDNode>(Op);
2141 assert(Store && Store->getValue().getValueType().isVector() &&
2142 "Expected vector store");
2143
2144 if (allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
2145 Store->getMemoryVT(),
2146 *Store->getMemOperand()))
2147 return SDValue();
2148
2149 SDLoc DL(Op);
2150 SDValue StoredVal = Store->getValue();
2151 MVT VT = StoredVal.getSimpleValueType();
2152 unsigned EltSizeBits = VT.getScalarSizeInBits();
2153 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
2154 "Unexpected unaligned RVV store type");
2155 MVT NewVT =
2156 MVT::getVectorVT(MVT::i8, VT.getVectorElementCount() * (EltSizeBits / 8));
2157 assert(NewVT.isValid() &&
2158 "Expecting equally-sized RVV vector types to be legal");
2159 StoredVal = DAG.getBitcast(NewVT, StoredVal);
2160 return DAG.getStore(Store->getChain(), DL, StoredVal, Store->getBasePtr(),
2161 Store->getPointerInfo(), Store->getOriginalAlign(),
2162 Store->getMemOperand()->getFlags());
2163 }
2164
2165 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
2166 SelectionDAG &DAG) const {
2167 switch (Op.getOpcode()) {
2168 default:
2169 report_fatal_error("unimplemented operand");
2170 case ISD::GlobalAddress:
2171 return lowerGlobalAddress(Op, DAG);
2172 case ISD::BlockAddress:
2173 return lowerBlockAddress(Op, DAG);
2174 case ISD::ConstantPool:
2175 return lowerConstantPool(Op, DAG);
2176 case ISD::JumpTable:
2177 return lowerJumpTable(Op, DAG);
2178 case ISD::GlobalTLSAddress:
2179 return lowerGlobalTLSAddress(Op, DAG);
2180 case ISD::SELECT:
2181 return lowerSELECT(Op, DAG);
2182 case ISD::BRCOND:
2183 return lowerBRCOND(Op, DAG);
2184 case ISD::VASTART:
2185 return lowerVASTART(Op, DAG);
2186 case ISD::FRAMEADDR:
2187 return lowerFRAMEADDR(Op, DAG);
2188 case ISD::RETURNADDR:
2189 return lowerRETURNADDR(Op, DAG);
2190 case ISD::SHL_PARTS:
2191 return lowerShiftLeftParts(Op, DAG);
2192 case ISD::SRA_PARTS:
2193 return lowerShiftRightParts(Op, DAG, true);
2194 case ISD::SRL_PARTS:
2195 return lowerShiftRightParts(Op, DAG, false);
2196 case ISD::BITCAST: {
2197 SDLoc DL(Op);
2198 EVT VT = Op.getValueType();
2199 SDValue Op0 = Op.getOperand(0);
2200 EVT Op0VT = Op0.getValueType();
2201 MVT XLenVT = Subtarget.getXLenVT();
2202 if (VT.isFixedLengthVector()) {
2203 // We can handle fixed length vector bitcasts with a simple replacement
2204 // in isel.
2205 if (Op0VT.isFixedLengthVector())
2206 return Op;
2207 // When bitcasting from scalar to fixed-length vector, insert the scalar
2208 // into a one-element vector of the result type, and perform a vector
2209 // bitcast.
2210 if (!Op0VT.isVector()) {
2211 auto BVT = EVT::getVectorVT(*DAG.getContext(), Op0VT, 1);
2212 return DAG.getBitcast(VT, DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, BVT,
2213 DAG.getUNDEF(BVT), Op0,
2214 DAG.getConstant(0, DL, XLenVT)));
2215 }
2216 return SDValue();
2217 }
2218 // Custom-legalize bitcasts from fixed-length vector types to scalar types
2219 // thus: bitcast the vector to a one-element vector type whose element type
2220 // is the same as the result type, and extract the first element.
2221 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
2222 LLVMContext &Context = *DAG.getContext();
2223 SDValue BVec = DAG.getBitcast(EVT::getVectorVT(Context, VT, 1), Op0);
2224 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
2225 DAG.getConstant(0, DL, XLenVT));
2226 }
2227 if (VT == MVT::f16 && Op0VT == MVT::i16 && Subtarget.hasStdExtZfh()) {
2228 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Op0);
2229 SDValue FPConv = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, NewOp0);
2230 return FPConv;
2231 }
2232 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
2233 Subtarget.hasStdExtF()) {
2234 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
2235 SDValue FPConv =
2236 DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
2237 return FPConv;
2238 }
2239 return SDValue();
2240 }
2241 case ISD::INTRINSIC_WO_CHAIN:
2242 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
2243 case ISD::INTRINSIC_W_CHAIN:
2244 return LowerINTRINSIC_W_CHAIN(Op, DAG);
2245 case ISD::BSWAP:
2246 case ISD::BITREVERSE: {
2247 // Convert BSWAP/BITREVERSE to GREVI to enable GREVI combinining.
2248 assert(Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
2249 MVT VT = Op.getSimpleValueType();
2250 SDLoc DL(Op);
2251 // Start with the maximum immediate value which is the bitwidth - 1.
2252 unsigned Imm = VT.getSizeInBits() - 1;
2253 // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
2254 if (Op.getOpcode() == ISD::BSWAP)
2255 Imm &= ~0x7U;
2256 return DAG.getNode(RISCVISD::GREV, DL, VT, Op.getOperand(0),
2257 DAG.getConstant(Imm, DL, VT));
2258 }
2259 case ISD::FSHL:
2260 case ISD::FSHR: {
2261 MVT VT = Op.getSimpleValueType();
2262 assert(VT == Subtarget.getXLenVT() && "Unexpected custom legalization");
2263 SDLoc DL(Op);
2264 if (Op.getOperand(2).getOpcode() == ISD::Constant)
2265 return Op;
2266 // FSL/FSR take a log2(XLen)+1 bit shift amount but XLenVT FSHL/FSHR only
2267 // use log(XLen) bits. Mask the shift amount accordingly.
2268 unsigned ShAmtWidth = Subtarget.getXLen() - 1;
2269 SDValue ShAmt = DAG.getNode(ISD::AND, DL, VT, Op.getOperand(2),
2270 DAG.getConstant(ShAmtWidth, DL, VT));
2271 unsigned Opc = Op.getOpcode() == ISD::FSHL ? RISCVISD::FSL : RISCVISD::FSR;
2272 return DAG.getNode(Opc, DL, VT, Op.getOperand(0), Op.getOperand(1), ShAmt);
2273 }
2274 case ISD::TRUNCATE: {
2275 SDLoc DL(Op);
2276 MVT VT = Op.getSimpleValueType();
2277 // Only custom-lower vector truncates
2278 if (!VT.isVector())
2279 return Op;
2280
2281 // Truncates to mask types are handled differently
2282 if (VT.getVectorElementType() == MVT::i1)
2283 return lowerVectorMaskTrunc(Op, DAG);
2284
2285 // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
2286 // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
2287 // truncate by one power of two at a time.
2288 MVT DstEltVT = VT.getVectorElementType();
2289
2290 SDValue Src = Op.getOperand(0);
2291 MVT SrcVT = Src.getSimpleValueType();
2292 MVT SrcEltVT = SrcVT.getVectorElementType();
2293
2294 assert(DstEltVT.bitsLT(SrcEltVT) &&
2295 isPowerOf2_64(DstEltVT.getSizeInBits()) &&
2296 isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
2297 "Unexpected vector truncate lowering");
2298
2299 MVT ContainerVT = SrcVT;
2300 if (SrcVT.isFixedLengthVector()) {
2301 ContainerVT = getContainerForFixedLengthVector(SrcVT);
2302 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
2303 }
2304
2305 SDValue Result = Src;
2306 SDValue Mask, VL;
2307 std::tie(Mask, VL) =
2308 getDefaultVLOps(SrcVT, ContainerVT, DL, DAG, Subtarget);
2309 LLVMContext &Context = *DAG.getContext();
2310 const ElementCount Count = ContainerVT.getVectorElementCount();
2311 do {
2312 SrcEltVT = MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2);
2313 EVT ResultVT = EVT::getVectorVT(Context, SrcEltVT, Count);
2314 Result = DAG.getNode(RISCVISD::TRUNCATE_VECTOR_VL, DL, ResultVT, Result,
2315 Mask, VL);
2316 } while (SrcEltVT != DstEltVT);
2317
2318 if (SrcVT.isFixedLengthVector())
2319 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
2320
2321 return Result;
2322 }
2323 case ISD::ANY_EXTEND:
2324 case ISD::ZERO_EXTEND:
2325 if (Op.getOperand(0).getValueType().isVector() &&
2326 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
2327 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ 1);
2328 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VZEXT_VL);
2329 case ISD::SIGN_EXTEND:
2330 if (Op.getOperand(0).getValueType().isVector() &&
2331 Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
2332 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ -1);
2333 return lowerFixedLengthVectorExtendToRVV(Op, DAG, RISCVISD::VSEXT_VL);
2334 case ISD::SPLAT_VECTOR_PARTS:
2335 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
2336 case ISD::INSERT_VECTOR_ELT:
2337 return lowerINSERT_VECTOR_ELT(Op, DAG);
2338 case ISD::EXTRACT_VECTOR_ELT:
2339 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
2340 case ISD::VSCALE: {
2341 MVT VT = Op.getSimpleValueType();
2342 SDLoc DL(Op);
2343 SDValue VLENB = DAG.getNode(RISCVISD::READ_VLENB, DL, VT);
2344 // We define our scalable vector types for lmul=1 to use a 64 bit known
2345 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
2346 // vscale as VLENB / 8.
2347 assert(RISCV::RVVBitsPerBlock == 64 && "Unexpected bits per block!");
2348 if (isa<ConstantSDNode>(Op.getOperand(0))) {
2349 // We assume VLENB is a multiple of 8. We manually choose the best shift
2350 // here because SimplifyDemandedBits isn't always able to simplify it.
2351 uint64_t Val = Op.getConstantOperandVal(0);
2352 if (isPowerOf2_64(Val)) {
2353 uint64_t Log2 = Log2_64(Val);
2354 if (Log2 < 3)
2355 return DAG.getNode(ISD::SRL, DL, VT, VLENB,
2356 DAG.getConstant(3 - Log2, DL, VT));
2357 if (Log2 > 3)
2358 return DAG.getNode(ISD::SHL, DL, VT, VLENB,
2359 DAG.getConstant(Log2 - 3, DL, VT));
2360 return VLENB;
2361 }
2362 // If the multiplier is a multiple of 8, scale it down to avoid needing
2363 // to shift the VLENB value.
2364 if ((Val % 8) == 0)
2365 return DAG.getNode(ISD::MUL, DL, VT, VLENB,
2366 DAG.getConstant(Val / 8, DL, VT));
2367 }
2368
2369 SDValue VScale = DAG.getNode(ISD::SRL, DL, VT, VLENB,
2370 DAG.getConstant(3, DL, VT));
2371 return DAG.getNode(ISD::MUL, DL, VT, VScale, Op.getOperand(0));
2372 }
2373 case ISD::FP_EXTEND: {
2374 // RVV can only do fp_extend to types double the size as the source. We
2375 // custom-lower f16->f64 extensions to two hops of ISD::FP_EXTEND, going
2376 // via f32.
2377 SDLoc DL(Op);
2378 MVT VT = Op.getSimpleValueType();
2379 SDValue Src = Op.getOperand(0);
2380 MVT SrcVT = Src.getSimpleValueType();
2381
2382 // Prepare any fixed-length vector operands.
2383 MVT ContainerVT = VT;
2384 if (SrcVT.isFixedLengthVector()) {
2385 ContainerVT = getContainerForFixedLengthVector(VT);
2386 MVT SrcContainerVT =
2387 ContainerVT.changeVectorElementType(SrcVT.getVectorElementType());
2388 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2389 }
2390
2391 if (!VT.isVector() || VT.getVectorElementType() != MVT::f64 ||
2392 SrcVT.getVectorElementType() != MVT::f16) {
2393 // For scalable vectors, we only need to close the gap between
2394 // vXf16->vXf64.
2395 if (!VT.isFixedLengthVector())
2396 return Op;
2397 // For fixed-length vectors, lower the FP_EXTEND to a custom "VL" version.
2398 Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
2399 return convertFromScalableVector(VT, Src, DAG, Subtarget);
2400 }
2401
2402 MVT InterVT = VT.changeVectorElementType(MVT::f32);
2403 MVT InterContainerVT = ContainerVT.changeVectorElementType(MVT::f32);
2404 SDValue IntermediateExtend = getRVVFPExtendOrRound(
2405 Src, InterVT, InterContainerVT, DL, DAG, Subtarget);
2406
2407 SDValue Extend = getRVVFPExtendOrRound(IntermediateExtend, VT, ContainerVT,
2408 DL, DAG, Subtarget);
2409 if (VT.isFixedLengthVector())
2410 return convertFromScalableVector(VT, Extend, DAG, Subtarget);
2411 return Extend;
2412 }
2413 case ISD::FP_ROUND: {
2414 // RVV can only do fp_round to types half the size as the source. We
2415 // custom-lower f64->f16 rounds via RVV's round-to-odd float
2416 // conversion instruction.
2417 SDLoc DL(Op);
2418 MVT VT = Op.getSimpleValueType();
2419 SDValue Src = Op.getOperand(0);
2420 MVT SrcVT = Src.getSimpleValueType();
2421
2422 // Prepare any fixed-length vector operands.
2423 MVT ContainerVT = VT;
2424 if (VT.isFixedLengthVector()) {
2425 MVT SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
2426 ContainerVT =
2427 SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
2428 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2429 }
2430
2431 if (!VT.isVector() || VT.getVectorElementType() != MVT::f16 ||
2432 SrcVT.getVectorElementType() != MVT::f64) {
2433 // For scalable vectors, we only need to close the gap between
2434 // vXf64<->vXf16.
2435 if (!VT.isFixedLengthVector())
2436 return Op;
2437 // For fixed-length vectors, lower the FP_ROUND to a custom "VL" version.
2438 Src = getRVVFPExtendOrRound(Src, VT, ContainerVT, DL, DAG, Subtarget);
2439 return convertFromScalableVector(VT, Src, DAG, Subtarget);
2440 }
2441
2442 SDValue Mask, VL;
2443 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2444
2445 MVT InterVT = ContainerVT.changeVectorElementType(MVT::f32);
2446 SDValue IntermediateRound =
2447 DAG.getNode(RISCVISD::VFNCVT_ROD_VL, DL, InterVT, Src, Mask, VL);
2448 SDValue Round = getRVVFPExtendOrRound(IntermediateRound, VT, ContainerVT,
2449 DL, DAG, Subtarget);
2450
2451 if (VT.isFixedLengthVector())
2452 return convertFromScalableVector(VT, Round, DAG, Subtarget);
2453 return Round;
2454 }
2455 case ISD::FP_TO_SINT:
2456 case ISD::FP_TO_UINT:
2457 case ISD::SINT_TO_FP:
2458 case ISD::UINT_TO_FP: {
2459 // RVV can only do fp<->int conversions to types half/double the size as
2460 // the source. We custom-lower any conversions that do two hops into
2461 // sequences.
2462 MVT VT = Op.getSimpleValueType();
2463 if (!VT.isVector())
2464 return Op;
2465 SDLoc DL(Op);
2466 SDValue Src = Op.getOperand(0);
2467 MVT EltVT = VT.getVectorElementType();
2468 MVT SrcVT = Src.getSimpleValueType();
2469 MVT SrcEltVT = SrcVT.getVectorElementType();
2470 unsigned EltSize = EltVT.getSizeInBits();
2471 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
2472 assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
2473 "Unexpected vector element types");
2474
2475 bool IsInt2FP = SrcEltVT.isInteger();
2476 // Widening conversions
2477 if (EltSize > SrcEltSize && (EltSize / SrcEltSize >= 4)) {
2478 if (IsInt2FP) {
2479 // Do a regular integer sign/zero extension then convert to float.
2480 MVT IVecVT = MVT::getVectorVT(MVT::getIntegerVT(EltVT.getSizeInBits()),
2481 VT.getVectorElementCount());
2482 unsigned ExtOpcode = Op.getOpcode() == ISD::UINT_TO_FP
2483 ? ISD::ZERO_EXTEND
2484 : ISD::SIGN_EXTEND;
2485 SDValue Ext = DAG.getNode(ExtOpcode, DL, IVecVT, Src);
2486 return DAG.getNode(Op.getOpcode(), DL, VT, Ext);
2487 }
2488 // FP2Int
2489 assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
2490 // Do one doubling fp_extend then complete the operation by converting
2491 // to int.
2492 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
2493 SDValue FExt = DAG.getFPExtendOrRound(Src, DL, InterimFVT);
2494 return DAG.getNode(Op.getOpcode(), DL, VT, FExt);
2495 }
2496
2497 // Narrowing conversions
2498 if (SrcEltSize > EltSize && (SrcEltSize / EltSize >= 4)) {
2499 if (IsInt2FP) {
2500 // One narrowing int_to_fp, then an fp_round.
2501 assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
2502 MVT InterimFVT = MVT::getVectorVT(MVT::f32, VT.getVectorElementCount());
2503 SDValue Int2FP = DAG.getNode(Op.getOpcode(), DL, InterimFVT, Src);
2504 return DAG.getFPExtendOrRound(Int2FP, DL, VT);
2505 }
2506 // FP2Int
2507 // One narrowing fp_to_int, then truncate the integer. If the float isn't
2508 // representable by the integer, the result is poison.
2509 MVT IVecVT =
2510 MVT::getVectorVT(MVT::getIntegerVT(SrcEltVT.getSizeInBits() / 2),
2511 VT.getVectorElementCount());
2512 SDValue FP2Int = DAG.getNode(Op.getOpcode(), DL, IVecVT, Src);
2513 return DAG.getNode(ISD::TRUNCATE, DL, VT, FP2Int);
2514 }
2515
2516 // Scalable vectors can exit here. Patterns will handle equally-sized
2517 // conversions halving/doubling ones.
2518 if (!VT.isFixedLengthVector())
2519 return Op;
2520
2521 // For fixed-length vectors we lower to a custom "VL" node.
2522 unsigned RVVOpc = 0;
2523 switch (Op.getOpcode()) {
2524 default:
2525 llvm_unreachable("Impossible opcode");
2526 case ISD::FP_TO_SINT:
2527 RVVOpc = RISCVISD::FP_TO_SINT_VL;
2528 break;
2529 case ISD::FP_TO_UINT:
2530 RVVOpc = RISCVISD::FP_TO_UINT_VL;
2531 break;
2532 case ISD::SINT_TO_FP:
2533 RVVOpc = RISCVISD::SINT_TO_FP_VL;
2534 break;
2535 case ISD::UINT_TO_FP:
2536 RVVOpc = RISCVISD::UINT_TO_FP_VL;
2537 break;
2538 }
2539
2540 MVT ContainerVT, SrcContainerVT;
2541 // Derive the reference container type from the larger vector type.
2542 if (SrcEltSize > EltSize) {
2543 SrcContainerVT = getContainerForFixedLengthVector(SrcVT);
2544 ContainerVT =
2545 SrcContainerVT.changeVectorElementType(VT.getVectorElementType());
2546 } else {
2547 ContainerVT = getContainerForFixedLengthVector(VT);
2548 SrcContainerVT = ContainerVT.changeVectorElementType(SrcEltVT);
2549 }
2550
2551 SDValue Mask, VL;
2552 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
2553
2554 Src = convertToScalableVector(SrcContainerVT, Src, DAG, Subtarget);
2555 Src = DAG.getNode(RVVOpc, DL, ContainerVT, Src, Mask, VL);
2556 return convertFromScalableVector(VT, Src, DAG, Subtarget);
2557 }
2558 case ISD::FP_TO_SINT_SAT:
2559 case ISD::FP_TO_UINT_SAT:
2560 return lowerFP_TO_INT_SAT(Op, DAG);
2561 case ISD::VECREDUCE_ADD:
2562 case ISD::VECREDUCE_UMAX:
2563 case ISD::VECREDUCE_SMAX:
2564 case ISD::VECREDUCE_UMIN:
2565 case ISD::VECREDUCE_SMIN:
2566 return lowerVECREDUCE(Op, DAG);
2567 case ISD::VECREDUCE_AND:
2568 case ISD::VECREDUCE_OR:
2569 case ISD::VECREDUCE_XOR:
2570 if (Op.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
2571 return lowerVectorMaskVECREDUCE(Op, DAG);
2572 return lowerVECREDUCE(Op, DAG);
2573 case ISD::VECREDUCE_FADD:
2574 case ISD::VECREDUCE_SEQ_FADD:
2575 case ISD::VECREDUCE_FMIN:
2576 case ISD::VECREDUCE_FMAX:
2577 return lowerFPVECREDUCE(Op, DAG);
2578 case ISD::INSERT_SUBVECTOR:
2579 return lowerINSERT_SUBVECTOR(Op, DAG);
2580 case ISD::EXTRACT_SUBVECTOR:
2581 return lowerEXTRACT_SUBVECTOR(Op, DAG);
2582 case ISD::STEP_VECTOR:
2583 return lowerSTEP_VECTOR(Op, DAG);
2584 case ISD::VECTOR_REVERSE:
2585 return lowerVECTOR_REVERSE(Op, DAG);
2586 case ISD::BUILD_VECTOR:
2587 return lowerBUILD_VECTOR(Op, DAG, Subtarget);
2588 case ISD::SPLAT_VECTOR:
2589 if (Op.getValueType().getVectorElementType() == MVT::i1)
2590 return lowerVectorMaskSplat(Op, DAG);
2591 return lowerSPLAT_VECTOR(Op, DAG, Subtarget);
2592 case ISD::VECTOR_SHUFFLE:
2593 return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
2594 case ISD::CONCAT_VECTORS: {
2595 // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
2596 // better than going through the stack, as the default expansion does.
2597 SDLoc DL(Op);
2598 MVT VT = Op.getSimpleValueType();
2599 unsigned NumOpElts =
2600 Op.getOperand(0).getSimpleValueType().getVectorMinNumElements();
2601 SDValue Vec = DAG.getUNDEF(VT);
2602 for (const auto &OpIdx : enumerate(Op->ops()))
2603 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, Vec, OpIdx.value(),
2604 DAG.getIntPtrConstant(OpIdx.index() * NumOpElts, DL));
2605 return Vec;
2606 }
2607 case ISD::LOAD:
2608 if (auto V = expandUnalignedRVVLoad(Op, DAG))
2609 return V;
2610 if (Op.getValueType().isFixedLengthVector())
2611 return lowerFixedLengthVectorLoadToRVV(Op, DAG);
2612 return Op;
2613 case ISD::STORE:
2614 if (auto V = expandUnalignedRVVStore(Op, DAG))
2615 return V;
2616 if (Op.getOperand(1).getValueType().isFixedLengthVector())
2617 return lowerFixedLengthVectorStoreToRVV(Op, DAG);
2618 return Op;
2619 case ISD::MLOAD:
2620 return lowerMLOAD(Op, DAG);
2621 case ISD::MSTORE:
2622 return lowerMSTORE(Op, DAG);
2623 case ISD::SETCC:
2624 return lowerFixedLengthVectorSetccToRVV(Op, DAG);
2625 case ISD::ADD:
2626 return lowerToScalableOp(Op, DAG, RISCVISD::ADD_VL);
2627 case ISD::SUB:
2628 return lowerToScalableOp(Op, DAG, RISCVISD::SUB_VL);
2629 case ISD::MUL:
2630 return lowerToScalableOp(Op, DAG, RISCVISD::MUL_VL);
2631 case ISD::MULHS:
2632 return lowerToScalableOp(Op, DAG, RISCVISD::MULHS_VL);
2633 case ISD::MULHU:
2634 return lowerToScalableOp(Op, DAG, RISCVISD::MULHU_VL);
2635 case ISD::AND:
2636 return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMAND_VL,
2637 RISCVISD::AND_VL);
2638 case ISD::OR:
2639 return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMOR_VL,
2640 RISCVISD::OR_VL);
2641 case ISD::XOR:
2642 return lowerFixedLengthVectorLogicOpToRVV(Op, DAG, RISCVISD::VMXOR_VL,
2643 RISCVISD::XOR_VL);
2644 case ISD::SDIV:
2645 return lowerToScalableOp(Op, DAG, RISCVISD::SDIV_VL);
2646 case ISD::SREM:
2647 return lowerToScalableOp(Op, DAG, RISCVISD::SREM_VL);
2648 case ISD::UDIV:
2649 return lowerToScalableOp(Op, DAG, RISCVISD::UDIV_VL);
2650 case ISD::UREM:
2651 return lowerToScalableOp(Op, DAG, RISCVISD::UREM_VL);
2652 case ISD::SHL:
2653 case ISD::SRA:
2654 case ISD::SRL:
2655 if (Op.getSimpleValueType().isFixedLengthVector())
2656 return lowerFixedLengthVectorShiftToRVV(Op, DAG);
2657 // This can be called for an i32 shift amount that needs to be promoted.
2658 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
2659 "Unexpected custom legalisation");
2660 return SDValue();
2661 case ISD::SADDSAT:
2662 return lowerToScalableOp(Op, DAG, RISCVISD::SADDSAT_VL);
2663 case ISD::UADDSAT:
2664 return lowerToScalableOp(Op, DAG, RISCVISD::UADDSAT_VL);
2665 case ISD::SSUBSAT:
2666 return lowerToScalableOp(Op, DAG, RISCVISD::SSUBSAT_VL);
2667 case ISD::USUBSAT:
2668 return lowerToScalableOp(Op, DAG, RISCVISD::USUBSAT_VL);
2669 case ISD::FADD:
2670 return lowerToScalableOp(Op, DAG, RISCVISD::FADD_VL);
2671 case ISD::FSUB:
2672 return lowerToScalableOp(Op, DAG, RISCVISD::FSUB_VL);
2673 case ISD::FMUL:
2674 return lowerToScalableOp(Op, DAG, RISCVISD::FMUL_VL);
2675 case ISD::FDIV:
2676 return lowerToScalableOp(Op, DAG, RISCVISD::FDIV_VL);
2677 case ISD::FNEG:
2678 return lowerToScalableOp(Op, DAG, RISCVISD::FNEG_VL);
2679 case ISD::FABS:
2680 return lowerToScalableOp(Op, DAG, RISCVISD::FABS_VL);
2681 case ISD::FSQRT:
2682 return lowerToScalableOp(Op, DAG, RISCVISD::FSQRT_VL);
2683 case ISD::FMA:
2684 return lowerToScalableOp(Op, DAG, RISCVISD::FMA_VL);
2685 case ISD::SMIN:
2686 return lowerToScalableOp(Op, DAG, RISCVISD::SMIN_VL);
2687 case ISD::SMAX:
2688 return lowerToScalableOp(Op, DAG, RISCVISD::SMAX_VL);
2689 case ISD::UMIN:
2690 return lowerToScalableOp(Op, DAG, RISCVISD::UMIN_VL);
2691 case ISD::UMAX:
2692 return lowerToScalableOp(Op, DAG, RISCVISD::UMAX_VL);
2693 case ISD::FMINNUM:
2694 return lowerToScalableOp(Op, DAG, RISCVISD::FMINNUM_VL);
2695 case ISD::FMAXNUM:
2696 return lowerToScalableOp(Op, DAG, RISCVISD::FMAXNUM_VL);
2697 case ISD::ABS:
2698 return lowerABS(Op, DAG);
2699 case ISD::VSELECT:
2700 return lowerFixedLengthVectorSelectToRVV(Op, DAG);
2701 case ISD::FCOPYSIGN:
2702 return lowerFixedLengthVectorFCOPYSIGNToRVV(Op, DAG);
2703 case ISD::MGATHER:
2704 return lowerMGATHER(Op, DAG);
2705 case ISD::MSCATTER:
2706 return lowerMSCATTER(Op, DAG);
2707 case ISD::FLT_ROUNDS_:
2708 return lowerGET_ROUNDING(Op, DAG);
2709 case ISD::SET_ROUNDING:
2710 return lowerSET_ROUNDING(Op, DAG);
2711 case ISD::VP_ADD:
2712 return lowerVPOp(Op, DAG, RISCVISD::ADD_VL);
2713 case ISD::VP_SUB:
2714 return lowerVPOp(Op, DAG, RISCVISD::SUB_VL);
2715 case ISD::VP_MUL:
2716 return lowerVPOp(Op, DAG, RISCVISD::MUL_VL);
2717 case ISD::VP_SDIV:
2718 return lowerVPOp(Op, DAG, RISCVISD::SDIV_VL);
2719 case ISD::VP_UDIV:
2720 return lowerVPOp(Op, DAG, RISCVISD::UDIV_VL);
2721 case ISD::VP_SREM:
2722 return lowerVPOp(Op, DAG, RISCVISD::SREM_VL);
2723 case ISD::VP_UREM:
2724 return lowerVPOp(Op, DAG, RISCVISD::UREM_VL);
2725 case ISD::VP_AND:
2726 return lowerVPOp(Op, DAG, RISCVISD::AND_VL);
2727 case ISD::VP_OR:
2728 return lowerVPOp(Op, DAG, RISCVISD::OR_VL);
2729 case ISD::VP_XOR:
2730 return lowerVPOp(Op, DAG, RISCVISD::XOR_VL);
2731 case ISD::VP_ASHR:
2732 return lowerVPOp(Op, DAG, RISCVISD::SRA_VL);
2733 case ISD::VP_LSHR:
2734 return lowerVPOp(Op, DAG, RISCVISD::SRL_VL);
2735 case ISD::VP_SHL:
2736 return lowerVPOp(Op, DAG, RISCVISD::SHL_VL);
2737 case ISD::VP_FADD:
2738 return lowerVPOp(Op, DAG, RISCVISD::FADD_VL);
2739 case ISD::VP_FSUB:
2740 return lowerVPOp(Op, DAG, RISCVISD::FSUB_VL);
2741 case ISD::VP_FMUL:
2742 return lowerVPOp(Op, DAG, RISCVISD::FMUL_VL);
2743 case ISD::VP_FDIV:
2744 return lowerVPOp(Op, DAG, RISCVISD::FDIV_VL);
2745 }
2746 }
2747
2748 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
2749 SelectionDAG &DAG, unsigned Flags) {
2750 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
2751 }
2752
2753 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
2754 SelectionDAG &DAG, unsigned Flags) {
2755 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
2756 Flags);
2757 }
2758
2759 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
2760 SelectionDAG &DAG, unsigned Flags) {
2761 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
2762 N->getOffset(), Flags);
2763 }
2764
2765 static SDValue getTargetNode(JumpTableSDNode *N, SDLoc DL, EVT Ty,
2766 SelectionDAG &DAG, unsigned Flags) {
2767 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
2768 }
2769
2770 template <class NodeTy>
2771 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
2772 bool IsLocal) const {
2773 SDLoc DL(N);
2774 EVT Ty = getPointerTy(DAG.getDataLayout());
2775
2776 if (isPositionIndependent()) {
2777 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
2778 if (IsLocal)
2779 // Use PC-relative addressing to access the symbol. This generates the
2780 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
2781 // %pcrel_lo(auipc)).
2782 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
2783
2784 // Use PC-relative addressing to access the GOT for this symbol, then load
2785 // the address from the GOT. This generates the pattern (PseudoLA sym),
2786 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
2787 return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
2788 }
2789
2790 switch (getTargetMachine().getCodeModel()) {
2791 default:
2792 report_fatal_error("Unsupported code model for lowering");
2793 case CodeModel::Small: {
2794 // Generate a sequence for accessing addresses within the first 2 GiB of
2795 // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
2796 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
2797 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
2798 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
2799 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
2800 }
2801 case CodeModel::Medium: {
2802 // Generate a sequence for accessing addresses within any 2GiB range within
2803 // the address space. This generates the pattern (PseudoLLA sym), which
2804 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
2805 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
2806 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
2807 }
2808 }
2809 }
2810
2811 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
2812 SelectionDAG &DAG) const {
2813 SDLoc DL(Op);
2814 EVT Ty = Op.getValueType();
2815 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
2816 int64_t Offset = N->getOffset();
2817 MVT XLenVT = Subtarget.getXLenVT();
2818
2819 const GlobalValue *GV = N->getGlobal();
2820 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
2821 SDValue Addr = getAddr(N, DAG, IsLocal);
2822
2823 // In order to maximise the opportunity for common subexpression elimination,
2824 // emit a separate ADD node for the global address offset instead of folding
2825 // it in the global address node. Later peephole optimisations may choose to
2826 // fold it back in when profitable.
2827 if (Offset != 0)
2828 return DAG.getNode(ISD::ADD, DL, Ty, Addr,
2829 DAG.getConstant(Offset, DL, XLenVT));
2830 return Addr;
2831 }
2832
2833 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
2834 SelectionDAG &DAG) const {
2835 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
2836
2837 return getAddr(N, DAG);
2838 }
2839
2840 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
2841 SelectionDAG &DAG) const {
2842 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
2843
2844 return getAddr(N, DAG);
2845 }
2846
2847 SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
2848 SelectionDAG &DAG) const {
2849 JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
2850
2851 return getAddr(N, DAG);
2852 }
2853
2854 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
2855 SelectionDAG &DAG,
2856 bool UseGOT) const {
2857 SDLoc DL(N);
2858 EVT Ty = getPointerTy(DAG.getDataLayout());
2859 const GlobalValue *GV = N->getGlobal();
2860 MVT XLenVT = Subtarget.getXLenVT();
2861
2862 if (UseGOT) {
2863 // Use PC-relative addressing to access the GOT for this TLS symbol, then
2864 // load the address from the GOT and add the thread pointer. This generates
2865 // the pattern (PseudoLA_TLS_IE sym), which expands to
2866 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
2867 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
2868 SDValue Load =
2869 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
2870
2871 // Add the thread pointer.
2872 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
2873 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
2874 }
2875
2876 // Generate a sequence for accessing the address relative to the thread
2877 // pointer, with the appropriate adjustment for the thread pointer offset.
2878 // This generates the pattern
2879 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
2880 SDValue AddrHi =
2881 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
2882 SDValue AddrAdd =
2883 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
2884 SDValue AddrLo =
2885 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
2886
2887 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
2888 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
2889 SDValue MNAdd = SDValue(
2890 DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
2891 0);
2892 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
2893 }
2894
2895 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
2896 SelectionDAG &DAG) const {
2897 SDLoc DL(N);
2898 EVT Ty = getPointerTy(DAG.getDataLayout());
2899 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
2900 const GlobalValue *GV = N->getGlobal();
2901
2902 // Use a PC-relative addressing mode to access the global dynamic GOT address.
2903 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
2904 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
2905 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
2906 SDValue Load =
2907 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
2908
2909 // Prepare argument list to generate call.
2910 ArgListTy Args;
2911 ArgListEntry Entry;
2912 Entry.Node = Load;
2913 Entry.Ty = CallTy;
2914 Args.push_back(Entry);
2915
2916 // Setup call to __tls_get_addr.
2917 TargetLowering::CallLoweringInfo CLI(DAG);
2918 CLI.setDebugLoc(DL)
2919 .setChain(DAG.getEntryNode())
2920 .setLibCallee(CallingConv::C, CallTy,
2921 DAG.getExternalSymbol("__tls_get_addr", Ty),
2922 std::move(Args));
2923
2924 return LowerCallTo(CLI).first;
2925 }
2926
2927 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
2928 SelectionDAG &DAG) const {
2929 SDLoc DL(Op);
2930 EVT Ty = Op.getValueType();
2931 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
2932 int64_t Offset = N->getOffset();
2933 MVT XLenVT = Subtarget.getXLenVT();
2934
2935 TLSModel::Model Model = getTargetMachine().getTLSModel(N->getGlobal());
2936
2937 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
2938 CallingConv::GHC)
2939 report_fatal_error("In GHC calling convention TLS is not supported");
2940
2941 SDValue Addr;
2942 switch (Model) {
2943 case TLSModel::LocalExec:
2944 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
2945 break;
2946 case TLSModel::InitialExec:
2947 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
2948 break;
2949 case TLSModel::LocalDynamic:
2950 case TLSModel::GeneralDynamic:
2951 Addr = getDynamicTLSAddr(N, DAG);
2952 break;
2953 }
2954
2955 // In order to maximise the opportunity for common subexpression elimination,
2956 // emit a separate ADD node for the global address offset instead of folding
2957 // it in the global address node. Later peephole optimisations may choose to
2958 // fold it back in when profitable.
2959 if (Offset != 0)
2960 return DAG.getNode(ISD::ADD, DL, Ty, Addr,
2961 DAG.getConstant(Offset, DL, XLenVT));
2962 return Addr;
2963 }
2964
2965 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
2966 SDValue CondV = Op.getOperand(0);
2967 SDValue TrueV = Op.getOperand(1);
2968 SDValue FalseV = Op.getOperand(2);
2969 SDLoc DL(Op);
2970 MVT VT = Op.getSimpleValueType();
2971 MVT XLenVT = Subtarget.getXLenVT();
2972
2973 // Lower vector SELECTs to VSELECTs by splatting the condition.
2974 if (VT.isVector()) {
2975 MVT SplatCondVT = VT.changeVectorElementType(MVT::i1);
2976 SDValue CondSplat = VT.isScalableVector()
2977 ? DAG.getSplatVector(SplatCondVT, DL, CondV)
2978 : DAG.getSplatBuildVector(SplatCondVT, DL, CondV);
2979 return DAG.getNode(ISD::VSELECT, DL, VT, CondSplat, TrueV, FalseV);
2980 }
2981
2982 // If the result type is XLenVT and CondV is the output of a SETCC node
2983 // which also operated on XLenVT inputs, then merge the SETCC node into the
2984 // lowered RISCVISD::SELECT_CC to take advantage of the integer
2985 // compare+branch instructions. i.e.:
2986 // (select (setcc lhs, rhs, cc), truev, falsev)
2987 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
2988 if (VT == XLenVT && CondV.getOpcode() == ISD::SETCC &&
2989 CondV.getOperand(0).getSimpleValueType() == XLenVT) {
2990 SDValue LHS = CondV.getOperand(0);
2991 SDValue RHS = CondV.getOperand(1);
2992 const auto *CC = cast<CondCodeSDNode>(CondV.getOperand(2));
2993 ISD::CondCode CCVal = CC->get();
2994
2995 // Special case for a select of 2 constants that have a diffence of 1.
2996 // Normally this is done by DAGCombine, but if the select is introduced by
2997 // type legalization or op legalization, we miss it. Restricting to SETLT
2998 // case for now because that is what signed saturating add/sub need.
2999 // FIXME: We don't need the condition to be SETLT or even a SETCC,
3000 // but we would probably want to swap the true/false values if the condition
3001 // is SETGE/SETLE to avoid an XORI.
3002 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
3003 CCVal == ISD::SETLT) {
3004 const APInt &TrueVal = cast<ConstantSDNode>(TrueV)->getAPIntValue();
3005 const APInt &FalseVal = cast<ConstantSDNode>(FalseV)->getAPIntValue();
3006 if (TrueVal - 1 == FalseVal)
3007 return DAG.getNode(ISD::ADD, DL, Op.getValueType(), CondV, FalseV);
3008 if (TrueVal + 1 == FalseVal)
3009 return DAG.getNode(ISD::SUB, DL, Op.getValueType(), FalseV, CondV);
3010 }
3011
3012 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3013
3014 SDValue TargetCC = DAG.getCondCode(CCVal);
3015 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
3016 return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3017 }
3018
3019 // Otherwise:
3020 // (select condv, truev, falsev)
3021 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
3022 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3023 SDValue SetNE = DAG.getCondCode(ISD::SETNE);
3024
3025 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
3026
3027 return DAG.getNode(RISCVISD::SELECT_CC, DL, Op.getValueType(), Ops);
3028 }
3029
3030 SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
3031 SDValue CondV = Op.getOperand(1);
3032 SDLoc DL(Op);
3033 MVT XLenVT = Subtarget.getXLenVT();
3034
3035 if (CondV.getOpcode() == ISD::SETCC &&
3036 CondV.getOperand(0).getValueType() == XLenVT) {
3037 SDValue LHS = CondV.getOperand(0);
3038 SDValue RHS = CondV.getOperand(1);
3039 ISD::CondCode CCVal = cast<CondCodeSDNode>(CondV.getOperand(2))->get();
3040
3041 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
3042
3043 SDValue TargetCC = DAG.getCondCode(CCVal);
3044 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3045 LHS, RHS, TargetCC, Op.getOperand(2));
3046 }
3047
3048 return DAG.getNode(RISCVISD::BR_CC, DL, Op.getValueType(), Op.getOperand(0),
3049 CondV, DAG.getConstant(0, DL, XLenVT),
3050 DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
3051 }
3052
3053 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3054 MachineFunction &MF = DAG.getMachineFunction();
3055 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
3056
3057 SDLoc DL(Op);
3058 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
3059 getPointerTy(MF.getDataLayout()));
3060
3061 // vastart just stores the address of the VarArgsFrameIndex slot into the
3062 // memory location argument.
3063 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
3064 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
3065 MachinePointerInfo(SV));
3066 }
3067
3068 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
3069 SelectionDAG &DAG) const {
3070 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3071 MachineFunction &MF = DAG.getMachineFunction();
3072 MachineFrameInfo &MFI = MF.getFrameInfo();
3073 MFI.setFrameAddressIsTaken(true);
3074 Register FrameReg = RI.getFrameRegister(MF);
3075 int XLenInBytes = Subtarget.getXLen() / 8;
3076
3077 EVT VT = Op.getValueType();
3078 SDLoc DL(Op);
3079 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
3080 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3081 while (Depth--) {
3082 int Offset = -(XLenInBytes * 2);
3083 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
3084 DAG.getIntPtrConstant(Offset, DL));
3085 FrameAddr =
3086 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
3087 }
3088 return FrameAddr;
3089 }
3090
3091 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
3092 SelectionDAG &DAG) const {
3093 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
3094 MachineFunction &MF = DAG.getMachineFunction();
3095 MachineFrameInfo &MFI = MF.getFrameInfo();
3096 MFI.setReturnAddressIsTaken(true);
3097 MVT XLenVT = Subtarget.getXLenVT();
3098 int XLenInBytes = Subtarget.getXLen() / 8;
3099
3100 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
3101 return SDValue();
3102
3103 EVT VT = Op.getValueType();
3104 SDLoc DL(Op);
3105 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
3106 if (Depth) {
3107 int Off = -XLenInBytes;
3108 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
3109 SDValue Offset = DAG.getConstant(Off, DL, VT);
3110 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
3111 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
3112 MachinePointerInfo());
3113 }
3114
3115 // Return the value of the return address register, marking it an implicit
3116 // live-in.
3117 Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
3118 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
3119 }
3120
3121 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
3122 SelectionDAG &DAG) const {
3123 SDLoc DL(Op);
3124 SDValue Lo = Op.getOperand(0);
3125 SDValue Hi = Op.getOperand(1);
3126 SDValue Shamt = Op.getOperand(2);
3127 EVT VT = Lo.getValueType();
3128
3129 // if Shamt-XLEN < 0: // Shamt < XLEN
3130 // Lo = Lo << Shamt
3131 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
3132 // else:
3133 // Lo = 0
3134 // Hi = Lo << (Shamt-XLEN)
3135
3136 SDValue Zero = DAG.getConstant(0, DL, VT);
3137 SDValue One = DAG.getConstant(1, DL, VT);
3138 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3139 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3140 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3141 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3142
3143 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
3144 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
3145 SDValue ShiftRightLo =
3146 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
3147 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
3148 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
3149 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
3150
3151 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3152
3153 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
3154 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3155
3156 SDValue Parts[2] = {Lo, Hi};
3157 return DAG.getMergeValues(Parts, DL);
3158 }
3159
3160 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
3161 bool IsSRA) const {
3162 SDLoc DL(Op);
3163 SDValue Lo = Op.getOperand(0);
3164 SDValue Hi = Op.getOperand(1);
3165 SDValue Shamt = Op.getOperand(2);
3166 EVT VT = Lo.getValueType();
3167
3168 // SRA expansion:
3169 // if Shamt-XLEN < 0: // Shamt < XLEN
3170 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3171 // Hi = Hi >>s Shamt
3172 // else:
3173 // Lo = Hi >>s (Shamt-XLEN);
3174 // Hi = Hi >>s (XLEN-1)
3175 //
3176 // SRL expansion:
3177 // if Shamt-XLEN < 0: // Shamt < XLEN
3178 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
3179 // Hi = Hi >>u Shamt
3180 // else:
3181 // Lo = Hi >>u (Shamt-XLEN);
3182 // Hi = 0;
3183
3184 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
3185
3186 SDValue Zero = DAG.getConstant(0, DL, VT);
3187 SDValue One = DAG.getConstant(1, DL, VT);
3188 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
3189 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
3190 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
3191 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
3192
3193 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
3194 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
3195 SDValue ShiftLeftHi =
3196 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
3197 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
3198 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
3199 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
3200 SDValue HiFalse =
3201 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
3202
3203 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
3204
3205 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
3206 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
3207
3208 SDValue Parts[2] = {Lo, Hi};
3209 return DAG.getMergeValues(Parts, DL);
3210 }
3211
3212 // Lower splats of i1 types to SETCC. For each mask vector type, we have a
3213 // legal equivalently-sized i8 type, so we can use that as a go-between.
3214 SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
3215 SelectionDAG &DAG) const {
3216 SDLoc DL(Op);
3217 MVT VT = Op.getSimpleValueType();
3218 SDValue SplatVal = Op.getOperand(0);
3219 // All-zeros or all-ones splats are handled specially.
3220 if (ISD::isConstantSplatVectorAllOnes(Op.getNode())) {
3221 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3222 return DAG.getNode(RISCVISD::VMSET_VL, DL, VT, VL);
3223 }
3224 if (ISD::isConstantSplatVectorAllZeros(Op.getNode())) {
3225 SDValue VL = getDefaultScalableVLOps(VT, DL, DAG, Subtarget).second;
3226 return DAG.getNode(RISCVISD::VMCLR_VL, DL, VT, VL);
3227 }
3228 MVT XLenVT = Subtarget.getXLenVT();
3229 assert(SplatVal.getValueType() == XLenVT &&
3230 "Unexpected type for i1 splat value");
3231 MVT InterVT = VT.changeVectorElementType(MVT::i8);
3232 SplatVal = DAG.getNode(ISD::AND, DL, XLenVT, SplatVal,
3233 DAG.getConstant(1, DL, XLenVT));
3234 SDValue LHS = DAG.getSplatVector(InterVT, DL, SplatVal);
3235 SDValue Zero = DAG.getConstant(0, DL, InterVT);
3236 return DAG.getSetCC(DL, VT, LHS, Zero, ISD::SETNE);
3237 }
3238
3239 // Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
3240 // illegal (currently only vXi64 RV32).
3241 // FIXME: We could also catch non-constant sign-extended i32 values and lower
3242 // them to SPLAT_VECTOR_I64
3243 SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
3244 SelectionDAG &DAG) const {
3245 SDLoc DL(Op);
3246 MVT VecVT = Op.getSimpleValueType();
3247 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
3248 "Unexpected SPLAT_VECTOR_PARTS lowering");
3249
3250 assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
3251 SDValue Lo = Op.getOperand(0);
3252 SDValue Hi = Op.getOperand(1);
3253
3254 if (VecVT.isFixedLengthVector()) {
3255 MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
3256 SDLoc DL(Op);
3257 SDValue Mask, VL;
3258 std::tie(Mask, VL) =
3259 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3260
3261 SDValue Res = splatPartsI64WithVL(DL, ContainerVT, Lo, Hi, VL, DAG);
3262 return convertFromScalableVector(VecVT, Res, DAG, Subtarget);
3263 }
3264
3265 if (isa<ConstantSDNode>(Lo) && isa<ConstantSDNode>(Hi)) {
3266 int32_t LoC = cast<ConstantSDNode>(Lo)->getSExtValue();
3267 int32_t HiC = cast<ConstantSDNode>(Hi)->getSExtValue();
3268 // If Hi constant is all the same sign bit as Lo, lower this as a custom
3269 // node in order to try and match RVV vector/scalar instructions.
3270 if ((LoC >> 31) == HiC)
3271 return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
3272 }
3273
3274 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
3275 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(0) == Lo &&
3276 isa<ConstantSDNode>(Hi.getOperand(1)) &&
3277 Hi.getConstantOperandVal(1) == 31)
3278 return DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, Lo);
3279
3280 // Fall back to use a stack store and stride x0 vector load. Use X0 as VL.
3281 return DAG.getNode(RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VecVT, Lo, Hi,
3282 DAG.getRegister(RISCV::X0, MVT::i64));
3283 }
3284
3285 // Custom-lower extensions from mask vectors by using a vselect either with 1
3286 // for zero/any-extension or -1 for sign-extension:
3287 // (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
3288 // Note that any-extension is lowered identically to zero-extension.
3289 SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
3290 int64_t ExtTrueVal) const {
3291 SDLoc DL(Op);
3292 MVT VecVT = Op.getSimpleValueType();
3293 SDValue Src = Op.getOperand(0);
3294 // Only custom-lower extensions from mask types
3295 assert(Src.getValueType().isVector() &&
3296 Src.getValueType().getVectorElementType() == MVT::i1);
3297
3298 MVT XLenVT = Subtarget.getXLenVT();
3299 SDValue SplatZero = DAG.getConstant(0, DL, XLenVT);
3300 SDValue SplatTrueVal = DAG.getConstant(ExtTrueVal, DL, XLenVT);
3301
3302 if (VecVT.isScalableVector()) {
3303 // Be careful not to introduce illegal scalar types at this stage, and be
3304 // careful also about splatting constants as on RV32, vXi64 SPLAT_VECTOR is
3305 // illegal and must be expanded. Since we know that the constants are
3306 // sign-extended 32-bit values, we use SPLAT_VECTOR_I64 directly.
3307 bool IsRV32E64 =
3308 !Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64;
3309
3310 if (!IsRV32E64) {
3311 SplatZero = DAG.getSplatVector(VecVT, DL, SplatZero);
3312 SplatTrueVal = DAG.getSplatVector(VecVT, DL, SplatTrueVal);
3313 } else {
3314 SplatZero = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatZero);
3315 SplatTrueVal =
3316 DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VecVT, SplatTrueVal);
3317 }
3318
3319 return DAG.getNode(ISD::VSELECT, DL, VecVT, Src, SplatTrueVal, SplatZero);
3320 }
3321
3322 MVT ContainerVT = getContainerForFixedLengthVector(VecVT);
3323 MVT I1ContainerVT =
3324 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3325
3326 SDValue CC = convertToScalableVector(I1ContainerVT, Src, DAG, Subtarget);
3327
3328 SDValue Mask, VL;
3329 std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3330
3331 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero, VL);
3332 SplatTrueVal =
3333 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatTrueVal, VL);
3334 SDValue Select = DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC,
3335 SplatTrueVal, SplatZero, VL);
3336
3337 return convertFromScalableVector(VecVT, Select, DAG, Subtarget);
3338 }
3339
3340 SDValue RISCVTargetLowering::lowerFixedLengthVectorExtendToRVV(
3341 SDValue Op, SelectionDAG &DAG, unsigned ExtendOpc) const {
3342 MVT ExtVT = Op.getSimpleValueType();
3343 // Only custom-lower extensions from fixed-length vector types.
3344 if (!ExtVT.isFixedLengthVector())
3345 return Op;
3346 MVT VT = Op.getOperand(0).getSimpleValueType();
3347 // Grab the canonical container type for the extended type. Infer the smaller
3348 // type from that to ensure the same number of vector elements, as we know
3349 // the LMUL will be sufficient to hold the smaller type.
3350 MVT ContainerExtVT = getContainerForFixedLengthVector(ExtVT);
3351 // Get the extended container type manually to ensure the same number of
3352 // vector elements between source and dest.
3353 MVT ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
3354 ContainerExtVT.getVectorElementCount());
3355
3356 SDValue Op1 =
3357 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
3358
3359 SDLoc DL(Op);
3360 SDValue Mask, VL;
3361 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
3362
3363 SDValue Ext = DAG.getNode(ExtendOpc, DL, ContainerExtVT, Op1, Mask, VL);
3364
3365 return convertFromScalableVector(ExtVT, Ext, DAG, Subtarget);
3366 }
3367
3368 // Custom-lower truncations from vectors to mask vectors by using a mask and a
3369 // setcc operation:
3370 // (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
3371 SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
3372 SelectionDAG &DAG) const {
3373 SDLoc DL(Op);
3374 EVT MaskVT = Op.getValueType();
3375 // Only expect to custom-lower truncations to mask types
3376 assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
3377 "Unexpected type for vector mask lowering");
3378 SDValue Src = Op.getOperand(0);
3379 MVT VecVT = Src.getSimpleValueType();
3380
3381 // If this is a fixed vector, we need to convert it to a scalable vector.
3382 MVT ContainerVT = VecVT;
3383 if (VecVT.isFixedLengthVector()) {
3384 ContainerVT = getContainerForFixedLengthVector(VecVT);
3385 Src = convertToScalableVector(ContainerVT, Src, DAG, Subtarget);
3386 }
3387
3388 SDValue SplatOne = DAG.getConstant(1, DL, Subtarget.getXLenVT());
3389 SDValue SplatZero = DAG.getConstant(0, DL, Subtarget.getXLenVT());
3390
3391 SplatOne = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatOne);
3392 SplatZero = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT, SplatZero);
3393
3394 if (VecVT.isScalableVector()) {
3395 SDValue Trunc = DAG.getNode(ISD::AND, DL, VecVT, Src, SplatOne);
3396 return DAG.getSetCC(DL, MaskVT, Trunc, SplatZero, ISD::SETNE);
3397 }
3398
3399 SDValue Mask, VL;
3400 std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3401
3402 MVT MaskContainerVT = ContainerVT.changeVectorElementType(MVT::i1);
3403 SDValue Trunc =
3404 DAG.getNode(RISCVISD::AND_VL, DL, ContainerVT, Src, SplatOne, Mask, VL);
3405 Trunc = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskContainerVT, Trunc, SplatZero,
3406 DAG.getCondCode(ISD::SETNE), Mask, VL);
3407 return convertFromScalableVector(MaskVT, Trunc, DAG, Subtarget);
3408 }
3409
3410 // Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
3411 // first position of a vector, and that vector is slid up to the insert index.
3412 // By limiting the active vector length to index+1 and merging with the
3413 // original vector (with an undisturbed tail policy for elements >= VL), we
3414 // achieve the desired result of leaving all elements untouched except the one
3415 // at VL-1, which is replaced with the desired value.
3416 SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
3417 SelectionDAG &DAG) const {
3418 SDLoc DL(Op);
3419 MVT VecVT = Op.getSimpleValueType();
3420 SDValue Vec = Op.getOperand(0);
3421 SDValue Val = Op.getOperand(1);
3422 SDValue Idx = Op.getOperand(2);
3423
3424 if (VecVT.getVectorElementType() == MVT::i1) {
3425 // FIXME: For now we just promote to an i8 vector and insert into that,
3426 // but this is probably not optimal.
3427 MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
3428 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
3429 Vec = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, WideVT, Vec, Val, Idx);
3430 return DAG.getNode(ISD::TRUNCATE, DL, VecVT, Vec);
3431 }
3432
3433 MVT ContainerVT = VecVT;
3434 // If the operand is a fixed-length vector, convert to a scalable one.
3435 if (VecVT.isFixedLengthVector()) {
3436 ContainerVT = getContainerForFixedLengthVector(VecVT);
3437 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3438 }
3439
3440 MVT XLenVT = Subtarget.getXLenVT();
3441
3442 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3443 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
3444 // Even i64-element vectors on RV32 can be lowered without scalar
3445 // legalization if the most-significant 32 bits of the value are not affected
3446 // by the sign-extension of the lower 32 bits.
3447 // TODO: We could also catch sign extensions of a 32-bit value.
3448 if (!IsLegalInsert && isa<ConstantSDNode>(Val)) {
3449 const auto *CVal = cast<ConstantSDNode>(Val);
3450 if (isInt<32>(CVal->getSExtValue())) {
3451 IsLegalInsert = true;
3452 Val = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
3453 }
3454 }
3455
3456 SDValue Mask, VL;
3457 std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3458
3459 SDValue ValInVec;
3460
3461 if (IsLegalInsert) {
3462 unsigned Opc =
3463 VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
3464 if (isNullConstant(Idx)) {
3465 Vec = DAG.getNode(Opc, DL, ContainerVT, Vec, Val, VL);
3466 if (!VecVT.isFixedLengthVector())
3467 return Vec;
3468 return convertFromScalableVector(VecVT, Vec, DAG, Subtarget);
3469 }
3470 ValInVec =
3471 DAG.getNode(Opc, DL, ContainerVT, DAG.getUNDEF(ContainerVT), Val, VL);
3472 } else {
3473 // On RV32, i64-element vectors must be specially handled to place the
3474 // value at element 0, by using two vslide1up instructions in sequence on
3475 // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
3476 // this.
3477 SDValue One = DAG.getConstant(1, DL, XLenVT);
3478 SDValue ValLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, Zero);
3479 SDValue ValHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Val, One);
3480 MVT I32ContainerVT =
3481 MVT::getVectorVT(MVT::i32, ContainerVT.getVectorElementCount() * 2);
3482 SDValue I32Mask =
3483 getDefaultScalableVLOps(I32ContainerVT, DL, DAG, Subtarget).first;
3484 // Limit the active VL to two.
3485 SDValue InsertI64VL = DAG.getConstant(2, DL, XLenVT);
3486 // Note: We can't pass a UNDEF to the first VSLIDE1UP_VL since an untied
3487 // undef doesn't obey the earlyclobber constraint. Just splat a zero value.
3488 ValInVec = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, I32ContainerVT, Zero,
3489 InsertI64VL);
3490 // First slide in the hi value, then the lo in underneath it.
3491 ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
3492 ValHi, I32Mask, InsertI64VL);
3493 ValInVec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32ContainerVT, ValInVec,
3494 ValLo, I32Mask, InsertI64VL);
3495 // Bitcast back to the right container type.
3496 ValInVec = DAG.getBitcast(ContainerVT, ValInVec);
3497 }
3498
3499 // Now that the value is in a vector, slide it into position.
3500 SDValue InsertVL =
3501 DAG.getNode(ISD::ADD, DL, XLenVT, Idx, DAG.getConstant(1, DL, XLenVT));
3502 SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
3503 ValInVec, Idx, Mask, InsertVL);
3504 if (!VecVT.isFixedLengthVector())
3505 return Slideup;
3506 return convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
3507 }
3508
3509 // Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
3510 // extract the first element: (extractelt (slidedown vec, idx), 0). For integer
3511 // types this is done using VMV_X_S to allow us to glean information about the
3512 // sign bits of the result.
3513 SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
3514 SelectionDAG &DAG) const {
3515 SDLoc DL(Op);
3516 SDValue Idx = Op.getOperand(1);
3517 SDValue Vec = Op.getOperand(0);
3518 EVT EltVT = Op.getValueType();
3519 MVT VecVT = Vec.getSimpleValueType();
3520 MVT XLenVT = Subtarget.getXLenVT();
3521
3522 if (VecVT.getVectorElementType() == MVT::i1) {
3523 // FIXME: For now we just promote to an i8 vector and extract from that,
3524 // but this is probably not optimal.
3525 MVT WideVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorElementCount());
3526 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, WideVT, Vec);
3527 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec, Idx);
3528 }
3529
3530 // If this is a fixed vector, we need to convert it to a scalable vector.
3531 MVT ContainerVT = VecVT;
3532 if (VecVT.isFixedLengthVector()) {
3533 ContainerVT = getContainerForFixedLengthVector(VecVT);
3534 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3535 }
3536
3537 // If the index is 0, the vector is already in the right position.
3538 if (!isNullConstant(Idx)) {
3539 // Use a VL of 1 to avoid processing more elements than we need.
3540 SDValue VL = DAG.getConstant(1, DL, XLenVT);
3541 MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
3542 SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
3543 Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
3544 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
3545 }
3546
3547 if (!EltVT.isInteger()) {
3548 // Floating-point extracts are handled in TableGen.
3549 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
3550 DAG.getConstant(0, DL, XLenVT));
3551 }
3552
3553 SDValue Elt0 = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
3554 return DAG.getNode(ISD::TRUNCATE, DL, EltVT, Elt0);
3555 }
3556
3557 // Some RVV intrinsics may claim that they want an integer operand to be
3558 // promoted or expanded.
3559 static SDValue lowerVectorIntrinsicSplats(SDValue Op, SelectionDAG &DAG,
3560 const RISCVSubtarget &Subtarget) {
3561 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3562 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3563 "Unexpected opcode");
3564
3565 if (!Subtarget.hasStdExtV())
3566 return SDValue();
3567
3568 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3569 unsigned IntNo = Op.getConstantOperandVal(HasChain ? 1 : 0);
3570 SDLoc DL(Op);
3571
3572 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3573 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntNo);
3574 if (!II || !II->SplatOperand)
3575 return SDValue();
3576
3577 unsigned SplatOp = II->SplatOperand + HasChain;
3578 assert(SplatOp < Op.getNumOperands());
3579
3580 SmallVector<SDValue, 8> Operands(Op->op_begin(), Op->op_end());
3581 SDValue &ScalarOp = Operands[SplatOp];
3582 MVT OpVT = ScalarOp.getSimpleValueType();
3583 MVT XLenVT = Subtarget.getXLenVT();
3584
3585 // If this isn't a scalar, or its type is XLenVT we're done.
3586 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
3587 return SDValue();
3588
3589 // Simplest case is that the operand needs to be promoted to XLenVT.
3590 if (OpVT.bitsLT(XLenVT)) {
3591 // If the operand is a constant, sign extend to increase our chances
3592 // of being able to use a .vi instruction. ANY_EXTEND would become a
3593 // a zero extend and the simm5 check in isel would fail.
3594 // FIXME: Should we ignore the upper bits in isel instead?
3595 unsigned ExtOpc =
3596 isa<ConstantSDNode>(ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
3597 ScalarOp = DAG.getNode(ExtOpc, DL, XLenVT, ScalarOp);
3598 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
3599 }
3600
3601 // Use the previous operand to get the vXi64 VT. The result might be a mask
3602 // VT for compares. Using the previous operand assumes that the previous
3603 // operand will never have a smaller element size than a scalar operand and
3604 // that a widening operation never uses SEW=64.
3605 // NOTE: If this fails the below assert, we can probably just find the
3606 // element count from any operand or result and use it to construct the VT.
3607 assert(II->SplatOperand > 1 && "Unexpected splat operand!");
3608 MVT VT = Op.getOperand(SplatOp - 1).getSimpleValueType();
3609
3610 // The more complex case is when the scalar is larger than XLenVT.
3611 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
3612 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
3613
3614 // If this is a sign-extended 32-bit constant, we can truncate it and rely
3615 // on the instruction to sign-extend since SEW>XLEN.
3616 if (auto *CVal = dyn_cast<ConstantSDNode>(ScalarOp)) {
3617 if (isInt<32>(CVal->getSExtValue())) {
3618 ScalarOp = DAG.getConstant(CVal->getSExtValue(), DL, MVT::i32);
3619 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
3620 }
3621 }
3622
3623 // We need to convert the scalar to a splat vector.
3624 // FIXME: Can we implicitly truncate the scalar if it is known to
3625 // be sign extended?
3626 // VL should be the last operand.
3627 SDValue VL = Op.getOperand(Op.getNumOperands() - 1);
3628 assert(VL.getValueType() == XLenVT);
3629 ScalarOp = splatSplitI64WithVL(DL, VT, ScalarOp, VL, DAG);
3630 return DAG.getNode(Op->getOpcode(), DL, Op->getVTList(), Operands);
3631 }
3632
3633 SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
3634 SelectionDAG &DAG) const {
3635 unsigned IntNo = Op.getConstantOperandVal(0);
3636 SDLoc DL(Op);
3637 MVT XLenVT = Subtarget.getXLenVT();
3638
3639 switch (IntNo) {
3640 default:
3641 break; // Don't custom lower most intrinsics.
3642 case Intrinsic::thread_pointer: {
3643 EVT PtrVT = getPointerTy(DAG.getDataLayout());
3644 return DAG.getRegister(RISCV::X4, PtrVT);
3645 }
3646 case Intrinsic::riscv_orc_b:
3647 // Lower to the GORCI encoding for orc.b.
3648 return DAG.getNode(RISCVISD::GORC, DL, XLenVT, Op.getOperand(1),
3649 DAG.getConstant(7, DL, XLenVT));
3650 case Intrinsic::riscv_grev:
3651 case Intrinsic::riscv_gorc: {
3652 unsigned Opc =
3653 IntNo == Intrinsic::riscv_grev ? RISCVISD::GREV : RISCVISD::GORC;
3654 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
3655 }
3656 case Intrinsic::riscv_shfl:
3657 case Intrinsic::riscv_unshfl: {
3658 unsigned Opc =
3659 IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
3660 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
3661 }
3662 case Intrinsic::riscv_bcompress:
3663 case Intrinsic::riscv_bdecompress: {
3664 unsigned Opc = IntNo == Intrinsic::riscv_bcompress ? RISCVISD::BCOMPRESS
3665 : RISCVISD::BDECOMPRESS;
3666 return DAG.getNode(Opc, DL, XLenVT, Op.getOperand(1), Op.getOperand(2));
3667 }
3668 case Intrinsic::riscv_vmv_x_s:
3669 assert(Op.getValueType() == XLenVT && "Unexpected VT!");
3670 return DAG.getNode(RISCVISD::VMV_X_S, DL, Op.getValueType(),
3671 Op.getOperand(1));
3672 case Intrinsic::riscv_vmv_v_x:
3673 return lowerScalarSplat(Op.getOperand(1), Op.getOperand(2),
3674 Op.getSimpleValueType(), DL, DAG, Subtarget);
3675 case Intrinsic::riscv_vfmv_v_f:
3676 return DAG.getNode(RISCVISD::VFMV_V_F_VL, DL, Op.getValueType(),
3677 Op.getOperand(1), Op.getOperand(2));
3678 case Intrinsic::riscv_vmv_s_x: {
3679 SDValue Scalar = Op.getOperand(2);
3680
3681 if (Scalar.getValueType().bitsLE(XLenVT)) {
3682 Scalar = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, Scalar);
3683 return DAG.getNode(RISCVISD::VMV_S_X_VL, DL, Op.getValueType(),
3684 Op.getOperand(1), Scalar, Op.getOperand(3));
3685 }
3686
3687 assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
3688
3689 // This is an i64 value that lives in two scalar registers. We have to
3690 // insert this in a convoluted way. First we build vXi64 splat containing
3691 // the/ two values that we assemble using some bit math. Next we'll use
3692 // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
3693 // to merge element 0 from our splat into the source vector.
3694 // FIXME: This is probably not the best way to do this, but it is
3695 // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
3696 // point.
3697 // sw lo, (a0)
3698 // sw hi, 4(a0)
3699 // vlse vX, (a0)
3700 //
3701 // vid.v vVid
3702 // vmseq.vx mMask, vVid, 0
3703 // vmerge.vvm vDest, vSrc, vVal, mMask
3704 MVT VT = Op.getSimpleValueType();
3705 SDValue Vec = Op.getOperand(1);
3706 SDValue VL = Op.getOperand(3);
3707
3708 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Scalar, VL, DAG);
3709 SDValue SplattedIdx = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
3710 DAG.getConstant(0, DL, MVT::i32), VL);
3711
3712 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorElementCount());
3713 SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
3714 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
3715 SDValue SelectCond =
3716 DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, VID, SplattedIdx,
3717 DAG.getCondCode(ISD::SETEQ), Mask, VL);
3718 return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, SelectCond, SplattedVal,
3719 Vec, VL);
3720 }
3721 case Intrinsic::riscv_vslide1up:
3722 case Intrinsic::riscv_vslide1down:
3723 case Intrinsic::riscv_vslide1up_mask:
3724 case Intrinsic::riscv_vslide1down_mask: {
3725 // We need to special case these when the scalar is larger than XLen.
3726 unsigned NumOps = Op.getNumOperands();
3727 bool IsMasked = NumOps == 6;
3728 unsigned OpOffset = IsMasked ? 1 : 0;
3729 SDValue Scalar = Op.getOperand(2 + OpOffset);
3730 if (Scalar.getValueType().bitsLE(XLenVT))
3731 break;
3732
3733 // Splatting a sign extended constant is fine.
3734 if (auto *CVal = dyn_cast<ConstantSDNode>(Scalar))
3735 if (isInt<32>(CVal->getSExtValue()))
3736 break;
3737
3738 MVT VT = Op.getSimpleValueType();
3739 assert(VT.getVectorElementType() == MVT::i64 &&
3740 Scalar.getValueType() == MVT::i64 && "Unexpected VTs");
3741
3742 // Convert the vector source to the equivalent nxvXi32 vector.
3743 MVT I32VT = MVT::getVectorVT(MVT::i32, VT.getVectorElementCount() * 2);
3744 SDValue Vec = DAG.getBitcast(I32VT, Op.getOperand(1 + OpOffset));
3745
3746 SDValue ScalarLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
3747 DAG.getConstant(0, DL, XLenVT));
3748 SDValue ScalarHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32, Scalar,
3749 DAG.getConstant(1, DL, XLenVT));
3750
3751 // Double the VL since we halved SEW.
3752 SDValue VL = Op.getOperand(NumOps - 1);
3753 SDValue I32VL =
3754 DAG.getNode(ISD::SHL, DL, XLenVT, VL, DAG.getConstant(1, DL, XLenVT));
3755
3756 MVT I32MaskVT = MVT::getVectorVT(MVT::i1, I32VT.getVectorElementCount());
3757 SDValue I32Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, I32MaskVT, VL);
3758
3759 // Shift the two scalar parts in using SEW=32 slide1up/slide1down
3760 // instructions.
3761 if (IntNo == Intrinsic::riscv_vslide1up ||
3762 IntNo == Intrinsic::riscv_vslide1up_mask) {
3763 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarHi,
3764 I32Mask, I32VL);
3765 Vec = DAG.getNode(RISCVISD::VSLIDE1UP_VL, DL, I32VT, Vec, ScalarLo,
3766 I32Mask, I32VL);
3767 } else {
3768 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarLo,
3769 I32Mask, I32VL);
3770 Vec = DAG.getNode(RISCVISD::VSLIDE1DOWN_VL, DL, I32VT, Vec, ScalarHi,
3771 I32Mask, I32VL);
3772 }
3773
3774 // Convert back to nxvXi64.
3775 Vec = DAG.getBitcast(VT, Vec);
3776
3777 if (!IsMasked)
3778 return Vec;
3779
3780 // Apply mask after the operation.
3781 SDValue Mask = Op.getOperand(NumOps - 2);
3782 SDValue MaskedOff = Op.getOperand(1);
3783 return DAG.getNode(RISCVISD::VSELECT_VL, DL, VT, Mask, Vec, MaskedOff, VL);
3784 }
3785 }
3786
3787 return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
3788 }
3789
3790 SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
3791 SelectionDAG &DAG) const {
3792 return lowerVectorIntrinsicSplats(Op, DAG, Subtarget);
3793 }
3794
3795 static MVT getLMUL1VT(MVT VT) {
3796 assert(VT.getVectorElementType().getSizeInBits() <= 64 &&
3797 "Unexpected vector MVT");
3798 return MVT::getScalableVectorVT(
3799 VT.getVectorElementType(),
3800 RISCV::RVVBitsPerBlock / VT.getVectorElementType().getSizeInBits());
3801 }
3802
3803 static unsigned getRVVReductionOp(unsigned ISDOpcode) {
3804 switch (ISDOpcode) {
3805 default:
3806 llvm_unreachable("Unhandled reduction");
3807 case ISD::VECREDUCE_ADD:
3808 return RISCVISD::VECREDUCE_ADD_VL;
3809 case ISD::VECREDUCE_UMAX:
3810 return RISCVISD::VECREDUCE_UMAX_VL;
3811 case ISD::VECREDUCE_SMAX:
3812 return RISCVISD::VECREDUCE_SMAX_VL;
3813 case ISD::VECREDUCE_UMIN:
3814 return RISCVISD::VECREDUCE_UMIN_VL;
3815 case ISD::VECREDUCE_SMIN:
3816 return RISCVISD::VECREDUCE_SMIN_VL;
3817 case ISD::VECREDUCE_AND:
3818 return RISCVISD::VECREDUCE_AND_VL;
3819 case ISD::VECREDUCE_OR:
3820 return RISCVISD::VECREDUCE_OR_VL;
3821 case ISD::VECREDUCE_XOR:
3822 return RISCVISD::VECREDUCE_XOR_VL;
3823 }
3824 }
3825
3826 SDValue RISCVTargetLowering::lowerVectorMaskVECREDUCE(SDValue Op,
3827 SelectionDAG &DAG) const {
3828 SDLoc DL(Op);
3829 SDValue Vec = Op.getOperand(0);
3830 MVT VecVT = Vec.getSimpleValueType();
3831 assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
3832 Op.getOpcode() == ISD::VECREDUCE_OR ||
3833 Op.getOpcode() == ISD::VECREDUCE_XOR) &&
3834 "Unexpected reduction lowering");
3835
3836 MVT XLenVT = Subtarget.getXLenVT();
3837 assert(Op.getValueType() == XLenVT &&
3838 "Expected reduction output to be legalized to XLenVT");
3839
3840 MVT ContainerVT = VecVT;
3841 if (VecVT.isFixedLengthVector()) {
3842 ContainerVT = getContainerForFixedLengthVector(VecVT);
3843 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3844 }
3845
3846 SDValue Mask, VL;
3847 std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3848 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
3849
3850 switch (Op.getOpcode()) {
3851 default:
3852 llvm_unreachable("Unhandled reduction");
3853 case ISD::VECREDUCE_AND:
3854 // vpopc ~x == 0
3855 Vec = DAG.getNode(RISCVISD::VMXOR_VL, DL, ContainerVT, Vec, Mask, VL);
3856 Vec = DAG.getNode(RISCVISD::VPOPC_VL, DL, XLenVT, Vec, Mask, VL);
3857 return DAG.getSetCC(DL, XLenVT, Vec, Zero, ISD::SETEQ);
3858 case ISD::VECREDUCE_OR:
3859 // vpopc x != 0
3860 Vec = DAG.getNode(RISCVISD::VPOPC_VL, DL, XLenVT, Vec, Mask, VL);
3861 return DAG.getSetCC(DL, XLenVT, Vec, Zero, ISD::SETNE);
3862 case ISD::VECREDUCE_XOR: {
3863 // ((vpopc x) & 1) != 0
3864 SDValue One = DAG.getConstant(1, DL, XLenVT);
3865 Vec = DAG.getNode(RISCVISD::VPOPC_VL, DL, XLenVT, Vec, Mask, VL);
3866 Vec = DAG.getNode(ISD::AND, DL, XLenVT, Vec, One);
3867 return DAG.getSetCC(DL, XLenVT, Vec, Zero, ISD::SETNE);
3868 }
3869 }
3870 }
3871
3872 SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
3873 SelectionDAG &DAG) const {
3874 SDLoc DL(Op);
3875 SDValue Vec = Op.getOperand(0);
3876 EVT VecEVT = Vec.getValueType();
3877
3878 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Op.getOpcode());
3879
3880 // Due to ordering in legalize types we may have a vector type that needs to
3881 // be split. Do that manually so we can get down to a legal type.
3882 while (getTypeAction(*DAG.getContext(), VecEVT) ==
3883 TargetLowering::TypeSplitVector) {
3884 SDValue Lo, Hi;
3885 std::tie(Lo, Hi) = DAG.SplitVector(Vec, DL);
3886 VecEVT = Lo.getValueType();
3887 Vec = DAG.getNode(BaseOpc, DL, VecEVT, Lo, Hi);
3888 }
3889
3890 // TODO: The type may need to be widened rather than split. Or widened before
3891 // it can be split.
3892 if (!isTypeLegal(VecEVT))
3893 return SDValue();
3894
3895 MVT VecVT = VecEVT.getSimpleVT();
3896 MVT VecEltVT = VecVT.getVectorElementType();
3897 unsigned RVVOpcode = getRVVReductionOp(Op.getOpcode());
3898
3899 MVT ContainerVT = VecVT;
3900 if (VecVT.isFixedLengthVector()) {
3901 ContainerVT = getContainerForFixedLengthVector(VecVT);
3902 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
3903 }
3904
3905 MVT M1VT = getLMUL1VT(ContainerVT);
3906
3907 SDValue Mask, VL;
3908 std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3909
3910 // FIXME: This is a VLMAX splat which might be too large and can prevent
3911 // vsetvli removal.
3912 SDValue NeutralElem =
3913 DAG.getNeutralElement(BaseOpc, DL, VecEltVT, SDNodeFlags());
3914 SDValue IdentitySplat = DAG.getSplatVector(M1VT, DL, NeutralElem);
3915 SDValue Reduction =
3916 DAG.getNode(RVVOpcode, DL, M1VT, Vec, IdentitySplat, Mask, VL);
3917 SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
3918 DAG.getConstant(0, DL, Subtarget.getXLenVT()));
3919 return DAG.getSExtOrTrunc(Elt0, DL, Op.getValueType());
3920 }
3921
3922 // Given a reduction op, this function returns the matching reduction opcode,
3923 // the vector SDValue and the scalar SDValue required to lower this to a
3924 // RISCVISD node.
3925 static std::tuple<unsigned, SDValue, SDValue>
3926 getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT) {
3927 SDLoc DL(Op);
3928 auto Flags = Op->getFlags();
3929 unsigned Opcode = Op.getOpcode();
3930 unsigned BaseOpcode = ISD::getVecReduceBaseOpcode(Opcode);
3931 switch (Opcode) {
3932 default:
3933 llvm_unreachable("Unhandled reduction");
3934 case ISD::VECREDUCE_FADD:
3935 return std::make_tuple(RISCVISD::VECREDUCE_FADD_VL, Op.getOperand(0),
3936 DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
3937 case ISD::VECREDUCE_SEQ_FADD:
3938 return std::make_tuple(RISCVISD::VECREDUCE_SEQ_FADD_VL, Op.getOperand(1),
3939 Op.getOperand(0));
3940 case ISD::VECREDUCE_FMIN:
3941 return std::make_tuple(RISCVISD::VECREDUCE_FMIN_VL, Op.getOperand(0),
3942 DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
3943 case ISD::VECREDUCE_FMAX:
3944 return std::make_tuple(RISCVISD::VECREDUCE_FMAX_VL, Op.getOperand(0),
3945 DAG.getNeutralElement(BaseOpcode, DL, EltVT, Flags));
3946 }
3947 }
3948
3949 SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
3950 SelectionDAG &DAG) const {
3951 SDLoc DL(Op);
3952 MVT VecEltVT = Op.getSimpleValueType();
3953
3954 unsigned RVVOpcode;
3955 SDValue VectorVal, ScalarVal;
3956 std::tie(RVVOpcode, VectorVal, ScalarVal) =
3957 getRVVFPReductionOpAndOperands(Op, DAG, VecEltVT);
3958 MVT VecVT = VectorVal.getSimpleValueType();
3959
3960 MVT ContainerVT = VecVT;
3961 if (VecVT.isFixedLengthVector()) {
3962 ContainerVT = getContainerForFixedLengthVector(VecVT);
3963 VectorVal = convertToScalableVector(ContainerVT, VectorVal, DAG, Subtarget);
3964 }
3965
3966 MVT M1VT = getLMUL1VT(VectorVal.getSimpleValueType());
3967
3968 SDValue Mask, VL;
3969 std::tie(Mask, VL) = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
3970
3971 // FIXME: This is a VLMAX splat which might be too large and can prevent
3972 // vsetvli removal.
3973 SDValue ScalarSplat = DAG.getSplatVector(M1VT, DL, ScalarVal);
3974 SDValue Reduction =
3975 DAG.getNode(RVVOpcode, DL, M1VT, VectorVal, ScalarSplat, Mask, VL);
3976 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VecEltVT, Reduction,
3977 DAG.getConstant(0, DL, Subtarget.getXLenVT()));
3978 }
3979
3980 SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
3981 SelectionDAG &DAG) const {
3982 SDValue Vec = Op.getOperand(0);
3983 SDValue SubVec = Op.getOperand(1);
3984 MVT VecVT = Vec.getSimpleValueType();
3985 MVT SubVecVT = SubVec.getSimpleValueType();
3986
3987 SDLoc DL(Op);
3988 MVT XLenVT = Subtarget.getXLenVT();
3989 unsigned OrigIdx = Op.getConstantOperandVal(2);
3990 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
3991
3992 // We don't have the ability to slide mask vectors up indexed by their i1
3993 // elements; the smallest we can do is i8. Often we are able to bitcast to
3994 // equivalent i8 vectors. Note that when inserting a fixed-length vector
3995 // into a scalable one, we might not necessarily have enough scalable
3996 // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
3997 if (SubVecVT.getVectorElementType() == MVT::i1 &&
3998 (OrigIdx != 0 || !Vec.isUndef())) {
3999 if (VecVT.getVectorMinNumElements() >= 8 &&
4000 SubVecVT.getVectorMinNumElements() >= 8) {
4001 assert(OrigIdx % 8 == 0 && "Invalid index");
4002 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
4003 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
4004 "Unexpected mask vector lowering");
4005 OrigIdx /= 8;
4006 SubVecVT =
4007 MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
4008 SubVecVT.isScalableVector());
4009 VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
4010 VecVT.isScalableVector());
4011 Vec = DAG.getBitcast(VecVT, Vec);
4012 SubVec = DAG.getBitcast(SubVecVT, SubVec);
4013 } else {
4014 // We can't slide this mask vector up indexed by its i1 elements.
4015 // This poses a problem when we wish to insert a scalable vector which
4016 // can't be re-expressed as a larger type. Just choose the slow path and
4017 // extend to a larger type, then truncate back down.
4018 MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
4019 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
4020 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
4021 SubVec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtSubVecVT, SubVec);
4022 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ExtVecVT, Vec, SubVec,
4023 Op.getOperand(2));
4024 SDValue SplatZero = DAG.getConstant(0, DL, ExtVecVT);
4025 return DAG.getSetCC(DL, VecVT, Vec, SplatZero, ISD::SETNE);
4026 }
4027 }
4028
4029 // If the subvector vector is a fixed-length type, we cannot use subregister
4030 // manipulation to simplify the codegen; we don't know which register of a
4031 // LMUL group contains the specific subvector as we only know the minimum
4032 // register size. Therefore we must slide the vector group up the full
4033 // amount.
4034 if (SubVecVT.isFixedLengthVector()) {
4035 if (OrigIdx == 0 && Vec.isUndef())
4036 return Op;
4037 MVT ContainerVT = VecVT;
4038 if (VecVT.isFixedLengthVector()) {
4039 ContainerVT = getContainerForFixedLengthVector(VecVT);
4040 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4041 }
4042 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ContainerVT,
4043 DAG.getUNDEF(ContainerVT), SubVec,
4044 DAG.getConstant(0, DL, XLenVT));
4045 SDValue Mask =
4046 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
4047 // Set the vector length to only the number of elements we care about. Note
4048 // that for slideup this includes the offset.
4049 SDValue VL =
4050 DAG.getConstant(OrigIdx + SubVecVT.getVectorNumElements(), DL, XLenVT);
4051 SDValue SlideupAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
4052 SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, ContainerVT, Vec,
4053 SubVec, SlideupAmt, Mask, VL);
4054 if (VecVT.isFixedLengthVector())
4055 Slideup = convertFromScalableVector(VecVT, Slideup, DAG, Subtarget);
4056 return DAG.getBitcast(Op.getValueType(), Slideup);
4057 }
4058
4059 unsigned SubRegIdx, RemIdx;
4060 std::tie(SubRegIdx, RemIdx) =
4061 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
4062 VecVT, SubVecVT, OrigIdx, TRI);
4063
4064 RISCVII::VLMUL SubVecLMUL = RISCVTargetLowering::getLMUL(SubVecVT);
4065 bool IsSubVecPartReg = SubVecLMUL == RISCVII::VLMUL::LMUL_F2 ||
4066 SubVecLMUL == RISCVII::VLMUL::LMUL_F4 ||
4067 SubVecLMUL == RISCVII::VLMUL::LMUL_F8;
4068
4069 // 1. If the Idx has been completely eliminated and this subvector's size is
4070 // a vector register or a multiple thereof, or the surrounding elements are
4071 // undef, then this is a subvector insert which naturally aligns to a vector
4072 // register. These can easily be handled using subregister manipulation.
4073 // 2. If the subvector is smaller than a vector register, then the insertion
4074 // must preserve the undisturbed elements of the register. We do this by
4075 // lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1 vector type
4076 // (which resolves to a subregister copy), performing a VSLIDEUP to place the
4077 // subvector within the vector register, and an INSERT_SUBVECTOR of that
4078 // LMUL=1 type back into the larger vector (resolving to another subregister
4079 // operation). See below for how our VSLIDEUP works. We go via a LMUL=1 type
4080 // to avoid allocating a large register group to hold our subvector.
4081 if (RemIdx == 0 && (!IsSubVecPartReg || Vec.isUndef()))
4082 return Op;
4083
4084 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
4085 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
4086 // (in our case undisturbed). This means we can set up a subvector insertion
4087 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
4088 // size of the subvector.
4089 MVT InterSubVT = VecVT;
4090 SDValue AlignedExtract = Vec;
4091 unsigned AlignedIdx = OrigIdx - RemIdx;
4092 if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
4093 InterSubVT = getLMUL1VT(VecVT);
4094 // Extract a subvector equal to the nearest full vector register type. This
4095 // should resolve to a EXTRACT_SUBREG instruction.
4096 AlignedExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
4097 DAG.getConstant(AlignedIdx, DL, XLenVT));
4098 }
4099
4100 SDValue SlideupAmt = DAG.getConstant(RemIdx, DL, XLenVT);
4101 // For scalable vectors this must be further multiplied by vscale.
4102 SlideupAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlideupAmt);
4103
4104 SDValue Mask, VL;
4105 std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
4106
4107 // Construct the vector length corresponding to RemIdx + length(SubVecVT).
4108 VL = DAG.getConstant(SubVecVT.getVectorMinNumElements(), DL, XLenVT);
4109 VL = DAG.getNode(ISD::VSCALE, DL, XLenVT, VL);
4110 VL = DAG.getNode(ISD::ADD, DL, XLenVT, SlideupAmt, VL);
4111
4112 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InterSubVT,
4113 DAG.getUNDEF(InterSubVT), SubVec,
4114 DAG.getConstant(0, DL, XLenVT));
4115
4116 SDValue Slideup = DAG.getNode(RISCVISD::VSLIDEUP_VL, DL, InterSubVT,
4117 AlignedExtract, SubVec, SlideupAmt, Mask, VL);
4118
4119 // If required, insert this subvector back into the correct vector register.
4120 // This should resolve to an INSERT_SUBREG instruction.
4121 if (VecVT.bitsGT(InterSubVT))
4122 Slideup = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, Vec, Slideup,
4123 DAG.getConstant(AlignedIdx, DL, XLenVT));
4124
4125 // We might have bitcast from a mask type: cast back to the original type if
4126 // required.
4127 return DAG.getBitcast(Op.getSimpleValueType(), Slideup);
4128 }
4129
4130 SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
4131 SelectionDAG &DAG) const {
4132 SDValue Vec = Op.getOperand(0);
4133 MVT SubVecVT = Op.getSimpleValueType();
4134 MVT VecVT = Vec.getSimpleValueType();
4135
4136 SDLoc DL(Op);
4137 MVT XLenVT = Subtarget.getXLenVT();
4138 unsigned OrigIdx = Op.getConstantOperandVal(1);
4139 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
4140
4141 // We don't have the ability to slide mask vectors down indexed by their i1
4142 // elements; the smallest we can do is i8. Often we are able to bitcast to
4143 // equivalent i8 vectors. Note that when extracting a fixed-length vector
4144 // from a scalable one, we might not necessarily have enough scalable
4145 // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
4146 if (SubVecVT.getVectorElementType() == MVT::i1 && OrigIdx != 0) {
4147 if (VecVT.getVectorMinNumElements() >= 8 &&
4148 SubVecVT.getVectorMinNumElements() >= 8) {
4149 assert(OrigIdx % 8 == 0 && "Invalid index");
4150 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
4151 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
4152 "Unexpected mask vector lowering");
4153 OrigIdx /= 8;
4154 SubVecVT =
4155 MVT::getVectorVT(MVT::i8, SubVecVT.getVectorMinNumElements() / 8,
4156 SubVecVT.isScalableVector());
4157 VecVT = MVT::getVectorVT(MVT::i8, VecVT.getVectorMinNumElements() / 8,
4158 VecVT.isScalableVector());
4159 Vec = DAG.getBitcast(VecVT, Vec);
4160 } else {
4161 // We can't slide this mask vector down, indexed by its i1 elements.
4162 // This poses a problem when we wish to extract a scalable vector which
4163 // can't be re-expressed as a larger type. Just choose the slow path and
4164 // extend to a larger type, then truncate back down.
4165 // TODO: We could probably improve this when extracting certain fixed
4166 // from fixed, where we can extract as i8 and shift the correct element
4167 // right to reach the desired subvector?
4168 MVT ExtVecVT = VecVT.changeVectorElementType(MVT::i8);
4169 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(MVT::i8);
4170 Vec = DAG.getNode(ISD::ZERO_EXTEND, DL, ExtVecVT, Vec);
4171 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtSubVecVT, Vec,
4172 Op.getOperand(1));
4173 SDValue SplatZero = DAG.getConstant(0, DL, ExtSubVecVT);
4174 return DAG.getSetCC(DL, SubVecVT, Vec, SplatZero, ISD::SETNE);
4175 }
4176 }
4177
4178 // If the subvector vector is a fixed-length type, we cannot use subregister
4179 // manipulation to simplify the codegen; we don't know which register of a
4180 // LMUL group contains the specific subvector as we only know the minimum
4181 // register size. Therefore we must slide the vector group down the full
4182 // amount.
4183 if (SubVecVT.isFixedLengthVector()) {
4184 // With an index of 0 this is a cast-like subvector, which can be performed
4185 // with subregister operations.
4186 if (OrigIdx == 0)
4187 return Op;
4188 MVT ContainerVT = VecVT;
4189 if (VecVT.isFixedLengthVector()) {
4190 ContainerVT = getContainerForFixedLengthVector(VecVT);
4191 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
4192 }
4193 SDValue Mask =
4194 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
4195 // Set the vector length to only the number of elements we care about. This
4196 // avoids sliding down elements we're going to discard straight away.
4197 SDValue VL = DAG.getConstant(SubVecVT.getVectorNumElements(), DL, XLenVT);
4198 SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
4199 SDValue Slidedown =
4200 DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
4201 DAG.getUNDEF(ContainerVT), Vec, SlidedownAmt, Mask, VL);
4202 // Now we can use a cast-like subvector extract to get the result.
4203 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
4204 DAG.getConstant(0, DL, XLenVT));
4205 return DAG.getBitcast(Op.getValueType(), Slidedown);
4206 }
4207
4208 unsigned SubRegIdx, RemIdx;
4209 std::tie(SubRegIdx, RemIdx) =
4210 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
4211 VecVT, SubVecVT, OrigIdx, TRI);
4212
4213 // If the Idx has been completely eliminated then this is a subvector extract
4214 // which naturally aligns to a vector register. These can easily be handled
4215 // using subregister manipulation.
4216 if (RemIdx == 0)
4217 return Op;
4218
4219 // Else we must shift our vector register directly to extract the subvector.
4220 // Do this using VSLIDEDOWN.
4221
4222 // If the vector type is an LMUL-group type, extract a subvector equal to the
4223 // nearest full vector register type. This should resolve to a EXTRACT_SUBREG
4224 // instruction.
4225 MVT InterSubVT = VecVT;
4226 if (VecVT.bitsGT(getLMUL1VT(VecVT))) {
4227 InterSubVT = getLMUL1VT(VecVT);
4228 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InterSubVT, Vec,
4229 DAG.getConstant(OrigIdx - RemIdx, DL, XLenVT));
4230 }
4231
4232 // Slide this vector register down by the desired number of elements in order
4233 // to place the desired subvector starting at element 0.
4234 SDValue SlidedownAmt = DAG.getConstant(RemIdx, DL, XLenVT);
4235 // For scalable vectors this must be further multiplied by vscale.
4236 SlidedownAmt = DAG.getNode(ISD::VSCALE, DL, XLenVT, SlidedownAmt);
4237
4238 SDValue Mask, VL;
4239 std::tie(Mask, VL) = getDefaultScalableVLOps(InterSubVT, DL, DAG, Subtarget);
4240 SDValue Slidedown =
4241 DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, InterSubVT,
4242 DAG.getUNDEF(InterSubVT), Vec, SlidedownAmt, Mask, VL);
4243
4244 // Now the vector is in the right position, extract our final subvector. This
4245 // should resolve to a COPY.
4246 Slidedown = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVecVT, Slidedown,
4247 DAG.getConstant(0, DL, XLenVT));
4248
4249 // We might have bitcast from a mask type: cast back to the original type if
4250 // required.
4251 return DAG.getBitcast(Op.getSimpleValueType(), Slidedown);
4252 }
4253
4254 // Lower step_vector to the vid instruction. Any non-identity step value must
4255 // be accounted for my manual expansion.
4256 SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
4257 SelectionDAG &DAG) const {
4258 SDLoc DL(Op);
4259 MVT VT = Op.getSimpleValueType();
4260 MVT XLenVT = Subtarget.getXLenVT();
4261 SDValue Mask, VL;
4262 std::tie(Mask, VL) = getDefaultScalableVLOps(VT, DL, DAG, Subtarget);
4263 SDValue StepVec = DAG.getNode(RISCVISD::VID_VL, DL, VT, Mask, VL);
4264 uint64_t StepValImm = Op.getConstantOperandVal(0);
4265 if (StepValImm != 1) {
4266 if (isPowerOf2_64(StepValImm)) {
4267 SDValue StepVal =
4268 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT,
4269 DAG.getConstant(Log2_64(StepValImm), DL, XLenVT));
4270 StepVec = DAG.getNode(ISD::SHL, DL, VT, StepVec, StepVal);
4271 } else {
4272 SDValue StepVal = lowerScalarSplat(
4273 DAG.getConstant(StepValImm, DL, VT.getVectorElementType()), VL, VT,
4274 DL, DAG, Subtarget);
4275 StepVec = DAG.getNode(ISD::MUL, DL, VT, StepVec, StepVal);
4276 }
4277 }
4278 return StepVec;
4279 }
4280
4281 // Implement vector_reverse using vrgather.vv with indices determined by
4282 // subtracting the id of each element from (VLMAX-1). This will convert
4283 // the indices like so:
4284 // (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
4285 // TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
4286 SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
4287 SelectionDAG &DAG) const {
4288 SDLoc DL(Op);
4289 MVT VecVT = Op.getSimpleValueType();
4290 unsigned EltSize = VecVT.getScalarSizeInBits();
4291 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
4292
4293 unsigned MaxVLMAX = 0;
4294 unsigned VectorBitsMax = Subtarget.getMaxRVVVectorSizeInBits();
4295 if (VectorBitsMax != 0)
4296 MaxVLMAX = ((VectorBitsMax / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
4297
4298 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
4299 MVT IntVT = VecVT.changeVectorElementTypeToInteger();
4300
4301 // If this is SEW=8 and VLMAX is unknown or more than 256, we need
4302 // to use vrgatherei16.vv.
4303 // TODO: It's also possible to use vrgatherei16.vv for other types to
4304 // decrease register width for the index calculation.
4305 if ((MaxVLMAX == 0 || MaxVLMAX > 256) && EltSize == 8) {
4306 // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
4307 // Reverse each half, then reassemble them in reverse order.
4308 // NOTE: It's also possible that after splitting that VLMAX no longer
4309 // requires vrgatherei16.vv.
4310 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
4311 SDValue Lo, Hi;
4312 std::tie(Lo, Hi) = DAG.SplitVectorOperand(Op.getNode(), 0);
4313 EVT LoVT, HiVT;
4314 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
4315 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, LoVT, Lo);
4316 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, HiVT, Hi);
4317 // Reassemble the low and high pieces reversed.
4318 // FIXME: This is a CONCAT_VECTORS.
4319 SDValue Res =
4320 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT, DAG.getUNDEF(VecVT), Hi,
4321 DAG.getIntPtrConstant(0, DL));
4322 return DAG.getNode(
4323 ISD::INSERT_SUBVECTOR, DL, VecVT, Res, Lo,
4324 DAG.getIntPtrConstant(LoVT.getVectorMinNumElements(), DL));
4325 }
4326
4327 // Just promote the int type to i16 which will double the LMUL.
4328 IntVT = MVT::getVectorVT(MVT::i16, VecVT.getVectorElementCount());
4329 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
4330 }
4331
4332 MVT XLenVT = Subtarget.getXLenVT();
4333 SDValue Mask, VL;
4334 std::tie(Mask, VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
4335
4336 // Calculate VLMAX-1 for the desired SEW.
4337 unsigned MinElts = VecVT.getVectorMinNumElements();
4338 SDValue VLMax = DAG.getNode(ISD::VSCALE, DL, XLenVT,
4339 DAG.getConstant(MinElts, DL, XLenVT));
4340 SDValue VLMinus1 =
4341 DAG.getNode(ISD::SUB, DL, XLenVT, VLMax, DAG.getConstant(1, DL, XLenVT));
4342
4343 // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
4344 bool IsRV32E64 =
4345 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
4346 SDValue SplatVL;
4347 if (!IsRV32E64)
4348 SplatVL = DAG.getSplatVector(IntVT, DL, VLMinus1);
4349 else
4350 SplatVL = DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, IntVT, VLMinus1);
4351
4352 SDValue VID = DAG.getNode(RISCVISD::VID_VL, DL, IntVT, Mask, VL);
4353 SDValue Indices =
4354 DAG.getNode(RISCVISD::SUB_VL, DL, IntVT, SplatVL, VID, Mask, VL);
4355
4356 return DAG.getNode(GatherOpc, DL, VecVT, Op.getOperand(0), Indices, Mask, VL);
4357 }
4358
4359 SDValue
4360 RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
4361 SelectionDAG &DAG) const {
4362 SDLoc DL(Op);
4363 auto *Load = cast<LoadSDNode>(Op);
4364
4365 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
4366 Load->getMemoryVT(),
4367 *Load->getMemOperand()) &&
4368 "Expecting a correctly-aligned load");
4369
4370 MVT VT = Op.getSimpleValueType();
4371 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4372
4373 SDValue VL =
4374 DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
4375
4376 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4377 SDValue NewLoad = DAG.getMemIntrinsicNode(
4378 RISCVISD::VLE_VL, DL, VTs, {Load->getChain(), Load->getBasePtr(), VL},
4379 Load->getMemoryVT(), Load->getMemOperand());
4380
4381 SDValue Result = convertFromScalableVector(VT, NewLoad, DAG, Subtarget);
4382 return DAG.getMergeValues({Result, Load->getChain()}, DL);
4383 }
4384
4385 SDValue
4386 RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
4387 SelectionDAG &DAG) const {
4388 SDLoc DL(Op);
4389 auto *Store = cast<StoreSDNode>(Op);
4390
4391 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
4392 Store->getMemoryVT(),
4393 *Store->getMemOperand()) &&
4394 "Expecting a correctly-aligned store");
4395
4396 SDValue StoreVal = Store->getValue();
4397 MVT VT = StoreVal.getSimpleValueType();
4398
4399 // If the size less than a byte, we need to pad with zeros to make a byte.
4400 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
4401 VT = MVT::v8i1;
4402 StoreVal = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
4403 DAG.getConstant(0, DL, VT), StoreVal,
4404 DAG.getIntPtrConstant(0, DL));
4405 }
4406
4407 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4408
4409 SDValue VL =
4410 DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
4411
4412 SDValue NewValue =
4413 convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
4414 return DAG.getMemIntrinsicNode(
4415 RISCVISD::VSE_VL, DL, DAG.getVTList(MVT::Other),
4416 {Store->getChain(), NewValue, Store->getBasePtr(), VL},
4417 Store->getMemoryVT(), Store->getMemOperand());
4418 }
4419
4420 SDValue RISCVTargetLowering::lowerMLOAD(SDValue Op, SelectionDAG &DAG) const {
4421 auto *Load = cast<MaskedLoadSDNode>(Op);
4422
4423 SDLoc DL(Op);
4424 MVT VT = Op.getSimpleValueType();
4425 MVT XLenVT = Subtarget.getXLenVT();
4426
4427 SDValue Mask = Load->getMask();
4428 SDValue PassThru = Load->getPassThru();
4429 SDValue VL;
4430
4431 MVT ContainerVT = VT;
4432 if (VT.isFixedLengthVector()) {
4433 ContainerVT = getContainerForFixedLengthVector(VT);
4434 MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4435
4436 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4437 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4438 VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4439 } else
4440 VL = DAG.getRegister(RISCV::X0, XLenVT);
4441
4442 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4443 SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vle_mask, DL, XLenVT);
4444 SDValue Ops[] = {Load->getChain(), IntID, PassThru,
4445 Load->getBasePtr(), Mask, VL};
4446 SDValue Result =
4447 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4448 Load->getMemoryVT(), Load->getMemOperand());
4449 SDValue Chain = Result.getValue(1);
4450
4451 if (VT.isFixedLengthVector())
4452 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4453
4454 return DAG.getMergeValues({Result, Chain}, DL);
4455 }
4456
4457 SDValue RISCVTargetLowering::lowerMSTORE(SDValue Op, SelectionDAG &DAG) const {
4458 auto *Store = cast<MaskedStoreSDNode>(Op);
4459
4460 SDLoc DL(Op);
4461 SDValue Val = Store->getValue();
4462 SDValue Mask = Store->getMask();
4463 MVT VT = Val.getSimpleValueType();
4464 MVT XLenVT = Subtarget.getXLenVT();
4465 SDValue VL;
4466
4467 MVT ContainerVT = VT;
4468 if (VT.isFixedLengthVector()) {
4469 ContainerVT = getContainerForFixedLengthVector(VT);
4470 MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4471
4472 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4473 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4474 VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4475 } else
4476 VL = DAG.getRegister(RISCV::X0, XLenVT);
4477
4478 SDValue IntID = DAG.getTargetConstant(Intrinsic::riscv_vse_mask, DL, XLenVT);
4479 return DAG.getMemIntrinsicNode(
4480 ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other),
4481 {Store->getChain(), IntID, Val, Store->getBasePtr(), Mask, VL},
4482 Store->getMemoryVT(), Store->getMemOperand());
4483 }
4484
4485 SDValue
4486 RISCVTargetLowering::lowerFixedLengthVectorSetccToRVV(SDValue Op,
4487 SelectionDAG &DAG) const {
4488 MVT InVT = Op.getOperand(0).getSimpleValueType();
4489 MVT ContainerVT = getContainerForFixedLengthVector(InVT);
4490
4491 MVT VT = Op.getSimpleValueType();
4492
4493 SDValue Op1 =
4494 convertToScalableVector(ContainerVT, Op.getOperand(0), DAG, Subtarget);
4495 SDValue Op2 =
4496 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
4497
4498 SDLoc DL(Op);
4499 SDValue VL =
4500 DAG.getConstant(VT.getVectorNumElements(), DL, Subtarget.getXLenVT());
4501
4502 MVT MaskVT = MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4503 SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
4504
4505 SDValue Cmp = DAG.getNode(RISCVISD::SETCC_VL, DL, MaskVT, Op1, Op2,
4506 Op.getOperand(2), Mask, VL);
4507
4508 return convertFromScalableVector(VT, Cmp, DAG, Subtarget);
4509 }
4510
4511 SDValue RISCVTargetLowering::lowerFixedLengthVectorLogicOpToRVV(
4512 SDValue Op, SelectionDAG &DAG, unsigned MaskOpc, unsigned VecOpc) const {
4513 MVT VT = Op.getSimpleValueType();
4514
4515 if (VT.getVectorElementType() == MVT::i1)
4516 return lowerToScalableOp(Op, DAG, MaskOpc, /*HasMask*/ false);
4517
4518 return lowerToScalableOp(Op, DAG, VecOpc, /*HasMask*/ true);
4519 }
4520
4521 SDValue
4522 RISCVTargetLowering::lowerFixedLengthVectorShiftToRVV(SDValue Op,
4523 SelectionDAG &DAG) const {
4524 unsigned Opc;
4525 switch (Op.getOpcode()) {
4526 default: llvm_unreachable("Unexpected opcode!");
4527 case ISD::SHL: Opc = RISCVISD::SHL_VL; break;
4528 case ISD::SRA: Opc = RISCVISD::SRA_VL; break;
4529 case ISD::SRL: Opc = RISCVISD::SRL_VL; break;
4530 }
4531
4532 return lowerToScalableOp(Op, DAG, Opc);
4533 }
4534
4535 // Lower vector ABS to smax(X, sub(0, X)).
4536 SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
4537 SDLoc DL(Op);
4538 MVT VT = Op.getSimpleValueType();
4539 SDValue X = Op.getOperand(0);
4540
4541 assert(VT.isFixedLengthVector() && "Unexpected type");
4542
4543 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4544 X = convertToScalableVector(ContainerVT, X, DAG, Subtarget);
4545
4546 SDValue Mask, VL;
4547 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4548
4549 SDValue SplatZero =
4550 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
4551 DAG.getConstant(0, DL, Subtarget.getXLenVT()));
4552 SDValue NegX =
4553 DAG.getNode(RISCVISD::SUB_VL, DL, ContainerVT, SplatZero, X, Mask, VL);
4554 SDValue Max =
4555 DAG.getNode(RISCVISD::SMAX_VL, DL, ContainerVT, X, NegX, Mask, VL);
4556
4557 return convertFromScalableVector(VT, Max, DAG, Subtarget);
4558 }
4559
4560 SDValue RISCVTargetLowering::lowerFixedLengthVectorFCOPYSIGNToRVV(
4561 SDValue Op, SelectionDAG &DAG) const {
4562 SDLoc DL(Op);
4563 MVT VT = Op.getSimpleValueType();
4564 SDValue Mag = Op.getOperand(0);
4565 SDValue Sign = Op.getOperand(1);
4566 assert(Mag.getValueType() == Sign.getValueType() &&
4567 "Can only handle COPYSIGN with matching types.");
4568
4569 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4570 Mag = convertToScalableVector(ContainerVT, Mag, DAG, Subtarget);
4571 Sign = convertToScalableVector(ContainerVT, Sign, DAG, Subtarget);
4572
4573 SDValue Mask, VL;
4574 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4575
4576 SDValue CopySign =
4577 DAG.getNode(RISCVISD::FCOPYSIGN_VL, DL, ContainerVT, Mag, Sign, Mask, VL);
4578
4579 return convertFromScalableVector(VT, CopySign, DAG, Subtarget);
4580 }
4581
4582 SDValue RISCVTargetLowering::lowerFixedLengthVectorSelectToRVV(
4583 SDValue Op, SelectionDAG &DAG) const {
4584 MVT VT = Op.getSimpleValueType();
4585 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4586
4587 MVT I1ContainerVT =
4588 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4589
4590 SDValue CC =
4591 convertToScalableVector(I1ContainerVT, Op.getOperand(0), DAG, Subtarget);
4592 SDValue Op1 =
4593 convertToScalableVector(ContainerVT, Op.getOperand(1), DAG, Subtarget);
4594 SDValue Op2 =
4595 convertToScalableVector(ContainerVT, Op.getOperand(2), DAG, Subtarget);
4596
4597 SDLoc DL(Op);
4598 SDValue Mask, VL;
4599 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4600
4601 SDValue Select =
4602 DAG.getNode(RISCVISD::VSELECT_VL, DL, ContainerVT, CC, Op1, Op2, VL);
4603
4604 return convertFromScalableVector(VT, Select, DAG, Subtarget);
4605 }
4606
4607 SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op, SelectionDAG &DAG,
4608 unsigned NewOpc,
4609 bool HasMask) const {
4610 MVT VT = Op.getSimpleValueType();
4611 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4612
4613 // Create list of operands by converting existing ones to scalable types.
4614 SmallVector<SDValue, 6> Ops;
4615 for (const SDValue &V : Op->op_values()) {
4616 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
4617
4618 // Pass through non-vector operands.
4619 if (!V.getValueType().isVector()) {
4620 Ops.push_back(V);
4621 continue;
4622 }
4623
4624 // "cast" fixed length vector to a scalable vector.
4625 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
4626 "Only fixed length vectors are supported!");
4627 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
4628 }
4629
4630 SDLoc DL(Op);
4631 SDValue Mask, VL;
4632 std::tie(Mask, VL) = getDefaultVLOps(VT, ContainerVT, DL, DAG, Subtarget);
4633 if (HasMask)
4634 Ops.push_back(Mask);
4635 Ops.push_back(VL);
4636
4637 SDValue ScalableRes = DAG.getNode(NewOpc, DL, ContainerVT, Ops);
4638 return convertFromScalableVector(VT, ScalableRes, DAG, Subtarget);
4639 }
4640
4641 // Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
4642 // * Operands of each node are assumed to be in the same order.
4643 // * The EVL operand is promoted from i32 to i64 on RV64.
4644 // * Fixed-length vectors are converted to their scalable-vector container
4645 // types.
4646 SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG,
4647 unsigned RISCVISDOpc) const {
4648 SDLoc DL(Op);
4649 MVT VT = Op.getSimpleValueType();
4650 SmallVector<SDValue, 4> Ops;
4651
4652 for (const auto &OpIdx : enumerate(Op->ops())) {
4653 SDValue V = OpIdx.value();
4654 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
4655 // Pass through operands which aren't fixed-length vectors.
4656 if (!V.getValueType().isFixedLengthVector()) {
4657 Ops.push_back(V);
4658 continue;
4659 }
4660 // "cast" fixed length vector to a scalable vector.
4661 MVT OpVT = V.getSimpleValueType();
4662 MVT ContainerVT = getContainerForFixedLengthVector(OpVT);
4663 assert(useRVVForFixedLengthVectorVT(OpVT) &&
4664 "Only fixed length vectors are supported!");
4665 Ops.push_back(convertToScalableVector(ContainerVT, V, DAG, Subtarget));
4666 }
4667
4668 if (!VT.isFixedLengthVector())
4669 return DAG.getNode(RISCVISDOpc, DL, VT, Ops);
4670
4671 MVT ContainerVT = getContainerForFixedLengthVector(VT);
4672
4673 SDValue VPOp = DAG.getNode(RISCVISDOpc, DL, ContainerVT, Ops);
4674
4675 return convertFromScalableVector(VT, VPOp, DAG, Subtarget);
4676 }
4677
4678 // Custom lower MGATHER to a legalized form for RVV. It will then be matched to
4679 // a RVV indexed load. The RVV indexed load instructions only support the
4680 // "unsigned unscaled" addressing mode; indices are implicitly zero-extended or
4681 // truncated to XLEN and are treated as byte offsets. Any signed or scaled
4682 // indexing is extended to the XLEN value type and scaled accordingly.
4683 SDValue RISCVTargetLowering::lowerMGATHER(SDValue Op, SelectionDAG &DAG) const {
4684 auto *MGN = cast<MaskedGatherSDNode>(Op.getNode());
4685 SDLoc DL(Op);
4686
4687 SDValue Index = MGN->getIndex();
4688 SDValue Mask = MGN->getMask();
4689 SDValue PassThru = MGN->getPassThru();
4690
4691 MVT VT = Op.getSimpleValueType();
4692 MVT IndexVT = Index.getSimpleValueType();
4693 MVT XLenVT = Subtarget.getXLenVT();
4694
4695 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
4696 "Unexpected VTs!");
4697 assert(MGN->getBasePtr().getSimpleValueType() == XLenVT &&
4698 "Unexpected pointer type");
4699 // Targets have to explicitly opt-in for extending vector loads.
4700 assert(MGN->getExtensionType() == ISD::NON_EXTLOAD &&
4701 "Unexpected extending MGATHER");
4702
4703 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4704 // the selection of the masked intrinsics doesn't do this for us.
4705 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4706
4707 SDValue VL;
4708 MVT ContainerVT = VT;
4709 if (VT.isFixedLengthVector()) {
4710 // We need to use the larger of the result and index type to determine the
4711 // scalable type to use so we don't increase LMUL for any operand/result.
4712 if (VT.bitsGE(IndexVT)) {
4713 ContainerVT = getContainerForFixedLengthVector(VT);
4714 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
4715 ContainerVT.getVectorElementCount());
4716 } else {
4717 IndexVT = getContainerForFixedLengthVector(IndexVT);
4718 ContainerVT = MVT::getVectorVT(ContainerVT.getVectorElementType(),
4719 IndexVT.getVectorElementCount());
4720 }
4721
4722 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
4723
4724 if (!IsUnmasked) {
4725 MVT MaskVT =
4726 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4727 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4728 PassThru = convertToScalableVector(ContainerVT, PassThru, DAG, Subtarget);
4729 }
4730
4731 VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4732 } else
4733 VL = DAG.getRegister(RISCV::X0, XLenVT);
4734
4735 unsigned IntID =
4736 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
4737 SmallVector<SDValue, 8> Ops{MGN->getChain(),
4738 DAG.getTargetConstant(IntID, DL, XLenVT)};
4739 if (!IsUnmasked)
4740 Ops.push_back(PassThru);
4741 Ops.push_back(MGN->getBasePtr());
4742 Ops.push_back(Index);
4743 if (!IsUnmasked)
4744 Ops.push_back(Mask);
4745 Ops.push_back(VL);
4746
4747 SDVTList VTs = DAG.getVTList({ContainerVT, MVT::Other});
4748 SDValue Result =
4749 DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, VTs, Ops,
4750 MGN->getMemoryVT(), MGN->getMemOperand());
4751 SDValue Chain = Result.getValue(1);
4752
4753 if (VT.isFixedLengthVector())
4754 Result = convertFromScalableVector(VT, Result, DAG, Subtarget);
4755
4756 return DAG.getMergeValues({Result, Chain}, DL);
4757 }
4758
4759 // Custom lower MSCATTER to a legalized form for RVV. It will then be matched to
4760 // a RVV indexed store. The RVV indexed store instructions only support the
4761 // "unsigned unscaled" addressing mode; indices are implicitly zero-extended or
4762 // truncated to XLEN and are treated as byte offsets. Any signed or scaled
4763 // indexing is extended to the XLEN value type and scaled accordingly.
4764 SDValue RISCVTargetLowering::lowerMSCATTER(SDValue Op,
4765 SelectionDAG &DAG) const {
4766 auto *MSN = cast<MaskedScatterSDNode>(Op.getNode());
4767 SDLoc DL(Op);
4768 SDValue Index = MSN->getIndex();
4769 SDValue Mask = MSN->getMask();
4770 SDValue Val = MSN->getValue();
4771
4772 MVT VT = Val.getSimpleValueType();
4773 MVT IndexVT = Index.getSimpleValueType();
4774 MVT XLenVT = Subtarget.getXLenVT();
4775
4776 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
4777 "Unexpected VTs!");
4778 assert(MSN->getBasePtr().getSimpleValueType() == XLenVT &&
4779 "Unexpected pointer type");
4780 // Targets have to explicitly opt-in for extending vector loads and
4781 // truncating vector stores.
4782 assert(!MSN->isTruncatingStore() && "Unexpected extending MSCATTER");
4783
4784 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
4785 // the selection of the masked intrinsics doesn't do this for us.
4786 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(Mask.getNode());
4787
4788 SDValue VL;
4789 if (VT.isFixedLengthVector()) {
4790 // We need to use the larger of the value and index type to determine the
4791 // scalable type to use so we don't increase LMUL for any operand/result.
4792 MVT ContainerVT;
4793 if (VT.bitsGE(IndexVT)) {
4794 ContainerVT = getContainerForFixedLengthVector(VT);
4795 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(),
4796 ContainerVT.getVectorElementCount());
4797 } else {
4798 IndexVT = getContainerForFixedLengthVector(IndexVT);
4799 ContainerVT = MVT::getVectorVT(VT.getVectorElementType(),
4800 IndexVT.getVectorElementCount());
4801 }
4802
4803 Index = convertToScalableVector(IndexVT, Index, DAG, Subtarget);
4804 Val = convertToScalableVector(ContainerVT, Val, DAG, Subtarget);
4805
4806 if (!IsUnmasked) {
4807 MVT MaskVT =
4808 MVT::getVectorVT(MVT::i1, ContainerVT.getVectorElementCount());
4809 Mask = convertToScalableVector(MaskVT, Mask, DAG, Subtarget);
4810 }
4811
4812 VL = DAG.getConstant(VT.getVectorNumElements(), DL, XLenVT);
4813 } else
4814 VL = DAG.getRegister(RISCV::X0, XLenVT);
4815
4816 unsigned IntID =
4817 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
4818 SmallVector<SDValue, 8> Ops{MSN->getChain(),
4819 DAG.getTargetConstant(IntID, DL, XLenVT)};
4820 Ops.push_back(Val);
4821 Ops.push_back(MSN->getBasePtr());
4822 Ops.push_back(Index);
4823 if (!IsUnmasked)
4824 Ops.push_back(Mask);
4825 Ops.push_back(VL);
4826
4827 return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, MSN->getVTList(), Ops,
4828 MSN->getMemoryVT(), MSN->getMemOperand());
4829 }
4830
4831 SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
4832 SelectionDAG &DAG) const {
4833 const MVT XLenVT = Subtarget.getXLenVT();
4834 SDLoc DL(Op);
4835 SDValue Chain = Op->getOperand(0);
4836 SDValue SysRegNo = DAG.getConstant(
4837 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
4838 SDVTList VTs = DAG.getVTList(XLenVT, MVT::Other);
4839 SDValue RM = DAG.getNode(RISCVISD::READ_CSR, DL, VTs, Chain, SysRegNo);
4840
4841 // Encoding used for rounding mode in RISCV differs from that used in
4842 // FLT_ROUNDS. To convert it the RISCV rounding mode is used as an index in a
4843 // table, which consists of a sequence of 4-bit fields, each representing
4844 // corresponding FLT_ROUNDS mode.
4845 static const int Table =
4846 (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
4847 (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
4848 (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
4849 (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
4850 (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
4851
4852 SDValue Shift =
4853 DAG.getNode(ISD::SHL, DL, XLenVT, RM, DAG.getConstant(2, DL, XLenVT));
4854 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
4855 DAG.getConstant(Table, DL, XLenVT), Shift);
4856 SDValue Masked = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
4857 DAG.getConstant(7, DL, XLenVT));
4858
4859 return DAG.getMergeValues({Masked, Chain}, DL);
4860 }
4861
4862 SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
4863 SelectionDAG &DAG) const {
4864 const MVT XLenVT = Subtarget.getXLenVT();
4865 SDLoc DL(Op);
4866 SDValue Chain = Op->getOperand(0);
4867 SDValue RMValue = Op->getOperand(1);
4868 SDValue SysRegNo = DAG.getConstant(
4869 RISCVSysReg::lookupSysRegByName("FRM")->Encoding, DL, XLenVT);
4870
4871 // Encoding used for rounding mode in RISCV differs from that used in
4872 // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
4873 // a table, which consists of a sequence of 4-bit fields, each representing
4874 // corresponding RISCV mode.
4875 static const unsigned Table =
4876 (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
4877 (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
4878 (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
4879 (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
4880 (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
4881
4882 SDValue Shift = DAG.getNode(ISD::SHL, DL, XLenVT, RMValue,
4883 DAG.getConstant(2, DL, XLenVT));
4884 SDValue Shifted = DAG.getNode(ISD::SRL, DL, XLenVT,
4885 DAG.getConstant(Table, DL, XLenVT), Shift);
4886 RMValue = DAG.getNode(ISD::AND, DL, XLenVT, Shifted,
4887 DAG.getConstant(0x7, DL, XLenVT));
4888 return DAG.getNode(RISCVISD::WRITE_CSR, DL, MVT::Other, Chain, SysRegNo,
4889 RMValue);
4890 }
4891
4892 // Returns the opcode of the target-specific SDNode that implements the 32-bit
4893 // form of the given Opcode.
4894 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
4895 switch (Opcode) {
4896 default:
4897 llvm_unreachable("Unexpected opcode");
4898 case ISD::SHL:
4899 return RISCVISD::SLLW;
4900 case ISD::SRA:
4901 return RISCVISD::SRAW;
4902 case ISD::SRL:
4903 return RISCVISD::SRLW;
4904 case ISD::SDIV:
4905 return RISCVISD::DIVW;
4906 case ISD::UDIV:
4907 return RISCVISD::DIVUW;
4908 case ISD::UREM:
4909 return RISCVISD::REMUW;
4910 case ISD::ROTL:
4911 return RISCVISD::ROLW;
4912 case ISD::ROTR:
4913 return RISCVISD::RORW;
4914 case RISCVISD::GREV:
4915 return RISCVISD::GREVW;
4916 case RISCVISD::GORC:
4917 return RISCVISD::GORCW;
4918 }
4919 }
4920
4921 // Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
4922 // node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
4923 // otherwise be promoted to i64, making it difficult to select the
4924 // SLLW/DIVUW/.../*W later one because the fact the operation was originally of
4925 // type i8/i16/i32 is lost.
4926 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
4927 unsigned ExtOpc = ISD::ANY_EXTEND) {
4928 SDLoc DL(N);
4929 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
4930 SDValue NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
4931 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
4932 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
4933 // ReplaceNodeResults requires we maintain the same type for the return value.
4934 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
4935 }
4936
4937 // Converts the given 32-bit operation to a i64 operation with signed extension
4938 // semantic to reduce the signed extension instructions.
4939 static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
4940 SDLoc DL(N);
4941 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
4942 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
4943 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
4944 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
4945 DAG.getValueType(MVT::i32));
4946 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
4947 }
4948
4949 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
4950 SmallVectorImpl<SDValue> &Results,
4951 SelectionDAG &DAG) const {
4952 SDLoc DL(N);
4953 switch (N->getOpcode()) {
4954 default:
4955 llvm_unreachable("Don't know how to custom type legalize this operation!");
4956 case ISD::STRICT_FP_TO_SINT:
4957 case ISD::STRICT_FP_TO_UINT:
4958 case ISD::FP_TO_SINT:
4959 case ISD::FP_TO_UINT: {
4960 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
4961 "Unexpected custom legalisation");
4962 bool IsStrict = N->isStrictFPOpcode();
4963 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
4964 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
4965 SDValue Op0 = IsStrict ? N->getOperand(1) : N->getOperand(0);
4966 if (getTypeAction(*DAG.getContext(), Op0.getValueType()) !=
4967 TargetLowering::TypeSoftenFloat) {
4968 // FIXME: Support strict FP.
4969 if (IsStrict)
4970 return;
4971 if (!isTypeLegal(Op0.getValueType()))
4972 return;
4973 unsigned Opc =
4974 IsSigned ? RISCVISD::FCVT_W_RTZ_RV64 : RISCVISD::FCVT_WU_RTZ_RV64;
4975 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, Op0);
4976 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
4977 return;
4978 }
4979 // If the FP type needs to be softened, emit a library call using the 'si'
4980 // version. If we left it to default legalization we'd end up with 'di'. If
4981 // the FP type doesn't need to be softened just let generic type
4982 // legalization promote the result type.
4983 RTLIB::Libcall LC;
4984 if (IsSigned)
4985 LC = RTLIB::getFPTOSINT(Op0.getValueType(), N->getValueType(0));
4986 else
4987 LC = RTLIB::getFPTOUINT(Op0.getValueType(), N->getValueType(0));
4988 MakeLibCallOptions CallOptions;
4989 EVT OpVT = Op0.getValueType();
4990 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
4991 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
4992 SDValue Result;
4993 std::tie(Result, Chain) =
4994 makeLibCall(DAG, LC, N->getValueType(0), Op0, CallOptions, DL, Chain);
4995 Results.push_back(Result);
4996 if (IsStrict)
4997 Results.push_back(Chain);
4998 break;
4999 }
5000 case ISD::READCYCLECOUNTER: {
5001 assert(!Subtarget.is64Bit() &&
5002 "READCYCLECOUNTER only has custom type legalization on riscv32");
5003
5004 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
5005 SDValue RCW =
5006 DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
5007
5008 Results.push_back(
5009 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, RCW, RCW.getValue(1)));
5010 Results.push_back(RCW.getValue(2));
5011 break;
5012 }
5013 case ISD::MUL: {
5014 unsigned Size = N->getSimpleValueType(0).getSizeInBits();
5015 unsigned XLen = Subtarget.getXLen();
5016 // This multiply needs to be expanded, try to use MULHSU+MUL if possible.
5017 if (Size > XLen) {
5018 assert(Size == (XLen * 2) && "Unexpected custom legalisation");
5019 SDValue LHS = N->getOperand(0);
5020 SDValue RHS = N->getOperand(1);
5021 APInt HighMask = APInt::getHighBitsSet(Size, XLen);
5022
5023 bool LHSIsU = DAG.MaskedValueIsZero(LHS, HighMask);
5024 bool RHSIsU = DAG.MaskedValueIsZero(RHS, HighMask);
5025 // We need exactly one side to be unsigned.
5026 if (LHSIsU == RHSIsU)
5027 return;
5028
5029 auto MakeMULPair = [&](SDValue S, SDValue U) {
5030 MVT XLenVT = Subtarget.getXLenVT();
5031 S = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, S);
5032 U = DAG.getNode(ISD::TRUNCATE, DL, XLenVT, U);
5033 SDValue Lo = DAG.getNode(ISD::MUL, DL, XLenVT, S, U);
5034 SDValue Hi = DAG.getNode(RISCVISD::MULHSU, DL, XLenVT, S, U);
5035 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getValueType(0), Lo, Hi);
5036 };
5037
5038 bool LHSIsS = DAG.ComputeNumSignBits(LHS) > XLen;
5039 bool RHSIsS = DAG.ComputeNumSignBits(RHS) > XLen;
5040
5041 // The other operand should be signed, but still prefer MULH when
5042 // possible.
5043 if (RHSIsU && LHSIsS && !RHSIsS)
5044 Results.push_back(MakeMULPair(LHS, RHS));
5045 else if (LHSIsU && RHSIsS && !LHSIsS)
5046 Results.push_back(MakeMULPair(RHS, LHS));
5047
5048 return;
5049 }
5050 LLVM_FALLTHROUGH;
5051 }
5052 case ISD::ADD:
5053 case ISD::SUB:
5054 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5055 "Unexpected custom legalisation");
5056 Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
5057 break;
5058 case ISD::SHL:
5059 case ISD::SRA:
5060 case ISD::SRL:
5061 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5062 "Unexpected custom legalisation");
5063 if (N->getOperand(1).getOpcode() == ISD::Constant)
5064 return;
5065 Results.push_back(customLegalizeToWOp(N, DAG));
5066 break;
5067 case ISD::ROTL:
5068 case ISD::ROTR:
5069 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5070 "Unexpected custom legalisation");
5071 Results.push_back(customLegalizeToWOp(N, DAG));
5072 break;
5073 case ISD::CTTZ:
5074 case ISD::CTTZ_ZERO_UNDEF:
5075 case ISD::CTLZ:
5076 case ISD::CTLZ_ZERO_UNDEF: {
5077 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5078 "Unexpected custom legalisation");
5079
5080 SDValue NewOp0 =
5081 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5082 bool IsCTZ =
5083 N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::CTTZ_ZERO_UNDEF;
5084 unsigned Opc = IsCTZ ? RISCVISD::CTZW : RISCVISD::CLZW;
5085 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp0);
5086 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5087 return;
5088 }
5089 case ISD::SDIV:
5090 case ISD::UDIV:
5091 case ISD::UREM: {
5092 MVT VT = N->getSimpleValueType(0);
5093 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
5094 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
5095 "Unexpected custom legalisation");
5096 // Don't promote division/remainder by constant since we should expand those
5097 // to multiply by magic constant.
5098 // FIXME: What if the expansion is disabled for minsize.
5099 if (N->getOperand(1).getOpcode() == ISD::Constant)
5100 return;
5101
5102 // If the input is i32, use ANY_EXTEND since the W instructions don't read
5103 // the upper 32 bits. For other types we need to sign or zero extend
5104 // based on the opcode.
5105 unsigned ExtOpc = ISD::ANY_EXTEND;
5106 if (VT != MVT::i32)
5107 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
5108 : ISD::ZERO_EXTEND;
5109
5110 Results.push_back(customLegalizeToWOp(N, DAG, ExtOpc));
5111 break;
5112 }
5113 case ISD::UADDO:
5114 case ISD::USUBO: {
5115 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5116 "Unexpected custom legalisation");
5117 bool IsAdd = N->getOpcode() == ISD::UADDO;
5118 // Create an ADDW or SUBW.
5119 SDValue LHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5120 SDValue RHS = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5121 SDValue Res =
5122 DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, DL, MVT::i64, LHS, RHS);
5123 Res = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, Res,
5124 DAG.getValueType(MVT::i32));
5125
5126 // Sign extend the LHS and perform an unsigned compare with the ADDW result.
5127 // Since the inputs are sign extended from i32, this is equivalent to
5128 // comparing the lower 32 bits.
5129 LHS = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
5130 SDValue Overflow = DAG.getSetCC(DL, N->getValueType(1), Res, LHS,
5131 IsAdd ? ISD::SETULT : ISD::SETUGT);
5132
5133 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5134 Results.push_back(Overflow);
5135 return;
5136 }
5137 case ISD::UADDSAT:
5138 case ISD::USUBSAT: {
5139 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5140 "Unexpected custom legalisation");
5141 if (Subtarget.hasStdExtZbb()) {
5142 // With Zbb we can sign extend and let LegalizeDAG use minu/maxu. Using
5143 // sign extend allows overflow of the lower 32 bits to be detected on
5144 // the promoted size.
5145 SDValue LHS =
5146 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(0));
5147 SDValue RHS =
5148 DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, N->getOperand(1));
5149 SDValue Res = DAG.getNode(N->getOpcode(), DL, MVT::i64, LHS, RHS);
5150 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5151 return;
5152 }
5153
5154 // Without Zbb, expand to UADDO/USUBO+select which will trigger our custom
5155 // promotion for UADDO/USUBO.
5156 Results.push_back(expandAddSubSat(N, DAG));
5157 return;
5158 }
5159 case ISD::BITCAST: {
5160 EVT VT = N->getValueType(0);
5161 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
5162 SDValue Op0 = N->getOperand(0);
5163 EVT Op0VT = Op0.getValueType();
5164 MVT XLenVT = Subtarget.getXLenVT();
5165 if (VT == MVT::i16 && Op0VT == MVT::f16 && Subtarget.hasStdExtZfh()) {
5166 SDValue FPConv = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, XLenVT, Op0);
5167 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FPConv));
5168 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
5169 Subtarget.hasStdExtF()) {
5170 SDValue FPConv =
5171 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
5172 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
5173 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
5174 isTypeLegal(Op0VT)) {
5175 // Custom-legalize bitcasts from fixed-length vector types to illegal
5176 // scalar types in order to improve codegen. Bitcast the vector to a
5177 // one-element vector type whose element type is the same as the result
5178 // type, and extract the first element.
5179 LLVMContext &Context = *DAG.getContext();
5180 SDValue BVec = DAG.getBitcast(EVT::getVectorVT(Context, VT, 1), Op0);
5181 Results.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, BVec,
5182 DAG.getConstant(0, DL, XLenVT)));
5183 }
5184 break;
5185 }
5186 case RISCVISD::GREV:
5187 case RISCVISD::GORC: {
5188 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5189 "Unexpected custom legalisation");
5190 assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
5191 // This is similar to customLegalizeToWOp, except that we pass the second
5192 // operand (a TargetConstant) straight through: it is already of type
5193 // XLenVT.
5194 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
5195 SDValue NewOp0 =
5196 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5197 SDValue NewOp1 =
5198 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5199 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
5200 // ReplaceNodeResults requires we maintain the same type for the return
5201 // value.
5202 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
5203 break;
5204 }
5205 case RISCVISD::SHFL: {
5206 // There is no SHFLIW instruction, but we can just promote the operation.
5207 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5208 "Unexpected custom legalisation");
5209 assert(isa<ConstantSDNode>(N->getOperand(1)) && "Expected constant");
5210 SDValue NewOp0 =
5211 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5212 SDValue NewOp1 =
5213 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5214 SDValue NewRes = DAG.getNode(RISCVISD::SHFL, DL, MVT::i64, NewOp0, NewOp1);
5215 // ReplaceNodeResults requires we maintain the same type for the return
5216 // value.
5217 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes));
5218 break;
5219 }
5220 case ISD::BSWAP:
5221 case ISD::BITREVERSE: {
5222 MVT VT = N->getSimpleValueType(0);
5223 MVT XLenVT = Subtarget.getXLenVT();
5224 assert((VT == MVT::i8 || VT == MVT::i16 ||
5225 (VT == MVT::i32 && Subtarget.is64Bit())) &&
5226 Subtarget.hasStdExtZbp() && "Unexpected custom legalisation");
5227 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, XLenVT, N->getOperand(0));
5228 unsigned Imm = VT.getSizeInBits() - 1;
5229 // If this is BSWAP rather than BITREVERSE, clear the lower 3 bits.
5230 if (N->getOpcode() == ISD::BSWAP)
5231 Imm &= ~0x7U;
5232 unsigned Opc = Subtarget.is64Bit() ? RISCVISD::GREVW : RISCVISD::GREV;
5233 SDValue GREVI =
5234 DAG.getNode(Opc, DL, XLenVT, NewOp0, DAG.getConstant(Imm, DL, XLenVT));
5235 // ReplaceNodeResults requires we maintain the same type for the return
5236 // value.
5237 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, GREVI));
5238 break;
5239 }
5240 case ISD::FSHL:
5241 case ISD::FSHR: {
5242 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5243 Subtarget.hasStdExtZbt() && "Unexpected custom legalisation");
5244 SDValue NewOp0 =
5245 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5246 SDValue NewOp1 =
5247 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5248 SDValue NewOp2 =
5249 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
5250 // FSLW/FSRW take a 6 bit shift amount but i32 FSHL/FSHR only use 5 bits.
5251 // Mask the shift amount to 5 bits.
5252 NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
5253 DAG.getConstant(0x1f, DL, MVT::i64));
5254 unsigned Opc =
5255 N->getOpcode() == ISD::FSHL ? RISCVISD::FSLW : RISCVISD::FSRW;
5256 SDValue NewOp = DAG.getNode(Opc, DL, MVT::i64, NewOp0, NewOp1, NewOp2);
5257 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewOp));
5258 break;
5259 }
5260 case ISD::EXTRACT_VECTOR_ELT: {
5261 // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
5262 // type is illegal (currently only vXi64 RV32).
5263 // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
5264 // transferred to the destination register. We issue two of these from the
5265 // upper- and lower- halves of the SEW-bit vector element, slid down to the
5266 // first element.
5267 SDValue Vec = N->getOperand(0);
5268 SDValue Idx = N->getOperand(1);
5269
5270 // The vector type hasn't been legalized yet so we can't issue target
5271 // specific nodes if it needs legalization.
5272 // FIXME: We would manually legalize if it's important.
5273 if (!isTypeLegal(Vec.getValueType()))
5274 return;
5275
5276 MVT VecVT = Vec.getSimpleValueType();
5277
5278 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
5279 VecVT.getVectorElementType() == MVT::i64 &&
5280 "Unexpected EXTRACT_VECTOR_ELT legalization");
5281
5282 // If this is a fixed vector, we need to convert it to a scalable vector.
5283 MVT ContainerVT = VecVT;
5284 if (VecVT.isFixedLengthVector()) {
5285 ContainerVT = getContainerForFixedLengthVector(VecVT);
5286 Vec = convertToScalableVector(ContainerVT, Vec, DAG, Subtarget);
5287 }
5288
5289 MVT XLenVT = Subtarget.getXLenVT();
5290
5291 // Use a VL of 1 to avoid processing more elements than we need.
5292 MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
5293 SDValue VL = DAG.getConstant(1, DL, XLenVT);
5294 SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5295
5296 // Unless the index is known to be 0, we must slide the vector down to get
5297 // the desired element into index 0.
5298 if (!isNullConstant(Idx)) {
5299 Vec = DAG.getNode(RISCVISD::VSLIDEDOWN_VL, DL, ContainerVT,
5300 DAG.getUNDEF(ContainerVT), Vec, Idx, Mask, VL);
5301 }
5302
5303 // Extract the lower XLEN bits of the correct vector element.
5304 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
5305
5306 // To extract the upper XLEN bits of the vector element, shift the first
5307 // element right by 32 bits and re-extract the lower XLEN bits.
5308 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, ContainerVT,
5309 DAG.getConstant(32, DL, XLenVT), VL);
5310 SDValue LShr32 = DAG.getNode(RISCVISD::SRL_VL, DL, ContainerVT, Vec,
5311 ThirtyTwoV, Mask, VL);
5312
5313 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
5314
5315 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
5316 break;
5317 }
5318 case ISD::INTRINSIC_WO_CHAIN: {
5319 unsigned IntNo = cast<ConstantSDNode>(N->getOperand(0))->getZExtValue();
5320 switch (IntNo) {
5321 default:
5322 llvm_unreachable(
5323 "Don't know how to custom type legalize this intrinsic!");
5324 case Intrinsic::riscv_orc_b: {
5325 // Lower to the GORCI encoding for orc.b with the operand extended.
5326 SDValue NewOp =
5327 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5328 // If Zbp is enabled, use GORCIW which will sign extend the result.
5329 unsigned Opc =
5330 Subtarget.hasStdExtZbp() ? RISCVISD::GORCW : RISCVISD::GORC;
5331 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp,
5332 DAG.getConstant(7, DL, MVT::i64));
5333 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5334 return;
5335 }
5336 case Intrinsic::riscv_grev:
5337 case Intrinsic::riscv_gorc: {
5338 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5339 "Unexpected custom legalisation");
5340 SDValue NewOp1 =
5341 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5342 SDValue NewOp2 =
5343 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
5344 unsigned Opc =
5345 IntNo == Intrinsic::riscv_grev ? RISCVISD::GREVW : RISCVISD::GORCW;
5346 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
5347 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5348 break;
5349 }
5350 case Intrinsic::riscv_shfl:
5351 case Intrinsic::riscv_unshfl: {
5352 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5353 "Unexpected custom legalisation");
5354 SDValue NewOp1 =
5355 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5356 SDValue NewOp2 =
5357 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
5358 unsigned Opc =
5359 IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFLW : RISCVISD::UNSHFLW;
5360 if (isa<ConstantSDNode>(N->getOperand(2))) {
5361 NewOp2 = DAG.getNode(ISD::AND, DL, MVT::i64, NewOp2,
5362 DAG.getConstant(0xf, DL, MVT::i64));
5363 Opc =
5364 IntNo == Intrinsic::riscv_shfl ? RISCVISD::SHFL : RISCVISD::UNSHFL;
5365 }
5366 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
5367 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5368 break;
5369 }
5370 case Intrinsic::riscv_bcompress:
5371 case Intrinsic::riscv_bdecompress: {
5372 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5373 "Unexpected custom legalisation");
5374 SDValue NewOp1 =
5375 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5376 SDValue NewOp2 =
5377 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(2));
5378 unsigned Opc = IntNo == Intrinsic::riscv_bcompress
5379 ? RISCVISD::BCOMPRESSW
5380 : RISCVISD::BDECOMPRESSW;
5381 SDValue Res = DAG.getNode(Opc, DL, MVT::i64, NewOp1, NewOp2);
5382 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
5383 break;
5384 }
5385 case Intrinsic::riscv_vmv_x_s: {
5386 EVT VT = N->getValueType(0);
5387 MVT XLenVT = Subtarget.getXLenVT();
5388 if (VT.bitsLT(XLenVT)) {
5389 // Simple case just extract using vmv.x.s and truncate.
5390 SDValue Extract = DAG.getNode(RISCVISD::VMV_X_S, DL,
5391 Subtarget.getXLenVT(), N->getOperand(1));
5392 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Extract));
5393 return;
5394 }
5395
5396 assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
5397 "Unexpected custom legalization");
5398
5399 // We need to do the move in two steps.
5400 SDValue Vec = N->getOperand(1);
5401 MVT VecVT = Vec.getSimpleValueType();
5402
5403 // First extract the lower XLEN bits of the element.
5404 SDValue EltLo = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, Vec);
5405
5406 // To extract the upper XLEN bits of the vector element, shift the first
5407 // element right by 32 bits and re-extract the lower XLEN bits.
5408 SDValue VL = DAG.getConstant(1, DL, XLenVT);
5409 MVT MaskVT = MVT::getVectorVT(MVT::i1, VecVT.getVectorElementCount());
5410 SDValue Mask = DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
5411 SDValue ThirtyTwoV = DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VecVT,
5412 DAG.getConstant(32, DL, XLenVT), VL);
5413 SDValue LShr32 =
5414 DAG.getNode(RISCVISD::SRL_VL, DL, VecVT, Vec, ThirtyTwoV, Mask, VL);
5415 SDValue EltHi = DAG.getNode(RISCVISD::VMV_X_S, DL, XLenVT, LShr32);
5416
5417 Results.push_back(
5418 DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, EltLo, EltHi));
5419 break;
5420 }
5421 }
5422 break;
5423 }
5424 case ISD::VECREDUCE_ADD:
5425 case ISD::VECREDUCE_AND:
5426 case ISD::VECREDUCE_OR:
5427 case ISD::VECREDUCE_XOR:
5428 case ISD::VECREDUCE_SMAX:
5429 case ISD::VECREDUCE_UMAX:
5430 case ISD::VECREDUCE_SMIN:
5431 case ISD::VECREDUCE_UMIN:
5432 if (SDValue V = lowerVECREDUCE(SDValue(N, 0), DAG))
5433 Results.push_back(V);
5434 break;
5435 case ISD::FLT_ROUNDS_: {
5436 SDVTList VTs = DAG.getVTList(Subtarget.getXLenVT(), MVT::Other);
5437 SDValue Res = DAG.getNode(ISD::FLT_ROUNDS_, DL, VTs, N->getOperand(0));
5438 Results.push_back(Res.getValue(0));
5439 Results.push_back(Res.getValue(1));
5440 break;
5441 }
5442 }
5443 }
5444
5445 // A structure to hold one of the bit-manipulation patterns below. Together, a
5446 // SHL and non-SHL pattern may form a bit-manipulation pair on a single source:
5447 // (or (and (shl x, 1), 0xAAAAAAAA),
5448 // (and (srl x, 1), 0x55555555))
5449 struct RISCVBitmanipPat {
5450 SDValue Op;
5451 unsigned ShAmt;
5452 bool IsSHL;
5453
5454 bool formsPairWith(const RISCVBitmanipPat &Other) const {
5455 return Op == Other.Op && ShAmt == Other.ShAmt && IsSHL != Other.IsSHL;
5456 }
5457 };
5458
5459 // Matches patterns of the form
5460 // (and (shl x, C2), (C1 << C2))
5461 // (and (srl x, C2), C1)
5462 // (shl (and x, C1), C2)
5463 // (srl (and x, (C1 << C2)), C2)
5464 // Where C2 is a power of 2 and C1 has at least that many leading zeroes.
5465 // The expected masks for each shift amount are specified in BitmanipMasks where
5466 // BitmanipMasks[log2(C2)] specifies the expected C1 value.
5467 // The max allowed shift amount is either XLen/2 or XLen/4 determined by whether
5468 // BitmanipMasks contains 6 or 5 entries assuming that the maximum possible
5469 // XLen is 64.
5470 static Optional<RISCVBitmanipPat>
5471 matchRISCVBitmanipPat(SDValue Op, ArrayRef<uint64_t> BitmanipMasks) {
5472 assert((BitmanipMasks.size() == 5 || BitmanipMasks.size() == 6) &&
5473 "Unexpected number of masks");
5474 Optional<uint64_t> Mask;
5475 // Optionally consume a mask around the shift operation.
5476 if (Op.getOpcode() == ISD::AND && isa<ConstantSDNode>(Op.getOperand(1))) {
5477 Mask = Op.getConstantOperandVal(1);
5478 Op = Op.getOperand(0);
5479 }
5480 if (Op.getOpcode() != ISD::SHL && Op.getOpcode() != ISD::SRL)
5481 return None;
5482 bool IsSHL = Op.getOpcode() == ISD::SHL;
5483
5484 if (!isa<ConstantSDNode>(Op.getOperand(1)))
5485 return None;
5486 uint64_t ShAmt = Op.getConstantOperandVal(1);
5487
5488 unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
5489 if (ShAmt >= Width || !isPowerOf2_64(ShAmt))
5490 return None;
5491 // If we don't have enough masks for 64 bit, then we must be trying to
5492 // match SHFL so we're only allowed to shift 1/4 of the width.
5493 if (BitmanipMasks.size() == 5 && ShAmt >= (Width / 2))
5494 return None;
5495
5496 SDValue Src = Op.getOperand(0);
5497
5498 // The expected mask is shifted left when the AND is found around SHL
5499 // patterns.
5500 // ((x >> 1) & 0x55555555)
5501 // ((x << 1) & 0xAAAAAAAA)
5502 bool SHLExpMask = IsSHL;
5503
5504 if (!Mask) {
5505 // Sometimes LLVM keeps the mask as an operand of the shift, typically when
5506 // the mask is all ones: consume that now.
5507 if (Src.getOpcode() == ISD::AND && isa<ConstantSDNode>(Src.getOperand(1))) {
5508 Mask = Src.getConstantOperandVal(1);
5509 Src = Src.getOperand(0);
5510 // The expected mask is now in fact shifted left for SRL, so reverse the
5511 // decision.
5512 // ((x & 0xAAAAAAAA) >> 1)
5513 // ((x & 0x55555555) << 1)
5514 SHLExpMask = !SHLExpMask;
5515 } else {
5516 // Use a default shifted mask of all-ones if there's no AND, truncated
5517 // down to the expected width. This simplifies the logic later on.
5518 Mask = maskTrailingOnes<uint64_t>(Width);
5519 *Mask &= (IsSHL ? *Mask << ShAmt : *Mask >> ShAmt);
5520 }
5521 }
5522
5523 unsigned MaskIdx = Log2_32(ShAmt);
5524 uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
5525
5526 if (SHLExpMask)
5527 ExpMask <<= ShAmt;
5528
5529 if (Mask != ExpMask)
5530 return None;
5531
5532 return RISCVBitmanipPat{Src, (unsigned)ShAmt, IsSHL};
5533 }
5534
5535 // Matches any of the following bit-manipulation patterns:
5536 // (and (shl x, 1), (0x55555555 << 1))
5537 // (and (srl x, 1), 0x55555555)
5538 // (shl (and x, 0x55555555), 1)
5539 // (srl (and x, (0x55555555 << 1)), 1)
5540 // where the shift amount and mask may vary thus:
5541 // [1] = 0x55555555 / 0xAAAAAAAA
5542 // [2] = 0x33333333 / 0xCCCCCCCC
5543 // [4] = 0x0F0F0F0F / 0xF0F0F0F0
5544 // [8] = 0x00FF00FF / 0xFF00FF00
5545 // [16] = 0x0000FFFF / 0xFFFFFFFF
5546 // [32] = 0x00000000FFFFFFFF / 0xFFFFFFFF00000000 (for RV64)
5547 static Optional<RISCVBitmanipPat> matchGREVIPat(SDValue Op) {
5548 // These are the unshifted masks which we use to match bit-manipulation
5549 // patterns. They may be shifted left in certain circumstances.
5550 static const uint64_t BitmanipMasks[] = {
5551 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
5552 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
5553
5554 return matchRISCVBitmanipPat(Op, BitmanipMasks);
5555 }
5556
5557 // Match the following pattern as a GREVI(W) operation
5558 // (or (BITMANIP_SHL x), (BITMANIP_SRL x))
5559 static SDValue combineORToGREV(SDValue Op, SelectionDAG &DAG,
5560 const RISCVSubtarget &Subtarget) {
5561 assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
5562 EVT VT = Op.getValueType();
5563
5564 if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
5565 auto LHS = matchGREVIPat(Op.getOperand(0));
5566 auto RHS = matchGREVIPat(Op.getOperand(1));
5567 if (LHS && RHS && LHS->formsPairWith(*RHS)) {
5568 SDLoc DL(Op);
5569 return DAG.getNode(RISCVISD::GREV, DL, VT, LHS->Op,
5570 DAG.getConstant(LHS->ShAmt, DL, VT));
5571 }
5572 }
5573 return SDValue();
5574 }
5575
5576 // Matches any the following pattern as a GORCI(W) operation
5577 // 1. (or (GREVI x, shamt), x) if shamt is a power of 2
5578 // 2. (or x, (GREVI x, shamt)) if shamt is a power of 2
5579 // 3. (or (or (BITMANIP_SHL x), x), (BITMANIP_SRL x))
5580 // Note that with the variant of 3.,
5581 // (or (or (BITMANIP_SHL x), (BITMANIP_SRL x)), x)
5582 // the inner pattern will first be matched as GREVI and then the outer
5583 // pattern will be matched to GORC via the first rule above.
5584 // 4. (or (rotl/rotr x, bitwidth/2), x)
5585 static SDValue combineORToGORC(SDValue Op, SelectionDAG &DAG,
5586 const RISCVSubtarget &Subtarget) {
5587 assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
5588 EVT VT = Op.getValueType();
5589
5590 if (VT == Subtarget.getXLenVT() || (Subtarget.is64Bit() && VT == MVT::i32)) {
5591 SDLoc DL(Op);
5592 SDValue Op0 = Op.getOperand(0);
5593 SDValue Op1 = Op.getOperand(1);
5594
5595 auto MatchOROfReverse = [&](SDValue Reverse, SDValue X) {
5596 if (Reverse.getOpcode() == RISCVISD::GREV && Reverse.getOperand(0) == X &&
5597 isa<ConstantSDNode>(Reverse.getOperand(1)) &&
5598 isPowerOf2_32(Reverse.getConstantOperandVal(1)))
5599 return DAG.getNode(RISCVISD::GORC, DL, VT, X, Reverse.getOperand(1));
5600 // We can also form GORCI from ROTL/ROTR by half the bitwidth.
5601 if ((Reverse.getOpcode() == ISD::ROTL ||
5602 Reverse.getOpcode() == ISD::ROTR) &&
5603 Reverse.getOperand(0) == X &&
5604 isa<ConstantSDNode>(Reverse.getOperand(1))) {
5605 uint64_t RotAmt = Reverse.getConstantOperandVal(1);
5606 if (RotAmt == (VT.getSizeInBits() / 2))
5607 return DAG.getNode(RISCVISD::GORC, DL, VT, X,
5608 DAG.getConstant(RotAmt, DL, VT));
5609 }
5610 return SDValue();
5611 };
5612
5613 // Check for either commutable permutation of (or (GREVI x, shamt), x)
5614 if (SDValue V = MatchOROfReverse(Op0, Op1))
5615 return V;
5616 if (SDValue V = MatchOROfReverse(Op1, Op0))
5617 return V;
5618
5619 // OR is commutable so canonicalize its OR operand to the left
5620 if (Op0.getOpcode() != ISD::OR && Op1.getOpcode() == ISD::OR)
5621 std::swap(Op0, Op1);
5622 if (Op0.getOpcode() != ISD::OR)
5623 return SDValue();
5624 SDValue OrOp0 = Op0.getOperand(0);
5625 SDValue OrOp1 = Op0.getOperand(1);
5626 auto LHS = matchGREVIPat(OrOp0);
5627 // OR is commutable so swap the operands and try again: x might have been
5628 // on the left
5629 if (!LHS) {
5630 std::swap(OrOp0, OrOp1);
5631 LHS = matchGREVIPat(OrOp0);
5632 }
5633 auto RHS = matchGREVIPat(Op1);
5634 if (LHS && RHS && LHS->formsPairWith(*RHS) && LHS->Op == OrOp1) {
5635 return DAG.getNode(RISCVISD::GORC, DL, VT, LHS->Op,
5636 DAG.getConstant(LHS->ShAmt, DL, VT));
5637 }
5638 }
5639 return SDValue();
5640 }
5641
5642 // Matches any of the following bit-manipulation patterns:
5643 // (and (shl x, 1), (0x22222222 << 1))
5644 // (and (srl x, 1), 0x22222222)
5645 // (shl (and x, 0x22222222), 1)
5646 // (srl (and x, (0x22222222 << 1)), 1)
5647 // where the shift amount and mask may vary thus:
5648 // [1] = 0x22222222 / 0x44444444
5649 // [2] = 0x0C0C0C0C / 0x3C3C3C3C
5650 // [4] = 0x00F000F0 / 0x0F000F00
5651 // [8] = 0x0000FF00 / 0x00FF0000
5652 // [16] = 0x00000000FFFF0000 / 0x0000FFFF00000000 (for RV64)
5653 static Optional<RISCVBitmanipPat> matchSHFLPat(SDValue Op) {
5654 // These are the unshifted masks which we use to match bit-manipulation
5655 // patterns. They may be shifted left in certain circumstances.
5656 static const uint64_t BitmanipMasks[] = {
5657 0x2222222222222222ULL, 0x0C0C0C0C0C0C0C0CULL, 0x00F000F000F000F0ULL,
5658 0x0000FF000000FF00ULL, 0x00000000FFFF0000ULL};
5659
5660 return matchRISCVBitmanipPat(Op, BitmanipMasks);
5661 }
5662
5663 // Match (or (or (SHFL_SHL x), (SHFL_SHR x)), (SHFL_AND x)
5664 static SDValue combineORToSHFL(SDValue Op, SelectionDAG &DAG,
5665 const RISCVSubtarget &Subtarget) {
5666 assert(Subtarget.hasStdExtZbp() && "Expected Zbp extenson");
5667 EVT VT = Op.getValueType();
5668
5669 if (VT != MVT::i32 && VT != Subtarget.getXLenVT())
5670 return SDValue();
5671
5672 SDValue Op0 = Op.getOperand(0);
5673 SDValue Op1 = Op.getOperand(1);
5674
5675 // Or is commutable so canonicalize the second OR to the LHS.
5676 if (Op0.getOpcode() != ISD::OR)
5677 std::swap(Op0, Op1);
5678 if (Op0.getOpcode() != ISD::OR)
5679 return SDValue();
5680
5681 // We found an inner OR, so our operands are the operands of the inner OR
5682 // and the other operand of the outer OR.
5683 SDValue A = Op0.getOperand(0);
5684 SDValue B = Op0.getOperand(1);
5685 SDValue C = Op1;
5686
5687 auto Match1 = matchSHFLPat(A);
5688 auto Match2 = matchSHFLPat(B);
5689
5690 // If neither matched, we failed.
5691 if (!Match1 && !Match2)
5692 return SDValue();
5693
5694 // We had at least one match. if one failed, try the remaining C operand.
5695 if (!Match1) {
5696 std::swap(A, C);
5697 Match1 = matchSHFLPat(A);
5698 if (!Match1)
5699 return SDValue();
5700 } else if (!Match2) {
5701 std::swap(B, C);
5702 Match2 = matchSHFLPat(B);
5703 if (!Match2)
5704 return SDValue();
5705 }
5706 assert(Match1 && Match2);
5707
5708 // Make sure our matches pair up.
5709 if (!Match1->formsPairWith(*Match2))
5710 return SDValue();
5711
5712 // All the remains is to make sure C is an AND with the same input, that masks
5713 // out the bits that are being shuffled.
5714 if (C.getOpcode() != ISD::AND || !isa<ConstantSDNode>(C.getOperand(1)) ||
5715 C.getOperand(0) != Match1->Op)
5716 return SDValue();
5717
5718 uint64_t Mask = C.getConstantOperandVal(1);
5719
5720 static const uint64_t BitmanipMasks[] = {
5721 0x9999999999999999ULL, 0xC3C3C3C3C3C3C3C3ULL, 0xF00FF00FF00FF00FULL,
5722 0xFF0000FFFF0000FFULL, 0xFFFF00000000FFFFULL,
5723 };
5724
5725 unsigned Width = Op.getValueType() == MVT::i64 ? 64 : 32;
5726 unsigned MaskIdx = Log2_32(Match1->ShAmt);
5727 uint64_t ExpMask = BitmanipMasks[MaskIdx] & maskTrailingOnes<uint64_t>(Width);
5728
5729 if (Mask != ExpMask)
5730 return SDValue();
5731
5732 SDLoc DL(Op);
5733 return DAG.getNode(RISCVISD::SHFL, DL, VT, Match1->Op,
5734 DAG.getConstant(Match1->ShAmt, DL, VT));
5735 }
5736
5737 // Combine (GREVI (GREVI x, C2), C1) -> (GREVI x, C1^C2) when C1^C2 is
5738 // non-zero, and to x when it is. Any repeated GREVI stage undoes itself.
5739 // Combine (GORCI (GORCI x, C2), C1) -> (GORCI x, C1|C2). Repeated stage does
5740 // not undo itself, but they are redundant.
5741 static SDValue combineGREVI_GORCI(SDNode *N, SelectionDAG &DAG) {
5742 SDValue Src = N->getOperand(0);
5743
5744 if (Src.getOpcode() != N->getOpcode())
5745 return SDValue();
5746
5747 if (!isa<ConstantSDNode>(N->getOperand(1)) ||
5748 !isa<ConstantSDNode>(Src.getOperand(1)))
5749 return SDValue();
5750
5751 unsigned ShAmt1 = N->getConstantOperandVal(1);
5752 unsigned ShAmt2 = Src.getConstantOperandVal(1);
5753 Src = Src.getOperand(0);
5754
5755 unsigned CombinedShAmt;
5756 if (N->getOpcode() == RISCVISD::GORC || N->getOpcode() == RISCVISD::GORCW)
5757 CombinedShAmt = ShAmt1 | ShAmt2;
5758 else
5759 CombinedShAmt = ShAmt1 ^ ShAmt2;
5760
5761 if (CombinedShAmt == 0)
5762 return Src;
5763
5764 SDLoc DL(N);
5765 return DAG.getNode(
5766 N->getOpcode(), DL, N->getValueType(0), Src,
5767 DAG.getConstant(CombinedShAmt, DL, N->getOperand(1).getValueType()));
5768 }
5769
5770 // Combine a constant select operand into its use:
5771 //
5772 // (and (select cond, -1, c), x)
5773 // -> (select cond, x, (and x, c)) [AllOnes=1]
5774 // (or (select cond, 0, c), x)
5775 // -> (select cond, x, (or x, c)) [AllOnes=0]
5776 // (xor (select cond, 0, c), x)
5777 // -> (select cond, x, (xor x, c)) [AllOnes=0]
5778 // (add (select cond, 0, c), x)
5779 // -> (select cond, x, (add x, c)) [AllOnes=0]
5780 // (sub x, (select cond, 0, c))
5781 // -> (select cond, x, (sub x, c)) [AllOnes=0]
5782 static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
5783 SelectionDAG &DAG, bool AllOnes) {
5784 EVT VT = N->getValueType(0);
5785
5786 // Skip vectors.
5787 if (VT.isVector())
5788 return SDValue();
5789
5790 if ((Slct.getOpcode() != ISD::SELECT &&
5791 Slct.getOpcode() != RISCVISD::SELECT_CC) ||
5792 !Slct.hasOneUse())
5793 return SDValue();
5794
5795 auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
5796 return AllOnes ? isAllOnesConstant(N) : isNullConstant(N);
5797 };
5798
5799 bool SwapSelectOps;
5800 unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
5801 SDValue TrueVal = Slct.getOperand(1 + OpOffset);
5802 SDValue FalseVal = Slct.getOperand(2 + OpOffset);
5803 SDValue NonConstantVal;
5804 if (isZeroOrAllOnes(TrueVal, AllOnes)) {
5805 SwapSelectOps = false;
5806 NonConstantVal = FalseVal;
5807 } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
5808 SwapSelectOps = true;
5809 NonConstantVal = TrueVal;
5810 } else
5811 return SDValue();
5812
5813 // Slct is now know to be the desired identity constant when CC is true.
5814 TrueVal = OtherOp;
5815 FalseVal = DAG.getNode(N->getOpcode(), SDLoc(N), VT, OtherOp, NonConstantVal);
5816 // Unless SwapSelectOps says the condition should be false.
5817 if (SwapSelectOps)
5818 std::swap(TrueVal, FalseVal);
5819
5820 if (Slct.getOpcode() == RISCVISD::SELECT_CC)
5821 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), VT,
5822 {Slct.getOperand(0), Slct.getOperand(1),
5823 Slct.getOperand(2), TrueVal, FalseVal});
5824
5825 return DAG.getNode(ISD::SELECT, SDLoc(N), VT,
5826 {Slct.getOperand(0), TrueVal, FalseVal});
5827 }
5828
5829 // Attempt combineSelectAndUse on each operand of a commutative operator N.
5830 static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
5831 bool AllOnes) {
5832 SDValue N0 = N->getOperand(0);
5833 SDValue N1 = N->getOperand(1);
5834 if (SDValue Result = combineSelectAndUse(N, N0, N1, DAG, AllOnes))
5835 return Result;
5836 if (SDValue Result = combineSelectAndUse(N, N1, N0, DAG, AllOnes))
5837 return Result;
5838 return SDValue();
5839 }
5840
5841 static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG) {
5842 // fold (add (select lhs, rhs, cc, 0, y), x) ->
5843 // (select lhs, rhs, cc, x, (add x, y))
5844 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
5845 }
5846
5847 static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG) {
5848 // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
5849 // (select lhs, rhs, cc, x, (sub x, y))
5850 SDValue N0 = N->getOperand(0);
5851 SDValue N1 = N->getOperand(1);
5852 return combineSelectAndUse(N, N1, N0, DAG, /*AllOnes*/ false);
5853 }
5854
5855 static SDValue performANDCombine(SDNode *N, SelectionDAG &DAG) {
5856 // fold (and (select lhs, rhs, cc, -1, y), x) ->
5857 // (select lhs, rhs, cc, x, (and x, y))
5858 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true);
5859 }
5860
5861 static SDValue performORCombine(SDNode *N, SelectionDAG &DAG,
5862 const RISCVSubtarget &Subtarget) {
5863 if (Subtarget.hasStdExtZbp()) {
5864 if (auto GREV = combineORToGREV(SDValue(N, 0), DAG, Subtarget))
5865 return GREV;
5866 if (auto GORC = combineORToGORC(SDValue(N, 0), DAG, Subtarget))
5867 return GORC;
5868 if (auto SHFL = combineORToSHFL(SDValue(N, 0), DAG, Subtarget))
5869 return SHFL;
5870 }
5871
5872 // fold (or (select cond, 0, y), x) ->
5873 // (select cond, x, (or x, y))
5874 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
5875 }
5876
5877 static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG) {
5878 // fold (xor (select cond, 0, y), x) ->
5879 // (select cond, x, (xor x, y))
5880 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false);
5881 }
5882
5883 // Attempt to turn ANY_EXTEND into SIGN_EXTEND if the input to the ANY_EXTEND
5884 // has users that require SIGN_EXTEND and the SIGN_EXTEND can be done for free
5885 // by an instruction like ADDW/SUBW/MULW. Without this the ANY_EXTEND would be
5886 // removed during type legalization leaving an ADD/SUB/MUL use that won't use
5887 // ADDW/SUBW/MULW.
5888 static SDValue performANY_EXTENDCombine(SDNode *N,
5889 TargetLowering::DAGCombinerInfo &DCI,
5890 const RISCVSubtarget &Subtarget) {
5891 if (!Subtarget.is64Bit())
5892 return SDValue();
5893
5894 SelectionDAG &DAG = DCI.DAG;
5895
5896 SDValue Src = N->getOperand(0);
5897 EVT VT = N->getValueType(0);
5898 if (VT != MVT::i64 || Src.getValueType() != MVT::i32)
5899 return SDValue();
5900
5901 // The opcode must be one that can implicitly sign_extend.
5902 // FIXME: Additional opcodes.
5903 switch (Src.getOpcode()) {
5904 default:
5905 return SDValue();
5906 case ISD::MUL:
5907 if (!Subtarget.hasStdExtM())
5908 return SDValue();
5909 LLVM_FALLTHROUGH;
5910 case ISD::ADD:
5911 case ISD::SUB:
5912 break;
5913 }
5914
5915 // Only handle cases where the result is used by a CopyToReg. That likely
5916 // means the value is a liveout of the basic block. This helps prevent
5917 // infinite combine loops like PR51206.
5918 if (none_of(N->uses(),
5919 [](SDNode *User) { return User->getOpcode() == ISD::CopyToReg; }))
5920 return SDValue();
5921
5922 SmallVector<SDNode *, 4> SetCCs;
5923 for (SDNode::use_iterator UI = Src.getNode()->use_begin(),
5924 UE = Src.getNode()->use_end();
5925 UI != UE; ++UI) {
5926 SDNode *User = *UI;
5927 if (User == N)
5928 continue;
5929 if (UI.getUse().getResNo() != Src.getResNo())
5930 continue;
5931 // All i32 setccs are legalized by sign extending operands.
5932 if (User->getOpcode() == ISD::SETCC) {
5933 SetCCs.push_back(User);
5934 continue;
5935 }
5936 // We don't know if we can extend this user.
5937 break;
5938 }
5939
5940 // If we don't have any SetCCs, this isn't worthwhile.
5941 if (SetCCs.empty())
5942 return SDValue();
5943
5944 SDLoc DL(N);
5945 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, Src);
5946 DCI.CombineTo(N, SExt);
5947
5948 // Promote all the setccs.
5949 for (SDNode *SetCC : SetCCs) {
5950 SmallVector<SDValue, 4> Ops;
5951
5952 for (unsigned j = 0; j != 2; ++j) {
5953 SDValue SOp = SetCC->getOperand(j);
5954 if (SOp == Src)
5955 Ops.push_back(SExt);
5956 else
5957 Ops.push_back(DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i64, SOp));
5958 }
5959
5960 Ops.push_back(SetCC->getOperand(2));
5961 DCI.CombineTo(SetCC,
5962 DAG.getNode(ISD::SETCC, DL, SetCC->getValueType(0), Ops));
5963 }
5964 return SDValue(N, 0);
5965 }
5966
5967 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
5968 DAGCombinerInfo &DCI) const {
5969 SelectionDAG &DAG = DCI.DAG;
5970
5971 // Helper to call SimplifyDemandedBits on an operand of N where only some low
5972 // bits are demanded. N will be added to the Worklist if it was not deleted.
5973 // Caller should return SDValue(N, 0) if this returns true.
5974 auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
5975 SDValue Op = N->getOperand(OpNo);
5976 APInt Mask = APInt::getLowBitsSet(Op.getValueSizeInBits(), LowBits);
5977 if (!SimplifyDemandedBits(Op, Mask, DCI))
5978 return false;
5979
5980 if (N->getOpcode() != ISD::DELETED_NODE)
5981 DCI.AddToWorklist(N);
5982 return true;
5983 };
5984
5985 switch (N->getOpcode()) {
5986 default:
5987 break;
5988 case RISCVISD::SplitF64: {
5989 SDValue Op0 = N->getOperand(0);
5990 // If the input to SplitF64 is just BuildPairF64 then the operation is
5991 // redundant. Instead, use BuildPairF64's operands directly.
5992 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
5993 return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
5994
5995 SDLoc DL(N);
5996
5997 // It's cheaper to materialise two 32-bit integers than to load a double
5998 // from the constant pool and transfer it to integer registers through the
5999 // stack.
6000 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
6001 APInt V = C->getValueAPF().bitcastToAPInt();
6002 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
6003 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
6004 return DCI.CombineTo(N, Lo, Hi);
6005 }
6006
6007 // This is a target-specific version of a DAGCombine performed in
6008 // DAGCombiner::visitBITCAST. It performs the equivalent of:
6009 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
6010 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
6011 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
6012 !Op0.getNode()->hasOneUse())
6013 break;
6014 SDValue NewSplitF64 =
6015 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
6016 Op0.getOperand(0));
6017 SDValue Lo = NewSplitF64.getValue(0);
6018 SDValue Hi = NewSplitF64.getValue(1);
6019 APInt SignBit = APInt::getSignMask(32);
6020 if (Op0.getOpcode() == ISD::FNEG) {
6021 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
6022 DAG.getConstant(SignBit, DL, MVT::i32));
6023 return DCI.CombineTo(N, Lo, NewHi);
6024 }
6025 assert(Op0.getOpcode() == ISD::FABS);
6026 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
6027 DAG.getConstant(~SignBit, DL, MVT::i32));
6028 return DCI.CombineTo(N, Lo, NewHi);
6029 }
6030 case RISCVISD::SLLW:
6031 case RISCVISD::SRAW:
6032 case RISCVISD::SRLW:
6033 case RISCVISD::ROLW:
6034 case RISCVISD::RORW: {
6035 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
6036 if (SimplifyDemandedLowBitsHelper(0, 32) ||
6037 SimplifyDemandedLowBitsHelper(1, 5))
6038 return SDValue(N, 0);
6039 break;
6040 }
6041 case RISCVISD::CLZW:
6042 case RISCVISD::CTZW: {
6043 // Only the lower 32 bits of the first operand are read
6044 if (SimplifyDemandedLowBitsHelper(0, 32))
6045 return SDValue(N, 0);
6046 break;
6047 }
6048 case RISCVISD::FSL:
6049 case RISCVISD::FSR: {
6050 // Only the lower log2(Bitwidth)+1 bits of the the shift amount are read.
6051 unsigned BitWidth = N->getOperand(2).getValueSizeInBits();
6052 assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
6053 if (SimplifyDemandedLowBitsHelper(2, Log2_32(BitWidth) + 1))
6054 return SDValue(N, 0);
6055 break;
6056 }
6057 case RISCVISD::FSLW:
6058 case RISCVISD::FSRW: {
6059 // Only the lower 32 bits of Values and lower 6 bits of shift amount are
6060 // read.
6061 if (SimplifyDemandedLowBitsHelper(0, 32) ||
6062 SimplifyDemandedLowBitsHelper(1, 32) ||
6063 SimplifyDemandedLowBitsHelper(2, 6))
6064 return SDValue(N, 0);
6065 break;
6066 }
6067 case RISCVISD::GREV:
6068 case RISCVISD::GORC: {
6069 // Only the lower log2(Bitwidth) bits of the the shift amount are read.
6070 unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
6071 assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
6072 if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth)))
6073 return SDValue(N, 0);
6074
6075 return combineGREVI_GORCI(N, DCI.DAG);
6076 }
6077 case RISCVISD::GREVW:
6078 case RISCVISD::GORCW: {
6079 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
6080 if (SimplifyDemandedLowBitsHelper(0, 32) ||
6081 SimplifyDemandedLowBitsHelper(1, 5))
6082 return SDValue(N, 0);
6083
6084 return combineGREVI_GORCI(N, DCI.DAG);
6085 }
6086 case RISCVISD::SHFL:
6087 case RISCVISD::UNSHFL: {
6088 // Only the lower log2(Bitwidth)-1 bits of the the shift amount are read.
6089 unsigned BitWidth = N->getOperand(1).getValueSizeInBits();
6090 assert(isPowerOf2_32(BitWidth) && "Unexpected bit width");
6091 if (SimplifyDemandedLowBitsHelper(1, Log2_32(BitWidth) - 1))
6092 return SDValue(N, 0);
6093
6094 break;
6095 }
6096 case RISCVISD::SHFLW:
6097 case RISCVISD::UNSHFLW: {
6098 // Only the lower 32 bits of LHS and lower 4 bits of RHS are read.
6099 SDValue LHS = N->getOperand(0);
6100 SDValue RHS = N->getOperand(1);
6101 APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
6102 APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 4);
6103 if (SimplifyDemandedLowBitsHelper(0, 32) ||
6104 SimplifyDemandedLowBitsHelper(1, 4))
6105 return SDValue(N, 0);
6106
6107 break;
6108 }
6109 case RISCVISD::BCOMPRESSW:
6110 case RISCVISD::BDECOMPRESSW: {
6111 // Only the lower 32 bits of LHS and RHS are read.
6112 if (SimplifyDemandedLowBitsHelper(0, 32) ||
6113 SimplifyDemandedLowBitsHelper(1, 32))
6114 return SDValue(N, 0);
6115
6116 break;
6117 }
6118 case RISCVISD::FMV_X_ANYEXTH:
6119 case RISCVISD::FMV_X_ANYEXTW_RV64: {
6120 SDLoc DL(N);
6121 SDValue Op0 = N->getOperand(0);
6122 MVT VT = N->getSimpleValueType(0);
6123 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
6124 // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
6125 // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
6126 if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
6127 Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
6128 (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
6129 Op0->getOpcode() == RISCVISD::FMV_H_X)) {
6130 assert(Op0.getOperand(0).getValueType() == VT &&
6131 "Unexpected value type!");
6132 return Op0.getOperand(0);
6133 }
6134
6135 // This is a target-specific version of a DAGCombine performed in
6136 // DAGCombiner::visitBITCAST. It performs the equivalent of:
6137 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
6138 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
6139 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
6140 !Op0.getNode()->hasOneUse())
6141 break;
6142 SDValue NewFMV = DAG.getNode(N->getOpcode(), DL, VT, Op0.getOperand(0));
6143 unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
6144 APInt SignBit = APInt::getSignMask(FPBits).sextOrSelf(VT.getSizeInBits());
6145 if (Op0.getOpcode() == ISD::FNEG)
6146 return DAG.getNode(ISD::XOR, DL, VT, NewFMV,
6147 DAG.getConstant(SignBit, DL, VT));
6148
6149 assert(Op0.getOpcode() == ISD::FABS);
6150 return DAG.getNode(ISD::AND, DL, VT, NewFMV,
6151 DAG.getConstant(~SignBit, DL, VT));
6152 }
6153 case ISD::ADD:
6154 return performADDCombine(N, DAG);
6155 case ISD::SUB:
6156 return performSUBCombine(N, DAG);
6157 case ISD::AND:
6158 return performANDCombine(N, DAG);
6159 case ISD::OR:
6160 return performORCombine(N, DAG, Subtarget);
6161 case ISD::XOR:
6162 return performXORCombine(N, DAG);
6163 case ISD::ANY_EXTEND:
6164 return performANY_EXTENDCombine(N, DCI, Subtarget);
6165 case ISD::ZERO_EXTEND:
6166 // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
6167 // type legalization. This is safe because fp_to_uint produces poison if
6168 // it overflows.
6169 if (N->getValueType(0) == MVT::i64 && Subtarget.is64Bit() &&
6170 N->getOperand(0).getOpcode() == ISD::FP_TO_UINT &&
6171 isTypeLegal(N->getOperand(0).getOperand(0).getValueType()))
6172 return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), MVT::i64,
6173 N->getOperand(0).getOperand(0));
6174 return SDValue();
6175 case RISCVISD::SELECT_CC: {
6176 // Transform
6177 SDValue LHS = N->getOperand(0);
6178 SDValue RHS = N->getOperand(1);
6179 ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(2))->get();
6180 if (!ISD::isIntEqualitySetCC(CCVal))
6181 break;
6182
6183 // Fold (select_cc (setlt X, Y), 0, ne, trueV, falseV) ->
6184 // (select_cc X, Y, lt, trueV, falseV)
6185 // Sometimes the setcc is introduced after select_cc has been formed.
6186 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
6187 LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
6188 // If we're looking for eq 0 instead of ne 0, we need to invert the
6189 // condition.
6190 bool Invert = CCVal == ISD::SETEQ;
6191 CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
6192 if (Invert)
6193 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
6194
6195 SDLoc DL(N);
6196 RHS = LHS.getOperand(1);
6197 LHS = LHS.getOperand(0);
6198 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
6199
6200 SDValue TargetCC = DAG.getCondCode(CCVal);
6201 return DAG.getNode(
6202 RISCVISD::SELECT_CC, DL, N->getValueType(0),
6203 {LHS, RHS, TargetCC, N->getOperand(3), N->getOperand(4)});
6204 }
6205
6206 // Fold (select_cc (xor X, Y), 0, eq/ne, trueV, falseV) ->
6207 // (select_cc X, Y, eq/ne, trueV, falseV)
6208 if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
6209 return DAG.getNode(RISCVISD::SELECT_CC, SDLoc(N), N->getValueType(0),
6210 {LHS.getOperand(0), LHS.getOperand(1),
6211 N->getOperand(2), N->getOperand(3),
6212 N->getOperand(4)});
6213 // (select_cc X, 1, setne, trueV, falseV) ->
6214 // (select_cc X, 0, seteq, trueV, falseV) if we can prove X is 0/1.
6215 // This can occur when legalizing some floating point comparisons.
6216 APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
6217 if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
6218 SDLoc DL(N);
6219 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
6220 SDValue TargetCC = DAG.getCondCode(CCVal);
6221 RHS = DAG.getConstant(0, DL, LHS.getValueType());
6222 return DAG.getNode(
6223 RISCVISD::SELECT_CC, DL, N->getValueType(0),
6224 {LHS, RHS, TargetCC, N->getOperand(3), N->getOperand(4)});
6225 }
6226
6227 break;
6228 }
6229 case RISCVISD::BR_CC: {
6230 SDValue LHS = N->getOperand(1);
6231 SDValue RHS = N->getOperand(2);
6232 ISD::CondCode CCVal = cast<CondCodeSDNode>(N->getOperand(3))->get();
6233 if (!ISD::isIntEqualitySetCC(CCVal))
6234 break;
6235
6236 // Fold (br_cc (setlt X, Y), 0, ne, dest) ->
6237 // (br_cc X, Y, lt, dest)
6238 // Sometimes the setcc is introduced after br_cc has been formed.
6239 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
6240 LHS.getOperand(0).getValueType() == Subtarget.getXLenVT()) {
6241 // If we're looking for eq 0 instead of ne 0, we need to invert the
6242 // condition.
6243 bool Invert = CCVal == ISD::SETEQ;
6244 CCVal = cast<CondCodeSDNode>(LHS.getOperand(2))->get();
6245 if (Invert)
6246 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
6247
6248 SDLoc DL(N);
6249 RHS = LHS.getOperand(1);
6250 LHS = LHS.getOperand(0);
6251 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
6252
6253 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
6254 N->getOperand(0), LHS, RHS, DAG.getCondCode(CCVal),
6255 N->getOperand(4));
6256 }
6257
6258 // Fold (br_cc (xor X, Y), 0, eq/ne, dest) ->
6259 // (br_cc X, Y, eq/ne, trueV, falseV)
6260 if (LHS.getOpcode() == ISD::XOR && isNullConstant(RHS))
6261 return DAG.getNode(RISCVISD::BR_CC, SDLoc(N), N->getValueType(0),
6262 N->getOperand(0), LHS.getOperand(0), LHS.getOperand(1),
6263 N->getOperand(3), N->getOperand(4));
6264
6265 // (br_cc X, 1, setne, br_cc) ->
6266 // (br_cc X, 0, seteq, br_cc) if we can prove X is 0/1.
6267 // This can occur when legalizing some floating point comparisons.
6268 APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
6269 if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
6270 SDLoc DL(N);
6271 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
6272 SDValue TargetCC = DAG.getCondCode(CCVal);
6273 RHS = DAG.getConstant(0, DL, LHS.getValueType());
6274 return DAG.getNode(RISCVISD::BR_CC, DL, N->getValueType(0),
6275 N->getOperand(0), LHS, RHS, TargetCC,
6276 N->getOperand(4));
6277 }
6278 break;
6279 }
6280 case ISD::FCOPYSIGN: {
6281 EVT VT = N->getValueType(0);
6282 if (!VT.isVector())
6283 break;
6284 // There is a form of VFSGNJ which injects the negated sign of its second
6285 // operand. Try and bubble any FNEG up after the extend/round to produce
6286 // this optimized pattern. Avoid modifying cases where FP_ROUND and
6287 // TRUNC=1.
6288 SDValue In2 = N->getOperand(1);
6289 // Avoid cases where the extend/round has multiple uses, as duplicating
6290 // those is typically more expensive than removing a fneg.
6291 if (!In2.hasOneUse())
6292 break;
6293 if (In2.getOpcode() != ISD::FP_EXTEND &&
6294 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(1) != 0))
6295 break;
6296 In2 = In2.getOperand(0);
6297 if (In2.getOpcode() != ISD::FNEG)
6298 break;
6299 SDLoc DL(N);
6300 SDValue NewFPExtRound = DAG.getFPExtendOrRound(In2.getOperand(0), DL, VT);
6301 return DAG.getNode(ISD::FCOPYSIGN, DL, VT, N->getOperand(0),
6302 DAG.getNode(ISD::FNEG, DL, VT, NewFPExtRound));
6303 }
6304 case ISD::MGATHER:
6305 case ISD::MSCATTER: {
6306 if (!DCI.isBeforeLegalize())
6307 break;
6308 MaskedGatherScatterSDNode *MGSN = cast<MaskedGatherScatterSDNode>(N);
6309 SDValue Index = MGSN->getIndex();
6310 EVT IndexVT = Index.getValueType();
6311 MVT XLenVT = Subtarget.getXLenVT();
6312 // RISCV indexed loads only support the "unsigned unscaled" addressing
6313 // mode, so anything else must be manually legalized.
6314 bool NeedsIdxLegalization = MGSN->isIndexScaled() ||
6315 (MGSN->isIndexSigned() &&
6316 IndexVT.getVectorElementType().bitsLT(XLenVT));
6317 if (!NeedsIdxLegalization)
6318 break;
6319
6320 SDLoc DL(N);
6321
6322 // Any index legalization should first promote to XLenVT, so we don't lose
6323 // bits when scaling. This may create an illegal index type so we let
6324 // LLVM's legalization take care of the splitting.
6325 if (IndexVT.getVectorElementType().bitsLT(XLenVT)) {
6326 IndexVT = IndexVT.changeVectorElementType(XLenVT);
6327 Index = DAG.getNode(MGSN->isIndexSigned() ? ISD::SIGN_EXTEND
6328 : ISD::ZERO_EXTEND,
6329 DL, IndexVT, Index);
6330 }
6331
6332 unsigned Scale = N->getConstantOperandVal(5);
6333 if (MGSN->isIndexScaled() && Scale != 1) {
6334 // Manually scale the indices by the element size.
6335 // TODO: Sanitize the scale operand here?
6336 assert(isPowerOf2_32(Scale) && "Expecting power-of-two types");
6337 SDValue SplatScale = DAG.getConstant(Log2_32(Scale), DL, IndexVT);
6338 Index = DAG.getNode(ISD::SHL, DL, IndexVT, Index, SplatScale);
6339 }
6340
6341 ISD::MemIndexType NewIndexTy = ISD::UNSIGNED_UNSCALED;
6342 if (const auto *MGN = dyn_cast<MaskedGatherSDNode>(N)) {
6343 return DAG.getMaskedGather(
6344 N->getVTList(), MGSN->getMemoryVT(), DL,
6345 {MGSN->getChain(), MGN->getPassThru(), MGSN->getMask(),
6346 MGSN->getBasePtr(), Index, MGN->getScale()},
6347 MGN->getMemOperand(), NewIndexTy, MGN->getExtensionType());
6348 }
6349 const auto *MSN = cast<MaskedScatterSDNode>(N);
6350 return DAG.getMaskedScatter(
6351 N->getVTList(), MGSN->getMemoryVT(), DL,
6352 {MGSN->getChain(), MSN->getValue(), MGSN->getMask(), MGSN->getBasePtr(),
6353 Index, MGSN->getScale()},
6354 MGSN->getMemOperand(), NewIndexTy, MSN->isTruncatingStore());
6355 }
6356 case RISCVISD::SRA_VL:
6357 case RISCVISD::SRL_VL:
6358 case RISCVISD::SHL_VL: {
6359 SDValue ShAmt = N->getOperand(1);
6360 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
6361 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
6362 SDLoc DL(N);
6363 SDValue VL = N->getOperand(3);
6364 EVT VT = N->getValueType(0);
6365 ShAmt =
6366 DAG.getNode(RISCVISD::VMV_V_X_VL, DL, VT, ShAmt.getOperand(0), VL);
6367 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt,
6368 N->getOperand(2), N->getOperand(3));
6369 }
6370 break;
6371 }
6372 case ISD::SRA:
6373 case ISD::SRL:
6374 case ISD::SHL: {
6375 SDValue ShAmt = N->getOperand(1);
6376 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
6377 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
6378 SDLoc DL(N);
6379 EVT VT = N->getValueType(0);
6380 ShAmt =
6381 DAG.getNode(RISCVISD::SPLAT_VECTOR_I64, DL, VT, ShAmt.getOperand(0));
6382 return DAG.getNode(N->getOpcode(), DL, VT, N->getOperand(0), ShAmt);
6383 }
6384 break;
6385 }
6386 case RISCVISD::MUL_VL: {
6387 // Try to form VWMUL or VWMULU.
6388 // FIXME: Look for splat of extended scalar as well.
6389 // FIXME: Support VWMULSU.
6390 SDValue Op0 = N->getOperand(0);
6391 SDValue Op1 = N->getOperand(1);
6392 bool IsSignExt = Op0.getOpcode() == RISCVISD::VSEXT_VL;
6393 bool IsZeroExt = Op0.getOpcode() == RISCVISD::VZEXT_VL;
6394 if ((!IsSignExt && !IsZeroExt) || Op0.getOpcode() != Op1.getOpcode())
6395 return SDValue();
6396
6397 // Make sure the extends have a single use.
6398 if (!Op0.hasOneUse() || !Op1.hasOneUse())
6399 return SDValue();
6400
6401 SDValue Mask = N->getOperand(2);
6402 SDValue VL = N->getOperand(3);
6403 if (Op0.getOperand(1) != Mask || Op1.getOperand(1) != Mask ||
6404 Op0.getOperand(2) != VL || Op1.getOperand(2) != VL)
6405 return SDValue();
6406
6407 Op0 = Op0.getOperand(0);
6408 Op1 = Op1.getOperand(0);
6409
6410 MVT VT = N->getSimpleValueType(0);
6411 MVT NarrowVT =
6412 MVT::getVectorVT(MVT::getIntegerVT(VT.getScalarSizeInBits() / 2),
6413 VT.getVectorElementCount());
6414
6415 SDLoc DL(N);
6416
6417 // Re-introduce narrower extends if needed.
6418 unsigned ExtOpc = IsSignExt ? RISCVISD::VSEXT_VL : RISCVISD::VZEXT_VL;
6419 if (Op0.getValueType() != NarrowVT)
6420 Op0 = DAG.getNode(ExtOpc, DL, NarrowVT, Op0, Mask, VL);
6421 if (Op1.getValueType() != NarrowVT)
6422 Op1 = DAG.getNode(ExtOpc, DL, NarrowVT, Op1, Mask, VL);
6423
6424 unsigned WMulOpc = IsSignExt ? RISCVISD::VWMUL_VL : RISCVISD::VWMULU_VL;
6425 return DAG.getNode(WMulOpc, DL, VT, Op0, Op1, Mask, VL);
6426 }
6427 }
6428
6429 return SDValue();
6430 }
6431
6432 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
6433 const SDNode *N, CombineLevel Level) const {
6434 // The following folds are only desirable if `(OP _, c1 << c2)` can be
6435 // materialised in fewer instructions than `(OP _, c1)`:
6436 //
6437 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
6438 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
6439 SDValue N0 = N->getOperand(0);
6440 EVT Ty = N0.getValueType();
6441 if (Ty.isScalarInteger() &&
6442 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
6443 auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
6444 auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
6445 if (C1 && C2) {
6446 const APInt &C1Int = C1->getAPIntValue();
6447 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
6448
6449 // We can materialise `c1 << c2` into an add immediate, so it's "free",
6450 // and the combine should happen, to potentially allow further combines
6451 // later.
6452 if (ShiftedC1Int.getMinSignedBits() <= 64 &&
6453 isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
6454 return true;
6455
6456 // We can materialise `c1` in an add immediate, so it's "free", and the
6457 // combine should be prevented.
6458 if (C1Int.getMinSignedBits() <= 64 &&
6459 isLegalAddImmediate(C1Int.getSExtValue()))
6460 return false;
6461
6462 // Neither constant will fit into an immediate, so find materialisation
6463 // costs.
6464 int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
6465 Subtarget.getFeatureBits(),
6466 /*CompressionCost*/true);
6467 int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
6468 ShiftedC1Int, Ty.getSizeInBits(), Subtarget.getFeatureBits(),
6469 /*CompressionCost*/true);
6470
6471 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
6472 // combine should be prevented.
6473 if (C1Cost < ShiftedC1Cost)
6474 return false;
6475 }
6476 }
6477 return true;
6478 }
6479
6480 bool RISCVTargetLowering::targetShrinkDemandedConstant(
6481 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
6482 TargetLoweringOpt &TLO) const {
6483 // Delay this optimization as late as possible.
6484 if (!TLO.LegalOps)
6485 return false;
6486
6487 EVT VT = Op.getValueType();
6488 if (VT.isVector())
6489 return false;
6490
6491 // Only handle AND for now.
6492 if (Op.getOpcode() != ISD::AND)
6493 return false;
6494
6495 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6496 if (!C)
6497 return false;
6498
6499 const APInt &Mask = C->getAPIntValue();
6500
6501 // Clear all non-demanded bits initially.
6502 APInt ShrunkMask = Mask & DemandedBits;
6503
6504 // Try to make a smaller immediate by setting undemanded bits.
6505
6506 APInt ExpandedMask = Mask | ~DemandedBits;
6507
6508 auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
6509 return ShrunkMask.isSubsetOf(Mask) && Mask.isSubsetOf(ExpandedMask);
6510 };
6511 auto UseMask = [Mask, Op, VT, &TLO](const APInt &NewMask) -> bool {
6512 if (NewMask == Mask)
6513 return true;
6514 SDLoc DL(Op);
6515 SDValue NewC = TLO.DAG.getConstant(NewMask, DL, VT);
6516 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
6517 return TLO.CombineTo(Op, NewOp);
6518 };
6519
6520 // If the shrunk mask fits in sign extended 12 bits, let the target
6521 // independent code apply it.
6522 if (ShrunkMask.isSignedIntN(12))
6523 return false;
6524
6525 // Preserve (and X, 0xffff) when zext.h is supported.
6526 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbp()) {
6527 APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
6528 if (IsLegalMask(NewMask))
6529 return UseMask(NewMask);
6530 }
6531
6532 // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
6533 if (VT == MVT::i64) {
6534 APInt NewMask = APInt(64, 0xffffffff);
6535 if (IsLegalMask(NewMask))
6536 return UseMask(NewMask);
6537 }
6538
6539 // For the remaining optimizations, we need to be able to make a negative
6540 // number through a combination of mask and undemanded bits.
6541 if (!ExpandedMask.isNegative())
6542 return false;
6543
6544 // What is the fewest number of bits we need to represent the negative number.
6545 unsigned MinSignedBits = ExpandedMask.getMinSignedBits();
6546
6547 // Try to make a 12 bit negative immediate. If that fails try to make a 32
6548 // bit negative immediate unless the shrunk immediate already fits in 32 bits.
6549 APInt NewMask = ShrunkMask;
6550 if (MinSignedBits <= 12)
6551 NewMask.setBitsFrom(11);
6552 else if (MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(32))
6553 NewMask.setBitsFrom(31);
6554 else
6555 return false;
6556
6557 // Sanity check that our new mask is a subset of the demanded mask.
6558 assert(IsLegalMask(NewMask));
6559 return UseMask(NewMask);
6560 }
6561
6562 static void computeGREV(APInt &Src, unsigned ShAmt) {
6563 ShAmt &= Src.getBitWidth() - 1;
6564 uint64_t x = Src.getZExtValue();
6565 if (ShAmt & 1)
6566 x = ((x & 0x5555555555555555LL) << 1) | ((x & 0xAAAAAAAAAAAAAAAALL) >> 1);
6567 if (ShAmt & 2)
6568 x = ((x & 0x3333333333333333LL) << 2) | ((x & 0xCCCCCCCCCCCCCCCCLL) >> 2);
6569 if (ShAmt & 4)
6570 x = ((x & 0x0F0F0F0F0F0F0F0FLL) << 4) | ((x & 0xF0F0F0F0F0F0F0F0LL) >> 4);
6571 if (ShAmt & 8)
6572 x = ((x & 0x00FF00FF00FF00FFLL) << 8) | ((x & 0xFF00FF00FF00FF00LL) >> 8);
6573 if (ShAmt & 16)
6574 x = ((x & 0x0000FFFF0000FFFFLL) << 16) | ((x & 0xFFFF0000FFFF0000LL) >> 16);
6575 if (ShAmt & 32)
6576 x = ((x & 0x00000000FFFFFFFFLL) << 32) | ((x & 0xFFFFFFFF00000000LL) >> 32);
6577 Src = x;
6578 }
6579
6580 void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
6581 KnownBits &Known,
6582 const APInt &DemandedElts,
6583 const SelectionDAG &DAG,
6584 unsigned Depth) const {
6585 unsigned BitWidth = Known.getBitWidth();
6586 unsigned Opc = Op.getOpcode();
6587 assert((Opc >= ISD::BUILTIN_OP_END ||
6588 Opc == ISD::INTRINSIC_WO_CHAIN ||
6589 Opc == ISD::INTRINSIC_W_CHAIN ||
6590 Opc == ISD::INTRINSIC_VOID) &&
6591 "Should use MaskedValueIsZero if you don't know whether Op"
6592 " is a target node!");
6593
6594 Known.resetAll();
6595 switch (Opc) {
6596 default: break;
6597 case RISCVISD::SELECT_CC: {
6598 Known = DAG.computeKnownBits(Op.getOperand(4), Depth + 1);
6599 // If we don't know any bits, early out.
6600 if (Known.isUnknown())
6601 break;
6602 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(3), Depth + 1);
6603
6604 // Only known if known in both the LHS and RHS.
6605 Known = KnownBits::commonBits(Known, Known2);
6606 break;
6607 }
6608 case RISCVISD::REMUW: {
6609 KnownBits Known2;
6610 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
6611 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
6612 // We only care about the lower 32 bits.
6613 Known = KnownBits::urem(Known.trunc(32), Known2.trunc(32));
6614 // Restore the original width by sign extending.
6615 Known = Known.sext(BitWidth);
6616 break;
6617 }
6618 case RISCVISD::DIVUW: {
6619 KnownBits Known2;
6620 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
6621 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
6622 // We only care about the lower 32 bits.
6623 Known = KnownBits::udiv(Known.trunc(32), Known2.trunc(32));
6624 // Restore the original width by sign extending.
6625 Known = Known.sext(BitWidth);
6626 break;
6627 }
6628 case RISCVISD::CTZW: {
6629 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
6630 unsigned PossibleTZ = Known2.trunc(32).countMaxTrailingZeros();
6631 unsigned LowBits = Log2_32(PossibleTZ) + 1;
6632 Known.Zero.setBitsFrom(LowBits);
6633 break;
6634 }
6635 case RISCVISD::CLZW: {
6636 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
6637 unsigned PossibleLZ = Known2.trunc(32).countMaxLeadingZeros();
6638 unsigned LowBits = Log2_32(PossibleLZ) + 1;
6639 Known.Zero.setBitsFrom(LowBits);
6640 break;
6641 }
6642 case RISCVISD::GREV:
6643 case RISCVISD::GREVW: {
6644 if (auto *C = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
6645 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
6646 if (Opc == RISCVISD::GREVW)
6647 Known = Known.trunc(32);
6648 unsigned ShAmt = C->getZExtValue();
6649 computeGREV(Known.Zero, ShAmt);
6650 computeGREV(Known.One, ShAmt);
6651 if (Opc == RISCVISD::GREVW)
6652 Known = Known.sext(BitWidth);
6653 }
6654 break;
6655 }
6656 case RISCVISD::READ_VLENB:
6657 // We assume VLENB is at least 16 bytes.
6658 Known.Zero.setLowBits(4);
6659 // We assume VLENB is no more than 65536 / 8 bytes.
6660 Known.Zero.setBitsFrom(14);
6661 break;
6662 case ISD::INTRINSIC_W_CHAIN: {
6663 unsigned IntNo = Op.getConstantOperandVal(1);
6664 switch (IntNo) {
6665 default:
6666 // We can't do anything for most intrinsics.
6667 break;
6668 case Intrinsic::riscv_vsetvli:
6669 case Intrinsic::riscv_vsetvlimax:
6670 // Assume that VL output is positive and would fit in an int32_t.
6671 // TODO: VLEN might be capped at 16 bits in a future V spec update.
6672 if (BitWidth >= 32)
6673 Known.Zero.setBitsFrom(31);
6674 break;
6675 }
6676 break;
6677 }
6678 }
6679 }
6680
6681 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
6682 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
6683 unsigned Depth) const {
6684 switch (Op.getOpcode()) {
6685 default:
6686 break;
6687 case RISCVISD::SELECT_CC: {
6688 unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth + 1);
6689 if (Tmp == 1) return 1; // Early out.
6690 unsigned Tmp2 = DAG.ComputeNumSignBits(Op.getOperand(4), DemandedElts, Depth + 1);
6691 return std::min(Tmp, Tmp2);
6692 }
6693 case RISCVISD::SLLW:
6694 case RISCVISD::SRAW:
6695 case RISCVISD::SRLW:
6696 case RISCVISD::DIVW:
6697 case RISCVISD::DIVUW:
6698 case RISCVISD::REMUW:
6699 case RISCVISD::ROLW:
6700 case RISCVISD::RORW:
6701 case RISCVISD::GREVW:
6702 case RISCVISD::GORCW:
6703 case RISCVISD::FSLW:
6704 case RISCVISD::FSRW:
6705 case RISCVISD::SHFLW:
6706 case RISCVISD::UNSHFLW:
6707 case RISCVISD::BCOMPRESSW:
6708 case RISCVISD::BDECOMPRESSW:
6709 case RISCVISD::FCVT_W_RTZ_RV64:
6710 case RISCVISD::FCVT_WU_RTZ_RV64:
6711 // TODO: As the result is sign-extended, this is conservatively correct. A
6712 // more precise answer could be calculated for SRAW depending on known
6713 // bits in the shift amount.
6714 return 33;
6715 case RISCVISD::SHFL:
6716 case RISCVISD::UNSHFL: {
6717 // There is no SHFLIW, but a i64 SHFLI with bit 4 of the control word
6718 // cleared doesn't affect bit 31. The upper 32 bits will be shuffled, but
6719 // will stay within the upper 32 bits. If there were more than 32 sign bits
6720 // before there will be at least 33 sign bits after.
6721 if (Op.getValueType() == MVT::i64 &&
6722 isa<ConstantSDNode>(Op.getOperand(1)) &&
6723 (Op.getConstantOperandVal(1) & 0x10) == 0) {
6724 unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
6725 if (Tmp > 32)
6726 return 33;
6727 }
6728 break;
6729 }
6730 case RISCVISD::VMV_X_S:
6731 // The number of sign bits of the scalar result is computed by obtaining the
6732 // element type of the input vector operand, subtracting its width from the
6733 // XLEN, and then adding one (sign bit within the element type). If the
6734 // element type is wider than XLen, the least-significant XLEN bits are
6735 // taken.
6736 if (Op.getOperand(0).getScalarValueSizeInBits() > Subtarget.getXLen())
6737 return 1;
6738 return Subtarget.getXLen() - Op.getOperand(0).getScalarValueSizeInBits() + 1;
6739 }
6740
6741 return 1;
6742 }
6743
6744 static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
6745 MachineBasicBlock *BB) {
6746 assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
6747
6748 // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
6749 // Should the count have wrapped while it was being read, we need to try
6750 // again.
6751 // ...
6752 // read:
6753 // rdcycleh x3 # load high word of cycle
6754 // rdcycle x2 # load low word of cycle
6755 // rdcycleh x4 # load high word of cycle
6756 // bne x3, x4, read # check if high word reads match, otherwise try again
6757 // ...
6758
6759 MachineFunction &MF = *BB->getParent();
6760 const BasicBlock *LLVM_BB = BB->getBasicBlock();
6761 MachineFunction::iterator It = ++BB->getIterator();
6762
6763 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
6764 MF.insert(It, LoopMBB);
6765
6766 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
6767 MF.insert(It, DoneMBB);
6768
6769 // Transfer the remainder of BB and its successor edges to DoneMBB.
6770 DoneMBB->splice(DoneMBB->begin(), BB,
6771 std::next(MachineBasicBlock::iterator(MI)), BB->end());
6772 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
6773
6774 BB->addSuccessor(LoopMBB);
6775
6776 MachineRegisterInfo &RegInfo = MF.getRegInfo();
6777 Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
6778 Register LoReg = MI.getOperand(0).getReg();
6779 Register HiReg = MI.getOperand(1).getReg();
6780 DebugLoc DL = MI.getDebugLoc();
6781
6782 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
6783 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
6784 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
6785 .addReg(RISCV::X0);
6786 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
6787 .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
6788 .addReg(RISCV::X0);
6789 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
6790 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
6791 .addReg(RISCV::X0);
6792
6793 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
6794 .addReg(HiReg)
6795 .addReg(ReadAgainReg)
6796 .addMBB(LoopMBB);
6797
6798 LoopMBB->addSuccessor(LoopMBB);
6799 LoopMBB->addSuccessor(DoneMBB);
6800
6801 MI.eraseFromParent();
6802
6803 return DoneMBB;
6804 }
6805
6806 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
6807 MachineBasicBlock *BB) {
6808 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
6809
6810 MachineFunction &MF = *BB->getParent();
6811 DebugLoc DL = MI.getDebugLoc();
6812 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
6813 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
6814 Register LoReg = MI.getOperand(0).getReg();
6815 Register HiReg = MI.getOperand(1).getReg();
6816 Register SrcReg = MI.getOperand(2).getReg();
6817 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
6818 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
6819
6820 TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
6821 RI);
6822 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
6823 MachineMemOperand *MMOLo =
6824 MF.getMachineMemOperand(MPI, MachineMemOperand::MOLoad, 4, Align(8));
6825 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
6826 MPI.getWithOffset(4), MachineMemOperand::MOLoad, 4, Align(8));
6827 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
6828 .addFrameIndex(FI)
6829 .addImm(0)
6830 .addMemOperand(MMOLo);
6831 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
6832 .addFrameIndex(FI)
6833 .addImm(4)
6834 .addMemOperand(MMOHi);
6835 MI.eraseFromParent(); // The pseudo instruction is gone now.
6836 return BB;
6837 }
6838
6839 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
6840 MachineBasicBlock *BB) {
6841 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
6842 "Unexpected instruction");
6843
6844 MachineFunction &MF = *BB->getParent();
6845 DebugLoc DL = MI.getDebugLoc();
6846 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
6847 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
6848 Register DstReg = MI.getOperand(0).getReg();
6849 Register LoReg = MI.getOperand(1).getReg();
6850 Register HiReg = MI.getOperand(2).getReg();
6851 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
6852 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
6853
6854 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
6855 MachineMemOperand *MMOLo =
6856 MF.getMachineMemOperand(MPI, MachineMemOperand::MOStore, 4, Align(8));
6857 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
6858 MPI.getWithOffset(4), MachineMemOperand::MOStore, 4, Align(8));
6859 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
6860 .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
6861 .addFrameIndex(FI)
6862 .addImm(0)
6863 .addMemOperand(MMOLo);
6864 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
6865 .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
6866 .addFrameIndex(FI)
6867 .addImm(4)
6868 .addMemOperand(MMOHi);
6869 TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
6870 MI.eraseFromParent(); // The pseudo instruction is gone now.
6871 return BB;
6872 }
6873
6874 static bool isSelectPseudo(MachineInstr &MI) {
6875 switch (MI.getOpcode()) {
6876 default:
6877 return false;
6878 case RISCV::Select_GPR_Using_CC_GPR:
6879 case RISCV::Select_FPR16_Using_CC_GPR:
6880 case RISCV::Select_FPR32_Using_CC_GPR:
6881 case RISCV::Select_FPR64_Using_CC_GPR:
6882 return true;
6883 }
6884 }
6885
6886 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
6887 MachineBasicBlock *BB,
6888 const RISCVSubtarget &Subtarget) {
6889 // To "insert" Select_* instructions, we actually have to insert the triangle
6890 // control-flow pattern. The incoming instructions know the destination vreg
6891 // to set, the condition code register to branch on, the true/false values to
6892 // select between, and the condcode to use to select the appropriate branch.
6893 //
6894 // We produce the following control flow:
6895 // HeadMBB
6896 // | \
6897 // | IfFalseMBB
6898 // | /
6899 // TailMBB
6900 //
6901 // When we find a sequence of selects we attempt to optimize their emission
6902 // by sharing the control flow. Currently we only handle cases where we have
6903 // multiple selects with the exact same condition (same LHS, RHS and CC).
6904 // The selects may be interleaved with other instructions if the other
6905 // instructions meet some requirements we deem safe:
6906 // - They are debug instructions. Otherwise,
6907 // - They do not have side-effects, do not access memory and their inputs do
6908 // not depend on the results of the select pseudo-instructions.
6909 // The TrueV/FalseV operands of the selects cannot depend on the result of
6910 // previous selects in the sequence.
6911 // These conditions could be further relaxed. See the X86 target for a
6912 // related approach and more information.
6913 Register LHS = MI.getOperand(1).getReg();
6914 Register RHS = MI.getOperand(2).getReg();
6915 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(3).getImm());
6916
6917 SmallVector<MachineInstr *, 4> SelectDebugValues;
6918 SmallSet<Register, 4> SelectDests;
6919 SelectDests.insert(MI.getOperand(0).getReg());
6920
6921 MachineInstr *LastSelectPseudo = &MI;
6922
6923 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
6924 SequenceMBBI != E; ++SequenceMBBI) {
6925 if (SequenceMBBI->isDebugInstr())
6926 continue;
6927 else if (isSelectPseudo(*SequenceMBBI)) {
6928 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
6929 SequenceMBBI->getOperand(2).getReg() != RHS ||
6930 SequenceMBBI->getOperand(3).getImm() != CC ||
6931 SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
6932 SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
6933 break;
6934 LastSelectPseudo = &*SequenceMBBI;
6935 SequenceMBBI->collectDebugValues(SelectDebugValues);
6936 SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
6937 } else {
6938 if (SequenceMBBI->hasUnmodeledSideEffects() ||
6939 SequenceMBBI->mayLoadOrStore())
6940 break;
6941 if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
6942 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
6943 }))
6944 break;
6945 }
6946 }
6947
6948 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
6949 const BasicBlock *LLVM_BB = BB->getBasicBlock();
6950 DebugLoc DL = MI.getDebugLoc();
6951 MachineFunction::iterator I = ++BB->getIterator();
6952
6953 MachineBasicBlock *HeadMBB = BB;
6954 MachineFunction *F = BB->getParent();
6955 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
6956 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
6957
6958 F->insert(I, IfFalseMBB);
6959 F->insert(I, TailMBB);
6960
6961 // Transfer debug instructions associated with the selects to TailMBB.
6962 for (MachineInstr *DebugInstr : SelectDebugValues) {
6963 TailMBB->push_back(DebugInstr->removeFromParent());
6964 }
6965
6966 // Move all instructions after the sequence to TailMBB.
6967 TailMBB->splice(TailMBB->end(), HeadMBB,
6968 std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
6969 // Update machine-CFG edges by transferring all successors of the current
6970 // block to the new block which will contain the Phi nodes for the selects.
6971 TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
6972 // Set the successors for HeadMBB.
6973 HeadMBB->addSuccessor(IfFalseMBB);
6974 HeadMBB->addSuccessor(TailMBB);
6975
6976 // Insert appropriate branch.
6977 BuildMI(HeadMBB, DL, TII.getBrCond(CC))
6978 .addReg(LHS)
6979 .addReg(RHS)
6980 .addMBB(TailMBB);
6981
6982 // IfFalseMBB just falls through to TailMBB.
6983 IfFalseMBB->addSuccessor(TailMBB);
6984
6985 // Create PHIs for all of the select pseudo-instructions.
6986 auto SelectMBBI = MI.getIterator();
6987 auto SelectEnd = std::next(LastSelectPseudo->getIterator());
6988 auto InsertionPoint = TailMBB->begin();
6989 while (SelectMBBI != SelectEnd) {
6990 auto Next = std::next(SelectMBBI);
6991 if (isSelectPseudo(*SelectMBBI)) {
6992 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
6993 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
6994 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
6995 .addReg(SelectMBBI->getOperand(4).getReg())
6996 .addMBB(HeadMBB)
6997 .addReg(SelectMBBI->getOperand(5).getReg())
6998 .addMBB(IfFalseMBB);
6999 SelectMBBI->eraseFromParent();
7000 }
7001 SelectMBBI = Next;
7002 }
7003
7004 F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
7005 return TailMBB;
7006 }
7007
7008 MachineBasicBlock *
7009 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
7010 MachineBasicBlock *BB) const {
7011 switch (MI.getOpcode()) {
7012 default:
7013 llvm_unreachable("Unexpected instr type to insert");
7014 case RISCV::ReadCycleWide:
7015 assert(!Subtarget.is64Bit() &&
7016 "ReadCycleWrite is only to be used on riscv32");
7017 return emitReadCycleWidePseudo(MI, BB);
7018 case RISCV::Select_GPR_Using_CC_GPR:
7019 case RISCV::Select_FPR16_Using_CC_GPR:
7020 case RISCV::Select_FPR32_Using_CC_GPR:
7021 case RISCV::Select_FPR64_Using_CC_GPR:
7022 return emitSelectPseudo(MI, BB, Subtarget);
7023 case RISCV::BuildPairF64Pseudo:
7024 return emitBuildPairF64Pseudo(MI, BB);
7025 case RISCV::SplitF64Pseudo:
7026 return emitSplitF64Pseudo(MI, BB);
7027 }
7028 }
7029
7030 // Calling Convention Implementation.
7031 // The expectations for frontend ABI lowering vary from target to target.
7032 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
7033 // details, but this is a longer term goal. For now, we simply try to keep the
7034 // role of the frontend as simple and well-defined as possible. The rules can
7035 // be summarised as:
7036 // * Never split up large scalar arguments. We handle them here.
7037 // * If a hardfloat calling convention is being used, and the struct may be
7038 // passed in a pair of registers (fp+fp, int+fp), and both registers are
7039 // available, then pass as two separate arguments. If either the GPRs or FPRs
7040 // are exhausted, then pass according to the rule below.
7041 // * If a struct could never be passed in registers or directly in a stack
7042 // slot (as it is larger than 2*XLEN and the floating point rules don't
7043 // apply), then pass it using a pointer with the byval attribute.
7044 // * If a struct is less than 2*XLEN, then coerce to either a two-element
7045 // word-sized array or a 2*XLEN scalar (depending on alignment).
7046 // * The frontend can determine whether a struct is returned by reference or
7047 // not based on its size and fields. If it will be returned by reference, the
7048 // frontend must modify the prototype so a pointer with the sret annotation is
7049 // passed as the first argument. This is not necessary for large scalar
7050 // returns.
7051 // * Struct return values and varargs should be coerced to structs containing
7052 // register-size fields in the same situations they would be for fixed
7053 // arguments.
7054
7055 static const MCPhysReg ArgGPRs[] = {
7056 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
7057 RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
7058 };
7059 static const MCPhysReg ArgFPR16s[] = {
7060 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
7061 RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
7062 };
7063 static const MCPhysReg ArgFPR32s[] = {
7064 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
7065 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
7066 };
7067 static const MCPhysReg ArgFPR64s[] = {
7068 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
7069 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
7070 };
7071 // This is an interim calling convention and it may be changed in the future.
7072 static const MCPhysReg ArgVRs[] = {
7073 RISCV::V8, RISCV::V9, RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
7074 RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
7075 RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23};
7076 static const MCPhysReg ArgVRM2s[] = {RISCV::V8M2, RISCV::V10M2, RISCV::V12M2,
7077 RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
7078 RISCV::V20M2, RISCV::V22M2};
7079 static const MCPhysReg ArgVRM4s[] = {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
7080 RISCV::V20M4};
7081 static const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8};
7082
7083 // Pass a 2*XLEN argument that has been split into two XLEN values through
7084 // registers or the stack as necessary.
7085 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
7086 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
7087 MVT ValVT2, MVT LocVT2,
7088 ISD::ArgFlagsTy ArgFlags2) {
7089 unsigned XLenInBytes = XLen / 8;
7090 if (Register Reg = State.AllocateReg(ArgGPRs)) {
7091 // At least one half can be passed via register.
7092 State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
7093 VA1.getLocVT(), CCValAssign::Full));
7094 } else {
7095 // Both halves must be passed on the stack, with proper alignment.
7096 Align StackAlign =
7097 std::max(Align(XLenInBytes), ArgFlags1.getNonZeroOrigAlign());
7098 State.addLoc(
7099 CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
7100 State.AllocateStack(XLenInBytes, StackAlign),
7101 VA1.getLocVT(), CCValAssign::Full));
7102 State.addLoc(CCValAssign::getMem(
7103 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
7104 LocVT2, CCValAssign::Full));
7105 return false;
7106 }
7107
7108 if (Register Reg = State.AllocateReg(ArgGPRs)) {
7109 // The second half can also be passed via register.
7110 State.addLoc(
7111 CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
7112 } else {
7113 // The second half is passed via the stack, without additional alignment.
7114 State.addLoc(CCValAssign::getMem(
7115 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, Align(XLenInBytes)),
7116 LocVT2, CCValAssign::Full));
7117 }
7118
7119 return false;
7120 }
7121
7122 static unsigned allocateRVVReg(MVT ValVT, unsigned ValNo,
7123 Optional<unsigned> FirstMaskArgument,
7124 CCState &State, const RISCVTargetLowering &TLI) {
7125 const TargetRegisterClass *RC = TLI.getRegClassFor(ValVT);
7126 if (RC == &RISCV::VRRegClass) {
7127 // Assign the first mask argument to V0.
7128 // This is an interim calling convention and it may be changed in the
7129 // future.
7130 if (FirstMaskArgument.hasValue() && ValNo == FirstMaskArgument.getValue())
7131 return State.AllocateReg(RISCV::V0);
7132 return State.AllocateReg(ArgVRs);
7133 }
7134 if (RC == &RISCV::VRM2RegClass)
7135 return State.AllocateReg(ArgVRM2s);
7136 if (RC == &RISCV::VRM4RegClass)
7137 return State.AllocateReg(ArgVRM4s);
7138 if (RC == &RISCV::VRM8RegClass)
7139 return State.AllocateReg(ArgVRM8s);
7140 llvm_unreachable("Unhandled register class for ValueType");
7141 }
7142
7143 // Implements the RISC-V calling convention. Returns true upon failure.
7144 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
7145 MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
7146 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
7147 bool IsRet, Type *OrigTy, const RISCVTargetLowering &TLI,
7148 Optional<unsigned> FirstMaskArgument) {
7149 unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
7150 assert(XLen == 32 || XLen == 64);
7151 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
7152
7153 // Any return value split in to more than two values can't be returned
7154 // directly. Vectors are returned via the available vector registers.
7155 if (!LocVT.isVector() && IsRet && ValNo > 1)
7156 return true;
7157
7158 // UseGPRForF16_F32 if targeting one of the soft-float ABIs, if passing a
7159 // variadic argument, or if no F16/F32 argument registers are available.
7160 bool UseGPRForF16_F32 = true;
7161 // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
7162 // variadic argument, or if no F64 argument registers are available.
7163 bool UseGPRForF64 = true;
7164
7165 switch (ABI) {
7166 default:
7167 llvm_unreachable("Unexpected ABI");
7168 case RISCVABI::ABI_ILP32:
7169 case RISCVABI::ABI_LP64:
7170 break;
7171 case RISCVABI::ABI_ILP32F:
7172 case RISCVABI::ABI_LP64F:
7173 UseGPRForF16_F32 = !IsFixed;
7174 break;
7175 case RISCVABI::ABI_ILP32D:
7176 case RISCVABI::ABI_LP64D:
7177 UseGPRForF16_F32 = !IsFixed;
7178 UseGPRForF64 = !IsFixed;
7179 break;
7180 }
7181
7182 // FPR16, FPR32, and FPR64 alias each other.
7183 if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s)) {
7184 UseGPRForF16_F32 = true;
7185 UseGPRForF64 = true;
7186 }
7187
7188 // From this point on, rely on UseGPRForF16_F32, UseGPRForF64 and
7189 // similar local variables rather than directly checking against the target
7190 // ABI.
7191
7192 if (UseGPRForF16_F32 && (ValVT == MVT::f16 || ValVT == MVT::f32)) {
7193 LocVT = XLenVT;
7194 LocInfo = CCValAssign::BCvt;
7195 } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
7196 LocVT = MVT::i64;
7197 LocInfo = CCValAssign::BCvt;
7198 }
7199
7200 // If this is a variadic argument, the RISC-V calling convention requires
7201 // that it is assigned an 'even' or 'aligned' register if it has 8-byte
7202 // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
7203 // be used regardless of whether the original argument was split during
7204 // legalisation or not. The argument will not be passed by registers if the
7205 // original type is larger than 2*XLEN, so the register alignment rule does
7206 // not apply.
7207 unsigned TwoXLenInBytes = (2 * XLen) / 8;
7208 if (!IsFixed && ArgFlags.getNonZeroOrigAlign() == TwoXLenInBytes &&
7209 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
7210 unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
7211 // Skip 'odd' register if necessary.
7212 if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
7213 State.AllocateReg(ArgGPRs);
7214 }
7215
7216 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
7217 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
7218 State.getPendingArgFlags();
7219
7220 assert(PendingLocs.size() == PendingArgFlags.size() &&
7221 "PendingLocs and PendingArgFlags out of sync");
7222
7223 // Handle passing f64 on RV32D with a soft float ABI or when floating point
7224 // registers are exhausted.
7225 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
7226 assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
7227 "Can't lower f64 if it is split");
7228 // Depending on available argument GPRS, f64 may be passed in a pair of
7229 // GPRs, split between a GPR and the stack, or passed completely on the
7230 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
7231 // cases.
7232 Register Reg = State.AllocateReg(ArgGPRs);
7233 LocVT = MVT::i32;
7234 if (!Reg) {
7235 unsigned StackOffset = State.AllocateStack(8, Align(8));
7236 State.addLoc(
7237 CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
7238 return false;
7239 }
7240 if (!State.AllocateReg(ArgGPRs))
7241 State.AllocateStack(4, Align(4));
7242 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7243 return false;
7244 }
7245
7246 // Fixed-length vectors are located in the corresponding scalable-vector
7247 // container types.
7248 if (ValVT.isFixedLengthVector())
7249 LocVT = TLI.getContainerForFixedLengthVector(LocVT);
7250
7251 // Split arguments might be passed indirectly, so keep track of the pending
7252 // values. Split vectors are passed via a mix of registers and indirectly, so
7253 // treat them as we would any other argument.
7254 if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
7255 LocVT = XLenVT;
7256 LocInfo = CCValAssign::Indirect;
7257 PendingLocs.push_back(
7258 CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
7259 PendingArgFlags.push_back(ArgFlags);
7260 if (!ArgFlags.isSplitEnd()) {
7261 return false;
7262 }
7263 }
7264
7265 // If the split argument only had two elements, it should be passed directly
7266 // in registers or on the stack.
7267 if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
7268 PendingLocs.size() <= 2) {
7269 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
7270 // Apply the normal calling convention rules to the first half of the
7271 // split argument.
7272 CCValAssign VA = PendingLocs[0];
7273 ISD::ArgFlagsTy AF = PendingArgFlags[0];
7274 PendingLocs.clear();
7275 PendingArgFlags.clear();
7276 return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
7277 ArgFlags);
7278 }
7279
7280 // Allocate to a register if possible, or else a stack slot.
7281 Register Reg;
7282 unsigned StoreSizeBytes = XLen / 8;
7283 Align StackAlign = Align(XLen / 8);
7284
7285 if (ValVT == MVT::f16 && !UseGPRForF16_F32)
7286 Reg = State.AllocateReg(ArgFPR16s);
7287 else if (ValVT == MVT::f32 && !UseGPRForF16_F32)
7288 Reg = State.AllocateReg(ArgFPR32s);
7289 else if (ValVT == MVT::f64 && !UseGPRForF64)
7290 Reg = State.AllocateReg(ArgFPR64s);
7291 else if (ValVT.isVector()) {
7292 Reg = allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI);
7293 if (!Reg) {
7294 // For return values, the vector must be passed fully via registers or
7295 // via the stack.
7296 // FIXME: The proposed vector ABI only mandates v8-v15 for return values,
7297 // but we're using all of them.
7298 if (IsRet)
7299 return true;
7300 // Try using a GPR to pass the address
7301 if ((Reg = State.AllocateReg(ArgGPRs))) {
7302 LocVT = XLenVT;
7303 LocInfo = CCValAssign::Indirect;
7304 } else if (ValVT.isScalableVector()) {
7305 report_fatal_error("Unable to pass scalable vector types on the stack");
7306 } else {
7307 // Pass fixed-length vectors on the stack.
7308 LocVT = ValVT;
7309 StoreSizeBytes = ValVT.getStoreSize();
7310 // Align vectors to their element sizes, being careful for vXi1
7311 // vectors.
7312 StackAlign = MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
7313 }
7314 }
7315 } else {
7316 Reg = State.AllocateReg(ArgGPRs);
7317 }
7318
7319 unsigned StackOffset =
7320 Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
7321
7322 // If we reach this point and PendingLocs is non-empty, we must be at the
7323 // end of a split argument that must be passed indirectly.
7324 if (!PendingLocs.empty()) {
7325 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
7326 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
7327
7328 for (auto &It : PendingLocs) {
7329 if (Reg)
7330 It.convertToReg(Reg);
7331 else
7332 It.convertToMem(StackOffset);
7333 State.addLoc(It);
7334 }
7335 PendingLocs.clear();
7336 PendingArgFlags.clear();
7337 return false;
7338 }
7339
7340 assert((!UseGPRForF16_F32 || !UseGPRForF64 || LocVT == XLenVT ||
7341 (TLI.getSubtarget().hasStdExtV() && ValVT.isVector())) &&
7342 "Expected an XLenVT or vector types at this stage");
7343
7344 if (Reg) {
7345 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7346 return false;
7347 }
7348
7349 // When a floating-point value is passed on the stack, no bit-conversion is
7350 // needed.
7351 if (ValVT.isFloatingPoint()) {
7352 LocVT = ValVT;
7353 LocInfo = CCValAssign::Full;
7354 }
7355 State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
7356 return false;
7357 }
7358
7359 template <typename ArgTy>
7360 static Optional<unsigned> preAssignMask(const ArgTy &Args) {
7361 for (const auto &ArgIdx : enumerate(Args)) {
7362 MVT ArgVT = ArgIdx.value().VT;
7363 if (ArgVT.isVector() && ArgVT.getVectorElementType() == MVT::i1)
7364 return ArgIdx.index();
7365 }
7366 return None;
7367 }
7368
7369 void RISCVTargetLowering::analyzeInputArgs(
7370 MachineFunction &MF, CCState &CCInfo,
7371 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
7372 RISCVCCAssignFn Fn) const {
7373 unsigned NumArgs = Ins.size();
7374 FunctionType *FType = MF.getFunction().getFunctionType();
7375
7376 Optional<unsigned> FirstMaskArgument;
7377 if (Subtarget.hasStdExtV())
7378 FirstMaskArgument = preAssignMask(Ins);
7379
7380 for (unsigned i = 0; i != NumArgs; ++i) {
7381 MVT ArgVT = Ins[i].VT;
7382 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
7383
7384 Type *ArgTy = nullptr;
7385 if (IsRet)
7386 ArgTy = FType->getReturnType();
7387 else if (Ins[i].isOrigArg())
7388 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
7389
7390 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
7391 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
7392 ArgFlags, CCInfo, /*IsFixed=*/true, IsRet, ArgTy, *this,
7393 FirstMaskArgument)) {
7394 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
7395 << EVT(ArgVT).getEVTString() << '\n');
7396 llvm_unreachable(nullptr);
7397 }
7398 }
7399 }
7400
7401 void RISCVTargetLowering::analyzeOutputArgs(
7402 MachineFunction &MF, CCState &CCInfo,
7403 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
7404 CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
7405 unsigned NumArgs = Outs.size();
7406
7407 Optional<unsigned> FirstMaskArgument;
7408 if (Subtarget.hasStdExtV())
7409 FirstMaskArgument = preAssignMask(Outs);
7410
7411 for (unsigned i = 0; i != NumArgs; i++) {
7412 MVT ArgVT = Outs[i].VT;
7413 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
7414 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
7415
7416 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
7417 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
7418 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy, *this,
7419 FirstMaskArgument)) {
7420 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
7421 << EVT(ArgVT).getEVTString() << "\n");
7422 llvm_unreachable(nullptr);
7423 }
7424 }
7425 }
7426
7427 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
7428 // values.
7429 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
7430 const CCValAssign &VA, const SDLoc &DL,
7431 const RISCVSubtarget &Subtarget) {
7432 switch (VA.getLocInfo()) {
7433 default:
7434 llvm_unreachable("Unexpected CCValAssign::LocInfo");
7435 case CCValAssign::Full:
7436 if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
7437 Val = convertFromScalableVector(VA.getValVT(), Val, DAG, Subtarget);
7438 break;
7439 case CCValAssign::BCvt:
7440 if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
7441 Val = DAG.getNode(RISCVISD::FMV_H_X, DL, MVT::f16, Val);
7442 else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
7443 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
7444 else
7445 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
7446 break;
7447 }
7448 return Val;
7449 }
7450
7451 // The caller is responsible for loading the full value if the argument is
7452 // passed with CCValAssign::Indirect.
7453 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
7454 const CCValAssign &VA, const SDLoc &DL,
7455 const RISCVTargetLowering &TLI) {
7456 MachineFunction &MF = DAG.getMachineFunction();
7457 MachineRegisterInfo &RegInfo = MF.getRegInfo();
7458 EVT LocVT = VA.getLocVT();
7459 SDValue Val;
7460 const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
7461 Register VReg = RegInfo.createVirtualRegister(RC);
7462 RegInfo.addLiveIn(VA.getLocReg(), VReg);
7463 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
7464
7465 if (VA.getLocInfo() == CCValAssign::Indirect)
7466 return Val;
7467
7468 return convertLocVTToValVT(DAG, Val, VA, DL, TLI.getSubtarget());
7469 }
7470
7471 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
7472 const CCValAssign &VA, const SDLoc &DL,
7473 const RISCVSubtarget &Subtarget) {
7474 EVT LocVT = VA.getLocVT();
7475
7476 switch (VA.getLocInfo()) {
7477 default:
7478 llvm_unreachable("Unexpected CCValAssign::LocInfo");
7479 case CCValAssign::Full:
7480 if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
7481 Val = convertToScalableVector(LocVT, Val, DAG, Subtarget);
7482 break;
7483 case CCValAssign::BCvt:
7484 if (VA.getLocVT().isInteger() && VA.getValVT() == MVT::f16)
7485 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTH, DL, VA.getLocVT(), Val);
7486 else if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
7487 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
7488 else
7489 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
7490 break;
7491 }
7492 return Val;
7493 }
7494
7495 // The caller is responsible for loading the full value if the argument is
7496 // passed with CCValAssign::Indirect.
7497 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
7498 const CCValAssign &VA, const SDLoc &DL) {
7499 MachineFunction &MF = DAG.getMachineFunction();
7500 MachineFrameInfo &MFI = MF.getFrameInfo();
7501 EVT LocVT = VA.getLocVT();
7502 EVT ValVT = VA.getValVT();
7503 EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
7504 int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
7505 /*Immutable=*/true);
7506 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
7507 SDValue Val;
7508
7509 ISD::LoadExtType ExtType;
7510 switch (VA.getLocInfo()) {
7511 default:
7512 llvm_unreachable("Unexpected CCValAssign::LocInfo");
7513 case CCValAssign::Full:
7514 case CCValAssign::Indirect:
7515 case CCValAssign::BCvt:
7516 ExtType = ISD::NON_EXTLOAD;
7517 break;
7518 }
7519 Val = DAG.getExtLoad(
7520 ExtType, DL, LocVT, Chain, FIN,
7521 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
7522 return Val;
7523 }
7524
7525 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
7526 const CCValAssign &VA, const SDLoc &DL) {
7527 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
7528 "Unexpected VA");
7529 MachineFunction &MF = DAG.getMachineFunction();
7530 MachineFrameInfo &MFI = MF.getFrameInfo();
7531 MachineRegisterInfo &RegInfo = MF.getRegInfo();
7532
7533 if (VA.isMemLoc()) {
7534 // f64 is passed on the stack.
7535 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
7536 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
7537 return DAG.getLoad(MVT::f64, DL, Chain, FIN,
7538 MachinePointerInfo::getFixedStack(MF, FI));
7539 }
7540
7541 assert(VA.isRegLoc() && "Expected register VA assignment");
7542
7543 Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
7544 RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
7545 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
7546 SDValue Hi;
7547 if (VA.getLocReg() == RISCV::X17) {
7548 // Second half of f64 is passed on the stack.
7549 int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
7550 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
7551 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
7552 MachinePointerInfo::getFixedStack(MF, FI));
7553 } else {
7554 // Second half of f64 is passed in another GPR.
7555 Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
7556 RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
7557 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
7558 }
7559 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
7560 }
7561
7562 // FastCC has less than 1% performance improvement for some particular
7563 // benchmark. But theoretically, it may has benenfit for some cases.
7564 static bool CC_RISCV_FastCC(const DataLayout &DL, RISCVABI::ABI ABI,
7565 unsigned ValNo, MVT ValVT, MVT LocVT,
7566 CCValAssign::LocInfo LocInfo,
7567 ISD::ArgFlagsTy ArgFlags, CCState &State,
7568 bool IsFixed, bool IsRet, Type *OrigTy,
7569 const RISCVTargetLowering &TLI,
7570 Optional<unsigned> FirstMaskArgument) {
7571
7572 // X5 and X6 might be used for save-restore libcall.
7573 static const MCPhysReg GPRList[] = {
7574 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
7575 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
7576 RISCV::X29, RISCV::X30, RISCV::X31};
7577
7578 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
7579 if (unsigned Reg = State.AllocateReg(GPRList)) {
7580 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7581 return false;
7582 }
7583 }
7584
7585 if (LocVT == MVT::f16) {
7586 static const MCPhysReg FPR16List[] = {
7587 RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H, RISCV::F14_H,
7588 RISCV::F15_H, RISCV::F16_H, RISCV::F17_H, RISCV::F0_H, RISCV::F1_H,
7589 RISCV::F2_H, RISCV::F3_H, RISCV::F4_H, RISCV::F5_H, RISCV::F6_H,
7590 RISCV::F7_H, RISCV::F28_H, RISCV::F29_H, RISCV::F30_H, RISCV::F31_H};
7591 if (unsigned Reg = State.AllocateReg(FPR16List)) {
7592 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7593 return false;
7594 }
7595 }
7596
7597 if (LocVT == MVT::f32) {
7598 static const MCPhysReg FPR32List[] = {
7599 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
7600 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
7601 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
7602 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
7603 if (unsigned Reg = State.AllocateReg(FPR32List)) {
7604 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7605 return false;
7606 }
7607 }
7608
7609 if (LocVT == MVT::f64) {
7610 static const MCPhysReg FPR64List[] = {
7611 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
7612 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
7613 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
7614 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
7615 if (unsigned Reg = State.AllocateReg(FPR64List)) {
7616 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7617 return false;
7618 }
7619 }
7620
7621 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
7622 unsigned Offset4 = State.AllocateStack(4, Align(4));
7623 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
7624 return false;
7625 }
7626
7627 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
7628 unsigned Offset5 = State.AllocateStack(8, Align(8));
7629 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
7630 return false;
7631 }
7632
7633 if (LocVT.isVector()) {
7634 if (unsigned Reg =
7635 allocateRVVReg(ValVT, ValNo, FirstMaskArgument, State, TLI)) {
7636 // Fixed-length vectors are located in the corresponding scalable-vector
7637 // container types.
7638 if (ValVT.isFixedLengthVector())
7639 LocVT = TLI.getContainerForFixedLengthVector(LocVT);
7640 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7641 } else {
7642 // Try and pass the address via a "fast" GPR.
7643 if (unsigned GPRReg = State.AllocateReg(GPRList)) {
7644 LocInfo = CCValAssign::Indirect;
7645 LocVT = TLI.getSubtarget().getXLenVT();
7646 State.addLoc(CCValAssign::getReg(ValNo, ValVT, GPRReg, LocVT, LocInfo));
7647 } else if (ValVT.isFixedLengthVector()) {
7648 auto StackAlign =
7649 MaybeAlign(ValVT.getScalarSizeInBits() / 8).valueOrOne();
7650 unsigned StackOffset =
7651 State.AllocateStack(ValVT.getStoreSize(), StackAlign);
7652 State.addLoc(
7653 CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
7654 } else {
7655 // Can't pass scalable vectors on the stack.
7656 return true;
7657 }
7658 }
7659
7660 return false;
7661 }
7662
7663 return true; // CC didn't match.
7664 }
7665
7666 static bool CC_RISCV_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
7667 CCValAssign::LocInfo LocInfo,
7668 ISD::ArgFlagsTy ArgFlags, CCState &State) {
7669
7670 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
7671 // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, R6, R7, SpLim
7672 // s1 s2 s3 s4 s5 s6 s7 s8 s9 s10 s11
7673 static const MCPhysReg GPRList[] = {
7674 RISCV::X9, RISCV::X18, RISCV::X19, RISCV::X20, RISCV::X21, RISCV::X22,
7675 RISCV::X23, RISCV::X24, RISCV::X25, RISCV::X26, RISCV::X27};
7676 if (unsigned Reg = State.AllocateReg(GPRList)) {
7677 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7678 return false;
7679 }
7680 }
7681
7682 if (LocVT == MVT::f32) {
7683 // Pass in STG registers: F1, ..., F6
7684 // fs0 ... fs5
7685 static const MCPhysReg FPR32List[] = {RISCV::F8_F, RISCV::F9_F,
7686 RISCV::F18_F, RISCV::F19_F,
7687 RISCV::F20_F, RISCV::F21_F};
7688 if (unsigned Reg = State.AllocateReg(FPR32List)) {
7689 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7690 return false;
7691 }
7692 }
7693
7694 if (LocVT == MVT::f64) {
7695 // Pass in STG registers: D1, ..., D6
7696 // fs6 ... fs11
7697 static const MCPhysReg FPR64List[] = {RISCV::F22_D, RISCV::F23_D,
7698 RISCV::F24_D, RISCV::F25_D,
7699 RISCV::F26_D, RISCV::F27_D};
7700 if (unsigned Reg = State.AllocateReg(FPR64List)) {
7701 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
7702 return false;
7703 }
7704 }
7705
7706 report_fatal_error("No registers left in GHC calling convention");
7707 return true;
7708 }
7709
7710 // Transform physical registers into virtual registers.
7711 SDValue RISCVTargetLowering::LowerFormalArguments(
7712 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
7713 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
7714 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7715
7716 MachineFunction &MF = DAG.getMachineFunction();
7717
7718 switch (CallConv) {
7719 default:
7720 report_fatal_error("Unsupported calling convention");
7721 case CallingConv::C:
7722 case CallingConv::Fast:
7723 break;
7724 case CallingConv::GHC:
7725 if (!MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtF] ||
7726 !MF.getSubtarget().getFeatureBits()[RISCV::FeatureStdExtD])
7727 report_fatal_error(
7728 "GHC calling convention requires the F and D instruction set extensions");
7729 }
7730
7731 const Function &Func = MF.getFunction();
7732 if (Func.hasFnAttribute("interrupt")) {
7733 if (!Func.arg_empty())
7734 report_fatal_error(
7735 "Functions with the interrupt attribute cannot have arguments!");
7736
7737 StringRef Kind =
7738 MF.getFunction().getFnAttribute("interrupt").getValueAsString();
7739
7740 if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
7741 report_fatal_error(
7742 "Function interrupt attribute argument not supported!");
7743 }
7744
7745 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7746 MVT XLenVT = Subtarget.getXLenVT();
7747 unsigned XLenInBytes = Subtarget.getXLen() / 8;
7748 // Used with vargs to acumulate store chains.
7749 std::vector<SDValue> OutChains;
7750
7751 // Assign locations to all of the incoming arguments.
7752 SmallVector<CCValAssign, 16> ArgLocs;
7753 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
7754
7755 if (CallConv == CallingConv::GHC)
7756 CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_GHC);
7757 else
7758 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
7759 CallConv == CallingConv::Fast ? CC_RISCV_FastCC
7760 : CC_RISCV);
7761
7762 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
7763 CCValAssign &VA = ArgLocs[i];
7764 SDValue ArgValue;
7765 // Passing f64 on RV32D with a soft float ABI must be handled as a special
7766 // case.
7767 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
7768 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
7769 else if (VA.isRegLoc())
7770 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, *this);
7771 else
7772 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
7773
7774 if (VA.getLocInfo() == CCValAssign::Indirect) {
7775 // If the original argument was split and passed by reference (e.g. i128
7776 // on RV32), we need to load all parts of it here (using the same
7777 // address). Vectors may be partly split to registers and partly to the
7778 // stack, in which case the base address is partly offset and subsequent
7779 // stores are relative to that.
7780 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
7781 MachinePointerInfo()));
7782 unsigned ArgIndex = Ins[i].OrigArgIndex;
7783 unsigned ArgPartOffset = Ins[i].PartOffset;
7784 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
7785 while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
7786 CCValAssign &PartVA = ArgLocs[i + 1];
7787 unsigned PartOffset = Ins[i + 1].PartOffset - ArgPartOffset;
7788 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
7789 if (PartVA.getValVT().isScalableVector())
7790 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
7791 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
7792 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
7793 MachinePointerInfo()));
7794 ++i;
7795 }
7796 continue;
7797 }
7798 InVals.push_back(ArgValue);
7799 }
7800
7801 if (IsVarArg) {
7802 ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
7803 unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
7804 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
7805 MachineFrameInfo &MFI = MF.getFrameInfo();
7806 MachineRegisterInfo &RegInfo = MF.getRegInfo();
7807 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
7808
7809 // Offset of the first variable argument from stack pointer, and size of
7810 // the vararg save area. For now, the varargs save area is either zero or
7811 // large enough to hold a0-a7.
7812 int VaArgOffset, VarArgsSaveSize;
7813
7814 // If all registers are allocated, then all varargs must be passed on the
7815 // stack and we don't need to save any argregs.
7816 if (ArgRegs.size() == Idx) {
7817 VaArgOffset = CCInfo.getNextStackOffset();
7818 VarArgsSaveSize = 0;
7819 } else {
7820 VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
7821 VaArgOffset = -VarArgsSaveSize;
7822 }
7823
7824 // Record the frame index of the first variable argument
7825 // which is a value necessary to VASTART.
7826 int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
7827 RVFI->setVarArgsFrameIndex(FI);
7828
7829 // If saving an odd number of registers then create an extra stack slot to
7830 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
7831 // offsets to even-numbered registered remain 2*XLEN-aligned.
7832 if (Idx % 2) {
7833 MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
7834 VarArgsSaveSize += XLenInBytes;
7835 }
7836
7837 // Copy the integer registers that may have been used for passing varargs
7838 // to the vararg save area.
7839 for (unsigned I = Idx; I < ArgRegs.size();
7840 ++I, VaArgOffset += XLenInBytes) {
7841 const Register Reg = RegInfo.createVirtualRegister(RC);
7842 RegInfo.addLiveIn(ArgRegs[I], Reg);
7843 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
7844 FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
7845 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
7846 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
7847 MachinePointerInfo::getFixedStack(MF, FI));
7848 cast<StoreSDNode>(Store.getNode())
7849 ->getMemOperand()
7850 ->setValue((Value *)nullptr);
7851 OutChains.push_back(Store);
7852 }
7853 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
7854 }
7855
7856 // All stores are grouped in one node to allow the matching between
7857 // the size of Ins and InVals. This only happens for vararg functions.
7858 if (!OutChains.empty()) {
7859 OutChains.push_back(Chain);
7860 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
7861 }
7862
7863 return Chain;
7864 }
7865
7866 /// isEligibleForTailCallOptimization - Check whether the call is eligible
7867 /// for tail call optimization.
7868 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
7869 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
7870 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
7871 const SmallVector<CCValAssign, 16> &ArgLocs) const {
7872
7873 auto &Callee = CLI.Callee;
7874 auto CalleeCC = CLI.CallConv;
7875 auto &Outs = CLI.Outs;
7876 auto &Caller = MF.getFunction();
7877 auto CallerCC = Caller.getCallingConv();
7878
7879 // Exception-handling functions need a special set of instructions to
7880 // indicate a return to the hardware. Tail-calling another function would
7881 // probably break this.
7882 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
7883 // should be expanded as new function attributes are introduced.
7884 if (Caller.hasFnAttribute("interrupt"))
7885 return false;
7886
7887 // Do not tail call opt if the stack is used to pass parameters.
7888 if (CCInfo.getNextStackOffset() != 0)
7889 return false;
7890
7891 // Do not tail call opt if any parameters need to be passed indirectly.
7892 // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
7893 // passed indirectly. So the address of the value will be passed in a
7894 // register, or if not available, then the address is put on the stack. In
7895 // order to pass indirectly, space on the stack often needs to be allocated
7896 // in order to store the value. In this case the CCInfo.getNextStackOffset()
7897 // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
7898 // are passed CCValAssign::Indirect.
7899 for (auto &VA : ArgLocs)
7900 if (VA.getLocInfo() == CCValAssign::Indirect)
7901 return false;
7902
7903 // Do not tail call opt if either caller or callee uses struct return
7904 // semantics.
7905 auto IsCallerStructRet = Caller.hasStructRetAttr();
7906 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
7907 if (IsCallerStructRet || IsCalleeStructRet)
7908 return false;
7909
7910 // Externally-defined functions with weak linkage should not be
7911 // tail-called. The behaviour of branch instructions in this situation (as
7912 // used for tail calls) is implementation-defined, so we cannot rely on the
7913 // linker replacing the tail call with a return.
7914 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
7915 const GlobalValue *GV = G->getGlobal();
7916 if (GV->hasExternalWeakLinkage())
7917 return false;
7918 }
7919
7920 // The callee has to preserve all registers the caller needs to preserve.
7921 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
7922 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
7923 if (CalleeCC != CallerCC) {
7924 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
7925 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
7926 return false;
7927 }
7928
7929 // Byval parameters hand the function a pointer directly into the stack area
7930 // we want to reuse during a tail call. Working around this *is* possible
7931 // but less efficient and uglier in LowerCall.
7932 for (auto &Arg : Outs)
7933 if (Arg.Flags.isByVal())
7934 return false;
7935
7936 return true;
7937 }
7938
7939 static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
7940 return DAG.getDataLayout().getPrefTypeAlign(
7941 VT.getTypeForEVT(*DAG.getContext()));
7942 }
7943
7944 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
7945 // and output parameter nodes.
7946 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
7947 SmallVectorImpl<SDValue> &InVals) const {
7948 SelectionDAG &DAG = CLI.DAG;
7949 SDLoc &DL = CLI.DL;
7950 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
7951 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
7952 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
7953 SDValue Chain = CLI.Chain;
7954 SDValue Callee = CLI.Callee;
7955 bool &IsTailCall = CLI.IsTailCall;
7956 CallingConv::ID CallConv = CLI.CallConv;
7957 bool IsVarArg = CLI.IsVarArg;
7958 EVT PtrVT = getPointerTy(DAG.getDataLayout());
7959 MVT XLenVT = Subtarget.getXLenVT();
7960
7961 MachineFunction &MF = DAG.getMachineFunction();
7962
7963 // Analyze the operands of the call, assigning locations to each operand.
7964 SmallVector<CCValAssign, 16> ArgLocs;
7965 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
7966
7967 if (CallConv == CallingConv::GHC)
7968 ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_GHC);
7969 else
7970 analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI,
7971 CallConv == CallingConv::Fast ? CC_RISCV_FastCC
7972 : CC_RISCV);
7973
7974 // Check if it's really possible to do a tail call.
7975 if (IsTailCall)
7976 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
7977
7978 if (IsTailCall)
7979 ++NumTailCalls;
7980 else if (CLI.CB && CLI.CB->isMustTailCall())
7981 report_fatal_error("failed to perform tail call elimination on a call "
7982 "site marked musttail");
7983
7984 // Get a count of how many bytes are to be pushed on the stack.
7985 unsigned NumBytes = ArgCCInfo.getNextStackOffset();
7986
7987 // Create local copies for byval args
7988 SmallVector<SDValue, 8> ByValArgs;
7989 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
7990 ISD::ArgFlagsTy Flags = Outs[i].Flags;
7991 if (!Flags.isByVal())
7992 continue;
7993
7994 SDValue Arg = OutVals[i];
7995 unsigned Size = Flags.getByValSize();
7996 Align Alignment = Flags.getNonZeroByValAlign();
7997
7998 int FI =
7999 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
8000 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
8001 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
8002
8003 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment,
8004 /*IsVolatile=*/false,
8005 /*AlwaysInline=*/false, IsTailCall,
8006 MachinePointerInfo(), MachinePointerInfo());
8007 ByValArgs.push_back(FIPtr);
8008 }
8009
8010 if (!IsTailCall)
8011 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
8012
8013 // Copy argument values to their designated locations.
8014 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
8015 SmallVector<SDValue, 8> MemOpChains;
8016 SDValue StackPtr;
8017 for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
8018 CCValAssign &VA = ArgLocs[i];
8019 SDValue ArgValue = OutVals[i];
8020 ISD::ArgFlagsTy Flags = Outs[i].Flags;
8021
8022 // Handle passing f64 on RV32D with a soft float ABI as a special case.
8023 bool IsF64OnRV32DSoftABI =
8024 VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
8025 if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
8026 SDValue SplitF64 = DAG.getNode(
8027 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
8028 SDValue Lo = SplitF64.getValue(0);
8029 SDValue Hi = SplitF64.getValue(1);
8030
8031 Register RegLo = VA.getLocReg();
8032 RegsToPass.push_back(std::make_pair(RegLo, Lo));
8033
8034 if (RegLo == RISCV::X17) {
8035 // Second half of f64 is passed on the stack.
8036 // Work out the address of the stack slot.
8037 if (!StackPtr.getNode())
8038 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
8039 // Emit the store.
8040 MemOpChains.push_back(
8041 DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
8042 } else {
8043 // Second half of f64 is passed in another GPR.
8044 assert(RegLo < RISCV::X31 && "Invalid register pair");
8045 Register RegHigh = RegLo + 1;
8046 RegsToPass.push_back(std::make_pair(RegHigh, Hi));
8047 }
8048 continue;
8049 }
8050
8051 // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
8052 // as any other MemLoc.
8053
8054 // Promote the value if needed.
8055 // For now, only handle fully promoted and indirect arguments.
8056 if (VA.getLocInfo() == CCValAssign::Indirect) {
8057 // Store the argument in a stack slot and pass its address.
8058 Align StackAlign =
8059 std::max(getPrefTypeAlign(Outs[i].ArgVT, DAG),
8060 getPrefTypeAlign(ArgValue.getValueType(), DAG));
8061 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
8062 // If the original argument was split (e.g. i128), we need
8063 // to store the required parts of it here (and pass just one address).
8064 // Vectors may be partly split to registers and partly to the stack, in
8065 // which case the base address is partly offset and subsequent stores are
8066 // relative to that.
8067 unsigned ArgIndex = Outs[i].OrigArgIndex;
8068 unsigned ArgPartOffset = Outs[i].PartOffset;
8069 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
8070 // Calculate the total size to store. We don't have access to what we're
8071 // actually storing other than performing the loop and collecting the
8072 // info.
8073 SmallVector<std::pair<SDValue, SDValue>> Parts;
8074 while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
8075 SDValue PartValue = OutVals[i + 1];
8076 unsigned PartOffset = Outs[i + 1].PartOffset - ArgPartOffset;
8077 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
8078 EVT PartVT = PartValue.getValueType();
8079 if (PartVT.isScalableVector())
8080 Offset = DAG.getNode(ISD::VSCALE, DL, XLenVT, Offset);
8081 StoredSize += PartVT.getStoreSize();
8082 StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
8083 Parts.push_back(std::make_pair(PartValue, Offset));
8084 ++i;
8085 }
8086 SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
8087 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
8088 MemOpChains.push_back(
8089 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
8090 MachinePointerInfo::getFixedStack(MF, FI)));
8091 for (const auto &Part : Parts) {
8092 SDValue PartValue = Part.first;
8093 SDValue PartOffset = Part.second;
8094 SDValue Address =
8095 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
8096 MemOpChains.push_back(
8097 DAG.getStore(Chain, DL, PartValue, Address,
8098 MachinePointerInfo::getFixedStack(MF, FI)));
8099 }
8100 ArgValue = SpillSlot;
8101 } else {
8102 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL, Subtarget);
8103 }
8104
8105 // Use local copy if it is a byval arg.
8106 if (Flags.isByVal())
8107 ArgValue = ByValArgs[j++];
8108
8109 if (VA.isRegLoc()) {
8110 // Queue up the argument copies and emit them at the end.
8111 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
8112 } else {
8113 assert(VA.isMemLoc() && "Argument not register or memory");
8114 assert(!IsTailCall && "Tail call not allowed if stack is used "
8115 "for passing parameters");
8116
8117 // Work out the address of the stack slot.
8118 if (!StackPtr.getNode())
8119 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
8120 SDValue Address =
8121 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
8122 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
8123
8124 // Emit the store.
8125 MemOpChains.push_back(
8126 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
8127 }
8128 }
8129
8130 // Join the stores, which are independent of one another.
8131 if (!MemOpChains.empty())
8132 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
8133
8134 SDValue Glue;
8135
8136 // Build a sequence of copy-to-reg nodes, chained and glued together.
8137 for (auto &Reg : RegsToPass) {
8138 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
8139 Glue = Chain.getValue(1);
8140 }
8141
8142 // Validate that none of the argument registers have been marked as
8143 // reserved, if so report an error. Do the same for the return address if this
8144 // is not a tailcall.
8145 validateCCReservedRegs(RegsToPass, MF);
8146 if (!IsTailCall &&
8147 MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
8148 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
8149 MF.getFunction(),
8150 "Return address register required, but has been reserved."});
8151
8152 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
8153 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
8154 // split it and then direct call can be matched by PseudoCALL.
8155 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
8156 const GlobalValue *GV = S->getGlobal();
8157
8158 unsigned OpFlags = RISCVII::MO_CALL;
8159 if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
8160 OpFlags = RISCVII::MO_PLT;
8161
8162 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
8163 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
8164 unsigned OpFlags = RISCVII::MO_CALL;
8165
8166 if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
8167 nullptr))
8168 OpFlags = RISCVII::MO_PLT;
8169
8170 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
8171 }
8172
8173 // The first call operand is the chain and the second is the target address.
8174 SmallVector<SDValue, 8> Ops;
8175 Ops.push_back(Chain);
8176 Ops.push_back(Callee);
8177
8178 // Add argument registers to the end of the list so that they are
8179 // known live into the call.
8180 for (auto &Reg : RegsToPass)
8181 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
8182
8183 if (!IsTailCall) {
8184 // Add a register mask operand representing the call-preserved registers.
8185 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
8186 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
8187 assert(Mask && "Missing call preserved mask for calling convention");
8188 Ops.push_back(DAG.getRegisterMask(Mask));
8189 }
8190
8191 // Glue the call to the argument copies, if any.
8192 if (Glue.getNode())
8193 Ops.push_back(Glue);
8194
8195 // Emit the call.
8196 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8197
8198 if (IsTailCall) {
8199 MF.getFrameInfo().setHasTailCall();
8200 return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
8201 }
8202
8203 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
8204 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
8205 Glue = Chain.getValue(1);
8206
8207 // Mark the end of the call, which is glued to the call itself.
8208 Chain = DAG.getCALLSEQ_END(Chain,
8209 DAG.getConstant(NumBytes, DL, PtrVT, true),
8210 DAG.getConstant(0, DL, PtrVT, true),
8211 Glue, DL);
8212 Glue = Chain.getValue(1);
8213
8214 // Assign locations to each value returned by this call.
8215 SmallVector<CCValAssign, 16> RVLocs;
8216 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
8217 analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_RISCV);
8218
8219 // Copy all of the result registers out of their specified physreg.
8220 for (auto &VA : RVLocs) {
8221 // Copy the value out
8222 SDValue RetValue =
8223 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
8224 // Glue the RetValue to the end of the call sequence
8225 Chain = RetValue.getValue(1);
8226 Glue = RetValue.getValue(2);
8227
8228 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
8229 assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
8230 SDValue RetValue2 =
8231 DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
8232 Chain = RetValue2.getValue(1);
8233 Glue = RetValue2.getValue(2);
8234 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
8235 RetValue2);
8236 }
8237
8238 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL, Subtarget);
8239
8240 InVals.push_back(RetValue);
8241 }
8242
8243 return Chain;
8244 }
8245
8246 bool RISCVTargetLowering::CanLowerReturn(
8247 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
8248 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
8249 SmallVector<CCValAssign, 16> RVLocs;
8250 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
8251
8252 Optional<unsigned> FirstMaskArgument;
8253 if (Subtarget.hasStdExtV())
8254 FirstMaskArgument = preAssignMask(Outs);
8255
8256 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
8257 MVT VT = Outs[i].VT;
8258 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
8259 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
8260 if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
8261 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr,
8262 *this, FirstMaskArgument))
8263 return false;
8264 }
8265 return true;
8266 }
8267
8268 SDValue
8269 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
8270 bool IsVarArg,
8271 const SmallVectorImpl<ISD::OutputArg> &Outs,
8272 const SmallVectorImpl<SDValue> &OutVals,
8273 const SDLoc &DL, SelectionDAG &DAG) const {
8274 const MachineFunction &MF = DAG.getMachineFunction();
8275 const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
8276
8277 // Stores the assignment of the return value to a location.
8278 SmallVector<CCValAssign, 16> RVLocs;
8279
8280 // Info about the registers and stack slot.
8281 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
8282 *DAG.getContext());
8283
8284 analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
8285 nullptr, CC_RISCV);
8286
8287 if (CallConv == CallingConv::GHC && !RVLocs.empty())
8288 report_fatal_error("GHC functions return void only");
8289
8290 SDValue Glue;
8291 SmallVector<SDValue, 4> RetOps(1, Chain);
8292
8293 // Copy the result values into the output registers.
8294 for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
8295 SDValue Val = OutVals[i];
8296 CCValAssign &VA = RVLocs[i];
8297 assert(VA.isRegLoc() && "Can only return in registers!");
8298
8299 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
8300 // Handle returning f64 on RV32D with a soft float ABI.
8301 assert(VA.isRegLoc() && "Expected return via registers");
8302 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
8303 DAG.getVTList(MVT::i32, MVT::i32), Val);
8304 SDValue Lo = SplitF64.getValue(0);
8305 SDValue Hi = SplitF64.getValue(1);
8306 Register RegLo = VA.getLocReg();
8307 assert(RegLo < RISCV::X31 && "Invalid register pair");
8308 Register RegHi = RegLo + 1;
8309
8310 if (STI.isRegisterReservedByUser(RegLo) ||
8311 STI.isRegisterReservedByUser(RegHi))
8312 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
8313 MF.getFunction(),
8314 "Return value register required, but has been reserved."});
8315
8316 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
8317 Glue = Chain.getValue(1);
8318 RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
8319 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
8320 Glue = Chain.getValue(1);
8321 RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
8322 } else {
8323 // Handle a 'normal' return.
8324 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
8325 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
8326
8327 if (STI.isRegisterReservedByUser(VA.getLocReg()))
8328 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
8329 MF.getFunction(),
8330 "Return value register required, but has been reserved."});
8331
8332 // Guarantee that all emitted copies are stuck together.
8333 Glue = Chain.getValue(1);
8334 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
8335 }
8336 }
8337
8338 RetOps[0] = Chain; // Update chain.
8339
8340 // Add the glue node if we have it.
8341 if (Glue.getNode()) {
8342 RetOps.push_back(Glue);
8343 }
8344
8345 unsigned RetOpc = RISCVISD::RET_FLAG;
8346 // Interrupt service routines use different return instructions.
8347 const Function &Func = DAG.getMachineFunction().getFunction();
8348 if (Func.hasFnAttribute("interrupt")) {
8349 if (!Func.getReturnType()->isVoidTy())
8350 report_fatal_error(
8351 "Functions with the interrupt attribute must have void return type!");
8352
8353 MachineFunction &MF = DAG.getMachineFunction();
8354 StringRef Kind =
8355 MF.getFunction().getFnAttribute("interrupt").getValueAsString();
8356
8357 if (Kind == "user")
8358 RetOpc = RISCVISD::URET_FLAG;
8359 else if (Kind == "supervisor")
8360 RetOpc = RISCVISD::SRET_FLAG;
8361 else
8362 RetOpc = RISCVISD::MRET_FLAG;
8363 }
8364
8365 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
8366 }
8367
8368 void RISCVTargetLowering::validateCCReservedRegs(
8369 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
8370 MachineFunction &MF) const {
8371 const Function &F = MF.getFunction();
8372 const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
8373
8374 if (llvm::any_of(Regs, [&STI](auto Reg) {
8375 return STI.isRegisterReservedByUser(Reg.first);
8376 }))
8377 F.getContext().diagnose(DiagnosticInfoUnsupported{
8378 F, "Argument register required, but has been reserved."});
8379 }
8380
8381 bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
8382 return CI->isTailCall();
8383 }
8384
8385 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
8386 #define NODE_NAME_CASE(NODE) \
8387 case RISCVISD::NODE: \
8388 return "RISCVISD::" #NODE;
8389 // clang-format off
8390 switch ((RISCVISD::NodeType)Opcode) {
8391 case RISCVISD::FIRST_NUMBER:
8392 break;
8393 NODE_NAME_CASE(RET_FLAG)
8394 NODE_NAME_CASE(URET_FLAG)
8395 NODE_NAME_CASE(SRET_FLAG)
8396 NODE_NAME_CASE(MRET_FLAG)
8397 NODE_NAME_CASE(CALL)
8398 NODE_NAME_CASE(SELECT_CC)
8399 NODE_NAME_CASE(BR_CC)
8400 NODE_NAME_CASE(BuildPairF64)
8401 NODE_NAME_CASE(SplitF64)
8402 NODE_NAME_CASE(TAIL)
8403 NODE_NAME_CASE(MULHSU)
8404 NODE_NAME_CASE(SLLW)
8405 NODE_NAME_CASE(SRAW)
8406 NODE_NAME_CASE(SRLW)
8407 NODE_NAME_CASE(DIVW)
8408 NODE_NAME_CASE(DIVUW)
8409 NODE_NAME_CASE(REMUW)
8410 NODE_NAME_CASE(ROLW)
8411 NODE_NAME_CASE(RORW)
8412 NODE_NAME_CASE(CLZW)
8413 NODE_NAME_CASE(CTZW)
8414 NODE_NAME_CASE(FSLW)
8415 NODE_NAME_CASE(FSRW)
8416 NODE_NAME_CASE(FSL)
8417 NODE_NAME_CASE(FSR)
8418 NODE_NAME_CASE(FMV_H_X)
8419 NODE_NAME_CASE(FMV_X_ANYEXTH)
8420 NODE_NAME_CASE(FMV_W_X_RV64)
8421 NODE_NAME_CASE(FMV_X_ANYEXTW_RV64)
8422 NODE_NAME_CASE(FCVT_X_RTZ)
8423 NODE_NAME_CASE(FCVT_XU_RTZ)
8424 NODE_NAME_CASE(FCVT_W_RTZ_RV64)
8425 NODE_NAME_CASE(FCVT_WU_RTZ_RV64)
8426 NODE_NAME_CASE(READ_CYCLE_WIDE)
8427 NODE_NAME_CASE(GREV)
8428 NODE_NAME_CASE(GREVW)
8429 NODE_NAME_CASE(GORC)
8430 NODE_NAME_CASE(GORCW)
8431 NODE_NAME_CASE(SHFL)
8432 NODE_NAME_CASE(SHFLW)
8433 NODE_NAME_CASE(UNSHFL)
8434 NODE_NAME_CASE(UNSHFLW)
8435 NODE_NAME_CASE(BCOMPRESS)
8436 NODE_NAME_CASE(BCOMPRESSW)
8437 NODE_NAME_CASE(BDECOMPRESS)
8438 NODE_NAME_CASE(BDECOMPRESSW)
8439 NODE_NAME_CASE(VMV_V_X_VL)
8440 NODE_NAME_CASE(VFMV_V_F_VL)
8441 NODE_NAME_CASE(VMV_X_S)
8442 NODE_NAME_CASE(VMV_S_X_VL)
8443 NODE_NAME_CASE(VFMV_S_F_VL)
8444 NODE_NAME_CASE(SPLAT_VECTOR_I64)
8445 NODE_NAME_CASE(SPLAT_VECTOR_SPLIT_I64_VL)
8446 NODE_NAME_CASE(READ_VLENB)
8447 NODE_NAME_CASE(TRUNCATE_VECTOR_VL)
8448 NODE_NAME_CASE(VSLIDEUP_VL)
8449 NODE_NAME_CASE(VSLIDE1UP_VL)
8450 NODE_NAME_CASE(VSLIDEDOWN_VL)
8451 NODE_NAME_CASE(VSLIDE1DOWN_VL)
8452 NODE_NAME_CASE(VID_VL)
8453 NODE_NAME_CASE(VFNCVT_ROD_VL)
8454 NODE_NAME_CASE(VECREDUCE_ADD_VL)
8455 NODE_NAME_CASE(VECREDUCE_UMAX_VL)
8456 NODE_NAME_CASE(VECREDUCE_SMAX_VL)
8457 NODE_NAME_CASE(VECREDUCE_UMIN_VL)
8458 NODE_NAME_CASE(VECREDUCE_SMIN_VL)
8459 NODE_NAME_CASE(VECREDUCE_AND_VL)
8460 NODE_NAME_CASE(VECREDUCE_OR_VL)
8461 NODE_NAME_CASE(VECREDUCE_XOR_VL)
8462 NODE_NAME_CASE(VECREDUCE_FADD_VL)
8463 NODE_NAME_CASE(VECREDUCE_SEQ_FADD_VL)
8464 NODE_NAME_CASE(VECREDUCE_FMIN_VL)
8465 NODE_NAME_CASE(VECREDUCE_FMAX_VL)
8466 NODE_NAME_CASE(ADD_VL)
8467 NODE_NAME_CASE(AND_VL)
8468 NODE_NAME_CASE(MUL_VL)
8469 NODE_NAME_CASE(OR_VL)
8470 NODE_NAME_CASE(SDIV_VL)
8471 NODE_NAME_CASE(SHL_VL)
8472 NODE_NAME_CASE(SREM_VL)
8473 NODE_NAME_CASE(SRA_VL)
8474 NODE_NAME_CASE(SRL_VL)
8475 NODE_NAME_CASE(SUB_VL)
8476 NODE_NAME_CASE(UDIV_VL)
8477 NODE_NAME_CASE(UREM_VL)
8478 NODE_NAME_CASE(XOR_VL)
8479 NODE_NAME_CASE(SADDSAT_VL)
8480 NODE_NAME_CASE(UADDSAT_VL)
8481 NODE_NAME_CASE(SSUBSAT_VL)
8482 NODE_NAME_CASE(USUBSAT_VL)
8483 NODE_NAME_CASE(FADD_VL)
8484 NODE_NAME_CASE(FSUB_VL)
8485 NODE_NAME_CASE(FMUL_VL)
8486 NODE_NAME_CASE(FDIV_VL)
8487 NODE_NAME_CASE(FNEG_VL)
8488 NODE_NAME_CASE(FABS_VL)
8489 NODE_NAME_CASE(FSQRT_VL)
8490 NODE_NAME_CASE(FMA_VL)
8491 NODE_NAME_CASE(FCOPYSIGN_VL)
8492 NODE_NAME_CASE(SMIN_VL)
8493 NODE_NAME_CASE(SMAX_VL)
8494 NODE_NAME_CASE(UMIN_VL)
8495 NODE_NAME_CASE(UMAX_VL)
8496 NODE_NAME_CASE(FMINNUM_VL)
8497 NODE_NAME_CASE(FMAXNUM_VL)
8498 NODE_NAME_CASE(MULHS_VL)
8499 NODE_NAME_CASE(MULHU_VL)
8500 NODE_NAME_CASE(FP_TO_SINT_VL)
8501 NODE_NAME_CASE(FP_TO_UINT_VL)
8502 NODE_NAME_CASE(SINT_TO_FP_VL)
8503 NODE_NAME_CASE(UINT_TO_FP_VL)
8504 NODE_NAME_CASE(FP_EXTEND_VL)
8505 NODE_NAME_CASE(FP_ROUND_VL)
8506 NODE_NAME_CASE(VWMUL_VL)
8507 NODE_NAME_CASE(VWMULU_VL)
8508 NODE_NAME_CASE(SETCC_VL)
8509 NODE_NAME_CASE(VSELECT_VL)
8510 NODE_NAME_CASE(VMAND_VL)
8511 NODE_NAME_CASE(VMOR_VL)
8512 NODE_NAME_CASE(VMXOR_VL)
8513 NODE_NAME_CASE(VMCLR_VL)
8514 NODE_NAME_CASE(VMSET_VL)
8515 NODE_NAME_CASE(VRGATHER_VX_VL)
8516 NODE_NAME_CASE(VRGATHER_VV_VL)
8517 NODE_NAME_CASE(VRGATHEREI16_VV_VL)
8518 NODE_NAME_CASE(VSEXT_VL)
8519 NODE_NAME_CASE(VZEXT_VL)
8520 NODE_NAME_CASE(VPOPC_VL)
8521 NODE_NAME_CASE(VLE_VL)
8522 NODE_NAME_CASE(VSE_VL)
8523 NODE_NAME_CASE(READ_CSR)
8524 NODE_NAME_CASE(WRITE_CSR)
8525 NODE_NAME_CASE(SWAP_CSR)
8526 }
8527 // clang-format on
8528 return nullptr;
8529 #undef NODE_NAME_CASE
8530 }
8531
8532 /// getConstraintType - Given a constraint letter, return the type of
8533 /// constraint it is for this target.
8534 RISCVTargetLowering::ConstraintType
8535 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
8536 if (Constraint.size() == 1) {
8537 switch (Constraint[0]) {
8538 default:
8539 break;
8540 case 'f':
8541 return C_RegisterClass;
8542 case 'I':
8543 case 'J':
8544 case 'K':
8545 return C_Immediate;
8546 case 'A':
8547 return C_Memory;
8548 case 'S': // A symbolic address
8549 return C_Other;
8550 }
8551 } else {
8552 if (Constraint == "vr" || Constraint == "vm")
8553 return C_RegisterClass;
8554 }
8555 return TargetLowering::getConstraintType(Constraint);
8556 }
8557
8558 std::pair<unsigned, const TargetRegisterClass *>
8559 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
8560 StringRef Constraint,
8561 MVT VT) const {
8562 // First, see if this is a constraint that directly corresponds to a
8563 // RISCV register class.
8564 if (Constraint.size() == 1) {
8565 switch (Constraint[0]) {
8566 case 'r':
8567 return std::make_pair(0U, &RISCV::GPRRegClass);
8568 case 'f':
8569 if (Subtarget.hasStdExtZfh() && VT == MVT::f16)
8570 return std::make_pair(0U, &RISCV::FPR16RegClass);
8571 if (Subtarget.hasStdExtF() && VT == MVT::f32)
8572 return std::make_pair(0U, &RISCV::FPR32RegClass);
8573 if (Subtarget.hasStdExtD() && VT == MVT::f64)
8574 return std::make_pair(0U, &RISCV::FPR64RegClass);
8575 break;
8576 default:
8577 break;
8578 }
8579 } else {
8580 if (Constraint == "vr") {
8581 for (const auto *RC : {&RISCV::VRRegClass, &RISCV::VRM2RegClass,
8582 &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
8583 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy))
8584 return std::make_pair(0U, RC);
8585 }
8586 } else if (Constraint == "vm") {
8587 if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
8588 return std::make_pair(0U, &RISCV::VMRegClass);
8589 }
8590 }
8591
8592 // Clang will correctly decode the usage of register name aliases into their
8593 // official names. However, other frontends like `rustc` do not. This allows
8594 // users of these frontends to use the ABI names for registers in LLVM-style
8595 // register constraints.
8596 unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
8597 .Case("{zero}", RISCV::X0)
8598 .Case("{ra}", RISCV::X1)
8599 .Case("{sp}", RISCV::X2)
8600 .Case("{gp}", RISCV::X3)
8601 .Case("{tp}", RISCV::X4)
8602 .Case("{t0}", RISCV::X5)
8603 .Case("{t1}", RISCV::X6)
8604 .Case("{t2}", RISCV::X7)
8605 .Cases("{s0}", "{fp}", RISCV::X8)
8606 .Case("{s1}", RISCV::X9)
8607 .Case("{a0}", RISCV::X10)
8608 .Case("{a1}", RISCV::X11)
8609 .Case("{a2}", RISCV::X12)
8610 .Case("{a3}", RISCV::X13)
8611 .Case("{a4}", RISCV::X14)
8612 .Case("{a5}", RISCV::X15)
8613 .Case("{a6}", RISCV::X16)
8614 .Case("{a7}", RISCV::X17)
8615 .Case("{s2}", RISCV::X18)
8616 .Case("{s3}", RISCV::X19)
8617 .Case("{s4}", RISCV::X20)
8618 .Case("{s5}", RISCV::X21)
8619 .Case("{s6}", RISCV::X22)
8620 .Case("{s7}", RISCV::X23)
8621 .Case("{s8}", RISCV::X24)
8622 .Case("{s9}", RISCV::X25)
8623 .Case("{s10}", RISCV::X26)
8624 .Case("{s11}", RISCV::X27)
8625 .Case("{t3}", RISCV::X28)
8626 .Case("{t4}", RISCV::X29)
8627 .Case("{t5}", RISCV::X30)
8628 .Case("{t6}", RISCV::X31)
8629 .Default(RISCV::NoRegister);
8630 if (XRegFromAlias != RISCV::NoRegister)
8631 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
8632
8633 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
8634 // TableGen record rather than the AsmName to choose registers for InlineAsm
8635 // constraints, plus we want to match those names to the widest floating point
8636 // register type available, manually select floating point registers here.
8637 //
8638 // The second case is the ABI name of the register, so that frontends can also
8639 // use the ABI names in register constraint lists.
8640 if (Subtarget.hasStdExtF()) {
8641 unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
8642 .Cases("{f0}", "{ft0}", RISCV::F0_F)
8643 .Cases("{f1}", "{ft1}", RISCV::F1_F)
8644 .Cases("{f2}", "{ft2}", RISCV::F2_F)
8645 .Cases("{f3}", "{ft3}", RISCV::F3_F)
8646 .Cases("{f4}", "{ft4}", RISCV::F4_F)
8647 .Cases("{f5}", "{ft5}", RISCV::F5_F)
8648 .Cases("{f6}", "{ft6}", RISCV::F6_F)
8649 .Cases("{f7}", "{ft7}", RISCV::F7_F)
8650 .Cases("{f8}", "{fs0}", RISCV::F8_F)
8651 .Cases("{f9}", "{fs1}", RISCV::F9_F)
8652 .Cases("{f10}", "{fa0}", RISCV::F10_F)
8653 .Cases("{f11}", "{fa1}", RISCV::F11_F)
8654 .Cases("{f12}", "{fa2}", RISCV::F12_F)
8655 .Cases("{f13}", "{fa3}", RISCV::F13_F)
8656 .Cases("{f14}", "{fa4}", RISCV::F14_F)
8657 .Cases("{f15}", "{fa5}", RISCV::F15_F)
8658 .Cases("{f16}", "{fa6}", RISCV::F16_F)
8659 .Cases("{f17}", "{fa7}", RISCV::F17_F)
8660 .Cases("{f18}", "{fs2}", RISCV::F18_F)
8661 .Cases("{f19}", "{fs3}", RISCV::F19_F)
8662 .Cases("{f20}", "{fs4}", RISCV::F20_F)
8663 .Cases("{f21}", "{fs5}", RISCV::F21_F)
8664 .Cases("{f22}", "{fs6}", RISCV::F22_F)
8665 .Cases("{f23}", "{fs7}", RISCV::F23_F)
8666 .Cases("{f24}", "{fs8}", RISCV::F24_F)
8667 .Cases("{f25}", "{fs9}", RISCV::F25_F)
8668 .Cases("{f26}", "{fs10}", RISCV::F26_F)
8669 .Cases("{f27}", "{fs11}", RISCV::F27_F)
8670 .Cases("{f28}", "{ft8}", RISCV::F28_F)
8671 .Cases("{f29}", "{ft9}", RISCV::F29_F)
8672 .Cases("{f30}", "{ft10}", RISCV::F30_F)
8673 .Cases("{f31}", "{ft11}", RISCV::F31_F)
8674 .Default(RISCV::NoRegister);
8675 if (FReg != RISCV::NoRegister) {
8676 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
8677 if (Subtarget.hasStdExtD()) {
8678 unsigned RegNo = FReg - RISCV::F0_F;
8679 unsigned DReg = RISCV::F0_D + RegNo;
8680 return std::make_pair(DReg, &RISCV::FPR64RegClass);
8681 }
8682 return std::make_pair(FReg, &RISCV::FPR32RegClass);
8683 }
8684 }
8685
8686 if (Subtarget.hasStdExtV()) {
8687 Register VReg = StringSwitch<Register>(Constraint.lower())
8688 .Case("{v0}", RISCV::V0)
8689 .Case("{v1}", RISCV::V1)
8690 .Case("{v2}", RISCV::V2)
8691 .Case("{v3}", RISCV::V3)
8692 .Case("{v4}", RISCV::V4)
8693 .Case("{v5}", RISCV::V5)
8694 .Case("{v6}", RISCV::V6)
8695 .Case("{v7}", RISCV::V7)
8696 .Case("{v8}", RISCV::V8)
8697 .Case("{v9}", RISCV::V9)
8698 .Case("{v10}", RISCV::V10)
8699 .Case("{v11}", RISCV::V11)
8700 .Case("{v12}", RISCV::V12)
8701 .Case("{v13}", RISCV::V13)
8702 .Case("{v14}", RISCV::V14)
8703 .Case("{v15}", RISCV::V15)
8704 .Case("{v16}", RISCV::V16)
8705 .Case("{v17}", RISCV::V17)
8706 .Case("{v18}", RISCV::V18)
8707 .Case("{v19}", RISCV::V19)
8708 .Case("{v20}", RISCV::V20)
8709 .Case("{v21}", RISCV::V21)
8710 .Case("{v22}", RISCV::V22)
8711 .Case("{v23}", RISCV::V23)
8712 .Case("{v24}", RISCV::V24)
8713 .Case("{v25}", RISCV::V25)
8714 .Case("{v26}", RISCV::V26)
8715 .Case("{v27}", RISCV::V27)
8716 .Case("{v28}", RISCV::V28)
8717 .Case("{v29}", RISCV::V29)
8718 .Case("{v30}", RISCV::V30)
8719 .Case("{v31}", RISCV::V31)
8720 .Default(RISCV::NoRegister);
8721 if (VReg != RISCV::NoRegister) {
8722 if (TRI->isTypeLegalForClass(RISCV::VMRegClass, VT.SimpleTy))
8723 return std::make_pair(VReg, &RISCV::VMRegClass);
8724 if (TRI->isTypeLegalForClass(RISCV::VRRegClass, VT.SimpleTy))
8725 return std::make_pair(VReg, &RISCV::VRRegClass);
8726 for (const auto *RC :
8727 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
8728 if (TRI->isTypeLegalForClass(*RC, VT.SimpleTy)) {
8729 VReg = TRI->getMatchingSuperReg(VReg, RISCV::sub_vrm1_0, RC);
8730 return std::make_pair(VReg, RC);
8731 }
8732 }
8733 }
8734 }
8735
8736 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
8737 }
8738
8739 unsigned
8740 RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
8741 // Currently only support length 1 constraints.
8742 if (ConstraintCode.size() == 1) {
8743 switch (ConstraintCode[0]) {
8744 case 'A':
8745 return InlineAsm::Constraint_A;
8746 default:
8747 break;
8748 }
8749 }
8750
8751 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
8752 }
8753
8754 void RISCVTargetLowering::LowerAsmOperandForConstraint(
8755 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
8756 SelectionDAG &DAG) const {
8757 // Currently only support length 1 constraints.
8758 if (Constraint.length() == 1) {
8759 switch (Constraint[0]) {
8760 case 'I':
8761 // Validate & create a 12-bit signed immediate operand.
8762 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
8763 uint64_t CVal = C->getSExtValue();
8764 if (isInt<12>(CVal))
8765 Ops.push_back(
8766 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
8767 }
8768 return;
8769 case 'J':
8770 // Validate & create an integer zero operand.
8771 if (auto *C = dyn_cast<ConstantSDNode>(Op))
8772 if (C->getZExtValue() == 0)
8773 Ops.push_back(
8774 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
8775 return;
8776 case 'K':
8777 // Validate & create a 5-bit unsigned immediate operand.
8778 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
8779 uint64_t CVal = C->getZExtValue();
8780 if (isUInt<5>(CVal))
8781 Ops.push_back(
8782 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
8783 }
8784 return;
8785 case 'S':
8786 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) {
8787 Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op),
8788 GA->getValueType(0)));
8789 } else if (const auto *BA = dyn_cast<BlockAddressSDNode>(Op)) {
8790 Ops.push_back(DAG.getTargetBlockAddress(BA->getBlockAddress(),
8791 BA->getValueType(0)));
8792 }
8793 return;
8794 default:
8795 break;
8796 }
8797 }
8798 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
8799 }
8800
8801 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
8802 Instruction *Inst,
8803 AtomicOrdering Ord) const {
8804 if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
8805 return Builder.CreateFence(Ord);
8806 if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
8807 return Builder.CreateFence(AtomicOrdering::Release);
8808 return nullptr;
8809 }
8810
8811 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
8812 Instruction *Inst,
8813 AtomicOrdering Ord) const {
8814 if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
8815 return Builder.CreateFence(AtomicOrdering::Acquire);
8816 return nullptr;
8817 }
8818
8819 TargetLowering::AtomicExpansionKind
8820 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
8821 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
8822 // point operations can't be used in an lr/sc sequence without breaking the
8823 // forward-progress guarantee.
8824 if (AI->isFloatingPointOperation())
8825 return AtomicExpansionKind::CmpXChg;
8826
8827 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
8828 if (Size == 8 || Size == 16)
8829 return AtomicExpansionKind::MaskedIntrinsic;
8830 return AtomicExpansionKind::None;
8831 }
8832
8833 static Intrinsic::ID
8834 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
8835 if (XLen == 32) {
8836 switch (BinOp) {
8837 default:
8838 llvm_unreachable("Unexpected AtomicRMW BinOp");
8839 case AtomicRMWInst::Xchg:
8840 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
8841 case AtomicRMWInst::Add:
8842 return Intrinsic::riscv_masked_atomicrmw_add_i32;
8843 case AtomicRMWInst::Sub:
8844 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
8845 case AtomicRMWInst::Nand:
8846 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
8847 case AtomicRMWInst::Max:
8848 return Intrinsic::riscv_masked_atomicrmw_max_i32;
8849 case AtomicRMWInst::Min:
8850 return Intrinsic::riscv_masked_atomicrmw_min_i32;
8851 case AtomicRMWInst::UMax:
8852 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
8853 case AtomicRMWInst::UMin:
8854 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
8855 }
8856 }
8857
8858 if (XLen == 64) {
8859 switch (BinOp) {
8860 default:
8861 llvm_unreachable("Unexpected AtomicRMW BinOp");
8862 case AtomicRMWInst::Xchg:
8863 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
8864 case AtomicRMWInst::Add:
8865 return Intrinsic::riscv_masked_atomicrmw_add_i64;
8866 case AtomicRMWInst::Sub:
8867 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
8868 case AtomicRMWInst::Nand:
8869 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
8870 case AtomicRMWInst::Max:
8871 return Intrinsic::riscv_masked_atomicrmw_max_i64;
8872 case AtomicRMWInst::Min:
8873 return Intrinsic::riscv_masked_atomicrmw_min_i64;
8874 case AtomicRMWInst::UMax:
8875 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
8876 case AtomicRMWInst::UMin:
8877 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
8878 }
8879 }
8880
8881 llvm_unreachable("Unexpected XLen\n");
8882 }
8883
8884 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
8885 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
8886 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
8887 unsigned XLen = Subtarget.getXLen();
8888 Value *Ordering =
8889 Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
8890 Type *Tys[] = {AlignedAddr->getType()};
8891 Function *LrwOpScwLoop = Intrinsic::getDeclaration(
8892 AI->getModule(),
8893 getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
8894
8895 if (XLen == 64) {
8896 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
8897 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
8898 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
8899 }
8900
8901 Value *Result;
8902
8903 // Must pass the shift amount needed to sign extend the loaded value prior
8904 // to performing a signed comparison for min/max. ShiftAmt is the number of
8905 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
8906 // is the number of bits to left+right shift the value in order to
8907 // sign-extend.
8908 if (AI->getOperation() == AtomicRMWInst::Min ||
8909 AI->getOperation() == AtomicRMWInst::Max) {
8910 const DataLayout &DL = AI->getModule()->getDataLayout();
8911 unsigned ValWidth =
8912 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
8913 Value *SextShamt =
8914 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
8915 Result = Builder.CreateCall(LrwOpScwLoop,
8916 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
8917 } else {
8918 Result =
8919 Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
8920 }
8921
8922 if (XLen == 64)
8923 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
8924 return Result;
8925 }
8926
8927 TargetLowering::AtomicExpansionKind
8928 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
8929 AtomicCmpXchgInst *CI) const {
8930 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
8931 if (Size == 8 || Size == 16)
8932 return AtomicExpansionKind::MaskedIntrinsic;
8933 return AtomicExpansionKind::None;
8934 }
8935
8936 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
8937 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
8938 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
8939 unsigned XLen = Subtarget.getXLen();
8940 Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
8941 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
8942 if (XLen == 64) {
8943 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
8944 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
8945 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
8946 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
8947 }
8948 Type *Tys[] = {AlignedAddr->getType()};
8949 Function *MaskedCmpXchg =
8950 Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
8951 Value *Result = Builder.CreateCall(
8952 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
8953 if (XLen == 64)
8954 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
8955 return Result;
8956 }
8957
8958 bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(EVT VT) const {
8959 return false;
8960 }
8961
8962 bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
8963 EVT VT) const {
8964 VT = VT.getScalarType();
8965
8966 if (!VT.isSimple())
8967 return false;
8968
8969 switch (VT.getSimpleVT().SimpleTy) {
8970 case MVT::f16:
8971 return Subtarget.hasStdExtZfh();
8972 case MVT::f32:
8973 return Subtarget.hasStdExtF();
8974 case MVT::f64:
8975 return Subtarget.hasStdExtD();
8976 default:
8977 break;
8978 }
8979
8980 return false;
8981 }
8982
8983 Register RISCVTargetLowering::getExceptionPointerRegister(
8984 const Constant *PersonalityFn) const {
8985 return RISCV::X10;
8986 }
8987
8988 Register RISCVTargetLowering::getExceptionSelectorRegister(
8989 const Constant *PersonalityFn) const {
8990 return RISCV::X11;
8991 }
8992
8993 bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
8994 // Return false to suppress the unnecessary extensions if the LibCall
8995 // arguments or return value is f32 type for LP64 ABI.
8996 RISCVABI::ABI ABI = Subtarget.getTargetABI();
8997 if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
8998 return false;
8999
9000 return true;
9001 }
9002
9003 bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
9004 if (Subtarget.is64Bit() && Type == MVT::i32)
9005 return true;
9006
9007 return IsSigned;
9008 }
9009
9010 bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
9011 SDValue C) const {
9012 // Check integral scalar types.
9013 if (VT.isScalarInteger()) {
9014 // Omit the optimization if the sub target has the M extension and the data
9015 // size exceeds XLen.
9016 if (Subtarget.hasStdExtM() && VT.getSizeInBits() > Subtarget.getXLen())
9017 return false;
9018 if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
9019 // Break the MUL to a SLLI and an ADD/SUB.
9020 const APInt &Imm = ConstNode->getAPIntValue();
9021 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
9022 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
9023 return true;
9024 // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
9025 if (Subtarget.hasStdExtZba() && !Imm.isSignedIntN(12) &&
9026 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
9027 (Imm - 8).isPowerOf2()))
9028 return true;
9029 // Omit the following optimization if the sub target has the M extension
9030 // and the data size >= XLen.
9031 if (Subtarget.hasStdExtM() && VT.getSizeInBits() >= Subtarget.getXLen())
9032 return false;
9033 // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
9034 // a pair of LUI/ADDI.
9035 if (!Imm.isSignedIntN(12) && Imm.countTrailingZeros() < 12) {
9036 APInt ImmS = Imm.ashr(Imm.countTrailingZeros());
9037 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
9038 (1 - ImmS).isPowerOf2())
9039 return true;
9040 }
9041 }
9042 }
9043
9044 return false;
9045 }
9046
9047 bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
9048 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
9049 bool *Fast) const {
9050 if (!VT.isVector())
9051 return false;
9052
9053 EVT ElemVT = VT.getVectorElementType();
9054 if (Alignment >= ElemVT.getStoreSize()) {
9055 if (Fast)
9056 *Fast = true;
9057 return true;
9058 }
9059
9060 return false;
9061 }
9062
9063 bool RISCVTargetLowering::splitValueIntoRegisterParts(
9064 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
9065 unsigned NumParts, MVT PartVT, Optional<CallingConv::ID> CC) const {
9066 bool IsABIRegCopy = CC.hasValue();
9067 EVT ValueVT = Val.getValueType();
9068 if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
9069 // Cast the f16 to i16, extend to i32, pad with ones to make a float nan,
9070 // and cast to f32.
9071 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
9072 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
9073 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
9074 DAG.getConstant(0xFFFF0000, DL, MVT::i32));
9075 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
9076 Parts[0] = Val;
9077 return true;
9078 }
9079
9080 if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
9081 LLVMContext &Context = *DAG.getContext();
9082 EVT ValueEltVT = ValueVT.getVectorElementType();
9083 EVT PartEltVT = PartVT.getVectorElementType();
9084 unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
9085 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
9086 if (PartVTBitSize % ValueVTBitSize == 0) {
9087 // If the element types are different, bitcast to the same element type of
9088 // PartVT first.
9089 if (ValueEltVT != PartEltVT) {
9090 unsigned Count = ValueVTBitSize / PartEltVT.getSizeInBits();
9091 assert(Count != 0 && "The number of element should not be zero.");
9092 EVT SameEltTypeVT =
9093 EVT::getVectorVT(Context, PartEltVT, Count, /*IsScalable=*/true);
9094 Val = DAG.getNode(ISD::BITCAST, DL, SameEltTypeVT, Val);
9095 }
9096 Val = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT),
9097 Val, DAG.getConstant(0, DL, Subtarget.getXLenVT()));
9098 Parts[0] = Val;
9099 return true;
9100 }
9101 }
9102 return false;
9103 }
9104
9105 SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
9106 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
9107 MVT PartVT, EVT ValueVT, Optional<CallingConv::ID> CC) const {
9108 bool IsABIRegCopy = CC.hasValue();
9109 if (IsABIRegCopy && ValueVT == MVT::f16 && PartVT == MVT::f32) {
9110 SDValue Val = Parts[0];
9111
9112 // Cast the f32 to i32, truncate to i16, and cast back to f16.
9113 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
9114 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
9115 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f16, Val);
9116 return Val;
9117 }
9118
9119 if (ValueVT.isScalableVector() && PartVT.isScalableVector()) {
9120 LLVMContext &Context = *DAG.getContext();
9121 SDValue Val = Parts[0];
9122 EVT ValueEltVT = ValueVT.getVectorElementType();
9123 EVT PartEltVT = PartVT.getVectorElementType();
9124 unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinSize();
9125 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinSize();
9126 if (PartVTBitSize % ValueVTBitSize == 0) {
9127 EVT SameEltTypeVT = ValueVT;
9128 // If the element types are different, convert it to the same element type
9129 // of PartVT.
9130 if (ValueEltVT != PartEltVT) {
9131 unsigned Count = ValueVTBitSize / PartEltVT.getSizeInBits();
9132 assert(Count != 0 && "The number of element should not be zero.");
9133 SameEltTypeVT =
9134 EVT::getVectorVT(Context, PartEltVT, Count, /*IsScalable=*/true);
9135 }
9136 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SameEltTypeVT, Val,
9137 DAG.getConstant(0, DL, Subtarget.getXLenVT()));
9138 if (ValueEltVT != PartEltVT)
9139 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
9140 return Val;
9141 }
9142 }
9143 return SDValue();
9144 }
9145
9146 #define GET_REGISTER_MATCHER
9147 #include "RISCVGenAsmMatcher.inc"
9148
9149 Register
9150 RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
9151 const MachineFunction &MF) const {
9152 Register Reg = MatchRegisterAltName(RegName);
9153 if (Reg == RISCV::NoRegister)
9154 Reg = MatchRegisterName(RegName);
9155 if (Reg == RISCV::NoRegister)
9156 report_fatal_error(
9157 Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
9158 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
9159 if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
9160 report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
9161 StringRef(RegName) + "\"."));
9162 return Reg;
9163 }
9164
9165 namespace llvm {
9166 namespace RISCVVIntrinsicsTable {
9167
9168 #define GET_RISCVVIntrinsicsTable_IMPL
9169 #include "RISCVGenSearchableTables.inc"
9170
9171 } // namespace RISCVVIntrinsicsTable
9172
9173 } // namespace llvm
LLVM monorepo