[LLVM][Alignment] Introduce Alignment Type
[llvm-core.git] / lib / Target / RISCV / RISCVISelLowering.cpp
blob21a15ecdb44088e7b925481d75dd0b2ed93427d9
1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
12 //===----------------------------------------------------------------------===//
14 #include "RISCVISelLowering.h"
15 #include "RISCV.h"
16 #include "RISCVMachineFunctionInfo.h"
17 #include "RISCVRegisterInfo.h"
18 #include "RISCVSubtarget.h"
19 #include "RISCVTargetMachine.h"
20 #include "Utils/RISCVMatInt.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
23 #include "llvm/CodeGen/CallingConvLower.h"
24 #include "llvm/CodeGen/MachineFrameInfo.h"
25 #include "llvm/CodeGen/MachineFunction.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/CodeGen/MachineRegisterInfo.h"
28 #include "llvm/CodeGen/SelectionDAGISel.h"
29 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30 #include "llvm/CodeGen/ValueTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/DiagnosticPrinter.h"
33 #include "llvm/Support/Debug.h"
34 #include "llvm/Support/ErrorHandling.h"
35 #include "llvm/Support/raw_ostream.h"
37 using namespace llvm;
39 #define DEBUG_TYPE "riscv-lower"
41 STATISTIC(NumTailCalls, "Number of tail calls");
43 RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
44 const RISCVSubtarget &STI)
45 : TargetLowering(TM), Subtarget(STI) {
47 if (Subtarget.isRV32E())
48 report_fatal_error("Codegen not yet implemented for RV32E");
50 RISCVABI::ABI ABI = Subtarget.getTargetABI();
51 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
53 switch (ABI) {
54 default:
55 report_fatal_error("Don't know how to lower this ABI");
56 case RISCVABI::ABI_ILP32:
57 case RISCVABI::ABI_ILP32F:
58 case RISCVABI::ABI_ILP32D:
59 case RISCVABI::ABI_LP64:
60 case RISCVABI::ABI_LP64F:
61 case RISCVABI::ABI_LP64D:
62 break;
65 MVT XLenVT = Subtarget.getXLenVT();
67 // Set up the register classes.
68 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
70 if (Subtarget.hasStdExtF())
71 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
72 if (Subtarget.hasStdExtD())
73 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
75 // Compute derived properties from the register classes.
76 computeRegisterProperties(STI.getRegisterInfo());
78 setStackPointerRegisterToSaveRestore(RISCV::X2);
80 for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
81 setLoadExtAction(N, XLenVT, MVT::i1, Promote);
83 // TODO: add all necessary setOperationAction calls.
84 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
86 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
87 setOperationAction(ISD::BR_CC, XLenVT, Expand);
88 setOperationAction(ISD::SELECT, XLenVT, Custom);
89 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
91 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
92 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
94 setOperationAction(ISD::VASTART, MVT::Other, Custom);
95 setOperationAction(ISD::VAARG, MVT::Other, Expand);
96 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
97 setOperationAction(ISD::VAEND, MVT::Other, Expand);
99 for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
100 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
102 if (Subtarget.is64Bit()) {
103 setOperationAction(ISD::SHL, MVT::i32, Custom);
104 setOperationAction(ISD::SRA, MVT::i32, Custom);
105 setOperationAction(ISD::SRL, MVT::i32, Custom);
108 if (!Subtarget.hasStdExtM()) {
109 setOperationAction(ISD::MUL, XLenVT, Expand);
110 setOperationAction(ISD::MULHS, XLenVT, Expand);
111 setOperationAction(ISD::MULHU, XLenVT, Expand);
112 setOperationAction(ISD::SDIV, XLenVT, Expand);
113 setOperationAction(ISD::UDIV, XLenVT, Expand);
114 setOperationAction(ISD::SREM, XLenVT, Expand);
115 setOperationAction(ISD::UREM, XLenVT, Expand);
118 if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) {
119 setOperationAction(ISD::SDIV, MVT::i32, Custom);
120 setOperationAction(ISD::UDIV, MVT::i32, Custom);
121 setOperationAction(ISD::UREM, MVT::i32, Custom);
124 setOperationAction(ISD::SDIVREM, XLenVT, Expand);
125 setOperationAction(ISD::UDIVREM, XLenVT, Expand);
126 setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
127 setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
129 setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
130 setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
131 setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
133 setOperationAction(ISD::ROTL, XLenVT, Expand);
134 setOperationAction(ISD::ROTR, XLenVT, Expand);
135 setOperationAction(ISD::BSWAP, XLenVT, Expand);
136 setOperationAction(ISD::CTTZ, XLenVT, Expand);
137 setOperationAction(ISD::CTLZ, XLenVT, Expand);
138 setOperationAction(ISD::CTPOP, XLenVT, Expand);
140 ISD::CondCode FPCCToExtend[] = {
141 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
142 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
143 ISD::SETGE, ISD::SETNE};
145 ISD::NodeType FPOpToExtend[] = {
146 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM};
148 if (Subtarget.hasStdExtF()) {
149 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
150 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
151 for (auto CC : FPCCToExtend)
152 setCondCodeAction(CC, MVT::f32, Expand);
153 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
154 setOperationAction(ISD::SELECT, MVT::f32, Custom);
155 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
156 for (auto Op : FPOpToExtend)
157 setOperationAction(Op, MVT::f32, Expand);
160 if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
161 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
163 if (Subtarget.hasStdExtD()) {
164 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
165 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
166 for (auto CC : FPCCToExtend)
167 setCondCodeAction(CC, MVT::f64, Expand);
168 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
169 setOperationAction(ISD::SELECT, MVT::f64, Custom);
170 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
171 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
172 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
173 for (auto Op : FPOpToExtend)
174 setOperationAction(Op, MVT::f64, Expand);
177 setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
178 setOperationAction(ISD::BlockAddress, XLenVT, Custom);
179 setOperationAction(ISD::ConstantPool, XLenVT, Custom);
181 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
183 // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
184 // Unfortunately this can't be determined just from the ISA naming string.
185 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
186 Subtarget.is64Bit() ? Legal : Custom);
188 if (Subtarget.hasStdExtA()) {
189 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
190 setMinCmpXchgSizeInBits(32);
191 } else {
192 setMaxAtomicSizeInBitsSupported(0);
195 setBooleanContents(ZeroOrOneBooleanContent);
197 // Function alignments (log2).
198 unsigned FunctionAlignment = Subtarget.hasStdExtC() ? 1 : 2;
199 setMinFunctionAlignment(FunctionAlignment);
200 setPrefFunctionAlignment(FunctionAlignment);
202 // Effectively disable jump table generation.
203 setMinimumJumpTableEntries(INT_MAX);
206 EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
207 EVT VT) const {
208 if (!VT.isVector())
209 return getPointerTy(DL);
210 return VT.changeVectorElementTypeToInteger();
213 bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
214 const CallInst &I,
215 MachineFunction &MF,
216 unsigned Intrinsic) const {
217 switch (Intrinsic) {
218 default:
219 return false;
220 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
221 case Intrinsic::riscv_masked_atomicrmw_add_i32:
222 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
223 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
224 case Intrinsic::riscv_masked_atomicrmw_max_i32:
225 case Intrinsic::riscv_masked_atomicrmw_min_i32:
226 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
227 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
228 case Intrinsic::riscv_masked_cmpxchg_i32:
229 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
230 Info.opc = ISD::INTRINSIC_W_CHAIN;
231 Info.memVT = MVT::getVT(PtrTy->getElementType());
232 Info.ptrVal = I.getArgOperand(0);
233 Info.offset = 0;
234 Info.align = Align(4);
235 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
236 MachineMemOperand::MOVolatile;
237 return true;
241 bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
242 const AddrMode &AM, Type *Ty,
243 unsigned AS,
244 Instruction *I) const {
245 // No global is ever allowed as a base.
246 if (AM.BaseGV)
247 return false;
249 // Require a 12-bit signed offset.
250 if (!isInt<12>(AM.BaseOffs))
251 return false;
253 switch (AM.Scale) {
254 case 0: // "r+i" or just "i", depending on HasBaseReg.
255 break;
256 case 1:
257 if (!AM.HasBaseReg) // allow "r+i".
258 break;
259 return false; // disallow "r+r" or "r+r+i".
260 default:
261 return false;
264 return true;
267 bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
268 return isInt<12>(Imm);
271 bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
272 return isInt<12>(Imm);
275 // On RV32, 64-bit integers are split into their high and low parts and held
276 // in two different registers, so the trunc is free since the low register can
277 // just be used.
278 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
279 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
280 return false;
281 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
282 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
283 return (SrcBits == 64 && DestBits == 32);
286 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
287 if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
288 !SrcVT.isInteger() || !DstVT.isInteger())
289 return false;
290 unsigned SrcBits = SrcVT.getSizeInBits();
291 unsigned DestBits = DstVT.getSizeInBits();
292 return (SrcBits == 64 && DestBits == 32);
295 bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
296 // Zexts are free if they can be combined with a load.
297 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
298 EVT MemVT = LD->getMemoryVT();
299 if ((MemVT == MVT::i8 || MemVT == MVT::i16 ||
300 (Subtarget.is64Bit() && MemVT == MVT::i32)) &&
301 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
302 LD->getExtensionType() == ISD::ZEXTLOAD))
303 return true;
306 return TargetLowering::isZExtFree(Val, VT2);
309 bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
310 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
313 bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
314 return (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
315 (VT == MVT::f64 && Subtarget.hasStdExtD());
318 // Changes the condition code and swaps operands if necessary, so the SetCC
319 // operation matches one of the comparisons supported directly in the RISC-V
320 // ISA.
321 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
322 switch (CC) {
323 default:
324 break;
325 case ISD::SETGT:
326 case ISD::SETLE:
327 case ISD::SETUGT:
328 case ISD::SETULE:
329 CC = ISD::getSetCCSwappedOperands(CC);
330 std::swap(LHS, RHS);
331 break;
335 // Return the RISC-V branch opcode that matches the given DAG integer
336 // condition code. The CondCode must be one of those supported by the RISC-V
337 // ISA (see normaliseSetCC).
338 static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC) {
339 switch (CC) {
340 default:
341 llvm_unreachable("Unsupported CondCode");
342 case ISD::SETEQ:
343 return RISCV::BEQ;
344 case ISD::SETNE:
345 return RISCV::BNE;
346 case ISD::SETLT:
347 return RISCV::BLT;
348 case ISD::SETGE:
349 return RISCV::BGE;
350 case ISD::SETULT:
351 return RISCV::BLTU;
352 case ISD::SETUGE:
353 return RISCV::BGEU;
357 SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
358 SelectionDAG &DAG) const {
359 switch (Op.getOpcode()) {
360 default:
361 report_fatal_error("unimplemented operand");
362 case ISD::GlobalAddress:
363 return lowerGlobalAddress(Op, DAG);
364 case ISD::BlockAddress:
365 return lowerBlockAddress(Op, DAG);
366 case ISD::ConstantPool:
367 return lowerConstantPool(Op, DAG);
368 case ISD::GlobalTLSAddress:
369 return lowerGlobalTLSAddress(Op, DAG);
370 case ISD::SELECT:
371 return lowerSELECT(Op, DAG);
372 case ISD::VASTART:
373 return lowerVASTART(Op, DAG);
374 case ISD::FRAMEADDR:
375 return lowerFRAMEADDR(Op, DAG);
376 case ISD::RETURNADDR:
377 return lowerRETURNADDR(Op, DAG);
378 case ISD::SHL_PARTS:
379 return lowerShiftLeftParts(Op, DAG);
380 case ISD::SRA_PARTS:
381 return lowerShiftRightParts(Op, DAG, true);
382 case ISD::SRL_PARTS:
383 return lowerShiftRightParts(Op, DAG, false);
384 case ISD::BITCAST: {
385 assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
386 "Unexpected custom legalisation");
387 SDLoc DL(Op);
388 SDValue Op0 = Op.getOperand(0);
389 if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
390 return SDValue();
391 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
392 SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
393 return FPConv;
398 static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
399 SelectionDAG &DAG, unsigned Flags) {
400 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
403 static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
404 SelectionDAG &DAG, unsigned Flags) {
405 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
406 Flags);
409 static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
410 SelectionDAG &DAG, unsigned Flags) {
411 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
412 N->getOffset(), Flags);
415 template <class NodeTy>
416 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
417 bool IsLocal) const {
418 SDLoc DL(N);
419 EVT Ty = getPointerTy(DAG.getDataLayout());
421 if (isPositionIndependent()) {
422 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
423 if (IsLocal)
424 // Use PC-relative addressing to access the symbol. This generates the
425 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
426 // %pcrel_lo(auipc)).
427 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
429 // Use PC-relative addressing to access the GOT for this symbol, then load
430 // the address from the GOT. This generates the pattern (PseudoLA sym),
431 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
432 return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
435 switch (getTargetMachine().getCodeModel()) {
436 default:
437 report_fatal_error("Unsupported code model for lowering");
438 case CodeModel::Small: {
439 // Generate a sequence for accessing addresses within the first 2 GiB of
440 // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
441 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
442 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
443 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
444 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
446 case CodeModel::Medium: {
447 // Generate a sequence for accessing addresses within any 2GiB range within
448 // the address space. This generates the pattern (PseudoLLA sym), which
449 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
450 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
451 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
456 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
457 SelectionDAG &DAG) const {
458 SDLoc DL(Op);
459 EVT Ty = Op.getValueType();
460 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
461 int64_t Offset = N->getOffset();
462 MVT XLenVT = Subtarget.getXLenVT();
464 const GlobalValue *GV = N->getGlobal();
465 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
466 SDValue Addr = getAddr(N, DAG, IsLocal);
468 // In order to maximise the opportunity for common subexpression elimination,
469 // emit a separate ADD node for the global address offset instead of folding
470 // it in the global address node. Later peephole optimisations may choose to
471 // fold it back in when profitable.
472 if (Offset != 0)
473 return DAG.getNode(ISD::ADD, DL, Ty, Addr,
474 DAG.getConstant(Offset, DL, XLenVT));
475 return Addr;
478 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
479 SelectionDAG &DAG) const {
480 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
482 return getAddr(N, DAG);
485 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
486 SelectionDAG &DAG) const {
487 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
489 return getAddr(N, DAG);
492 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
493 SelectionDAG &DAG,
494 bool UseGOT) const {
495 SDLoc DL(N);
496 EVT Ty = getPointerTy(DAG.getDataLayout());
497 const GlobalValue *GV = N->getGlobal();
498 MVT XLenVT = Subtarget.getXLenVT();
500 if (UseGOT) {
501 // Use PC-relative addressing to access the GOT for this TLS symbol, then
502 // load the address from the GOT and add the thread pointer. This generates
503 // the pattern (PseudoLA_TLS_IE sym), which expands to
504 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
505 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
506 SDValue Load =
507 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
509 // Add the thread pointer.
510 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
511 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
514 // Generate a sequence for accessing the address relative to the thread
515 // pointer, with the appropriate adjustment for the thread pointer offset.
516 // This generates the pattern
517 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
518 SDValue AddrHi =
519 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
520 SDValue AddrAdd =
521 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
522 SDValue AddrLo =
523 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
525 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
526 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
527 SDValue MNAdd = SDValue(
528 DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
530 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
533 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
534 SelectionDAG &DAG) const {
535 SDLoc DL(N);
536 EVT Ty = getPointerTy(DAG.getDataLayout());
537 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
538 const GlobalValue *GV = N->getGlobal();
540 // Use a PC-relative addressing mode to access the global dynamic GOT address.
541 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
542 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
543 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
544 SDValue Load =
545 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
547 // Prepare argument list to generate call.
548 ArgListTy Args;
549 ArgListEntry Entry;
550 Entry.Node = Load;
551 Entry.Ty = CallTy;
552 Args.push_back(Entry);
554 // Setup call to __tls_get_addr.
555 TargetLowering::CallLoweringInfo CLI(DAG);
556 CLI.setDebugLoc(DL)
557 .setChain(DAG.getEntryNode())
558 .setLibCallee(CallingConv::C, CallTy,
559 DAG.getExternalSymbol("__tls_get_addr", Ty),
560 std::move(Args));
562 return LowerCallTo(CLI).first;
565 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
566 SelectionDAG &DAG) const {
567 SDLoc DL(Op);
568 EVT Ty = Op.getValueType();
569 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
570 int64_t Offset = N->getOffset();
571 MVT XLenVT = Subtarget.getXLenVT();
573 // Non-PIC TLS lowering should always use the LocalExec model.
574 TLSModel::Model Model = isPositionIndependent()
575 ? getTargetMachine().getTLSModel(N->getGlobal())
576 : TLSModel::LocalExec;
578 SDValue Addr;
579 switch (Model) {
580 case TLSModel::LocalExec:
581 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
582 break;
583 case TLSModel::InitialExec:
584 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
585 break;
586 case TLSModel::LocalDynamic:
587 case TLSModel::GeneralDynamic:
588 Addr = getDynamicTLSAddr(N, DAG);
589 break;
592 // In order to maximise the opportunity for common subexpression elimination,
593 // emit a separate ADD node for the global address offset instead of folding
594 // it in the global address node. Later peephole optimisations may choose to
595 // fold it back in when profitable.
596 if (Offset != 0)
597 return DAG.getNode(ISD::ADD, DL, Ty, Addr,
598 DAG.getConstant(Offset, DL, XLenVT));
599 return Addr;
602 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
603 SDValue CondV = Op.getOperand(0);
604 SDValue TrueV = Op.getOperand(1);
605 SDValue FalseV = Op.getOperand(2);
606 SDLoc DL(Op);
607 MVT XLenVT = Subtarget.getXLenVT();
609 // If the result type is XLenVT and CondV is the output of a SETCC node
610 // which also operated on XLenVT inputs, then merge the SETCC node into the
611 // lowered RISCVISD::SELECT_CC to take advantage of the integer
612 // compare+branch instructions. i.e.:
613 // (select (setcc lhs, rhs, cc), truev, falsev)
614 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
615 if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC &&
616 CondV.getOperand(0).getSimpleValueType() == XLenVT) {
617 SDValue LHS = CondV.getOperand(0);
618 SDValue RHS = CondV.getOperand(1);
619 auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
620 ISD::CondCode CCVal = CC->get();
622 normaliseSetCC(LHS, RHS, CCVal);
624 SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
625 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
626 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
627 return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
630 // Otherwise:
631 // (select condv, truev, falsev)
632 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
633 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
634 SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
636 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
637 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
639 return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
642 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
643 MachineFunction &MF = DAG.getMachineFunction();
644 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
646 SDLoc DL(Op);
647 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
648 getPointerTy(MF.getDataLayout()));
650 // vastart just stores the address of the VarArgsFrameIndex slot into the
651 // memory location argument.
652 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
653 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
654 MachinePointerInfo(SV));
657 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
658 SelectionDAG &DAG) const {
659 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
660 MachineFunction &MF = DAG.getMachineFunction();
661 MachineFrameInfo &MFI = MF.getFrameInfo();
662 MFI.setFrameAddressIsTaken(true);
663 unsigned FrameReg = RI.getFrameRegister(MF);
664 int XLenInBytes = Subtarget.getXLen() / 8;
666 EVT VT = Op.getValueType();
667 SDLoc DL(Op);
668 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
669 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
670 while (Depth--) {
671 int Offset = -(XLenInBytes * 2);
672 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
673 DAG.getIntPtrConstant(Offset, DL));
674 FrameAddr =
675 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
677 return FrameAddr;
680 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
681 SelectionDAG &DAG) const {
682 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
683 MachineFunction &MF = DAG.getMachineFunction();
684 MachineFrameInfo &MFI = MF.getFrameInfo();
685 MFI.setReturnAddressIsTaken(true);
686 MVT XLenVT = Subtarget.getXLenVT();
687 int XLenInBytes = Subtarget.getXLen() / 8;
689 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
690 return SDValue();
692 EVT VT = Op.getValueType();
693 SDLoc DL(Op);
694 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
695 if (Depth) {
696 int Off = -XLenInBytes;
697 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
698 SDValue Offset = DAG.getConstant(Off, DL, VT);
699 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
700 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
701 MachinePointerInfo());
704 // Return the value of the return address register, marking it an implicit
705 // live-in.
706 unsigned Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
707 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
710 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
711 SelectionDAG &DAG) const {
712 SDLoc DL(Op);
713 SDValue Lo = Op.getOperand(0);
714 SDValue Hi = Op.getOperand(1);
715 SDValue Shamt = Op.getOperand(2);
716 EVT VT = Lo.getValueType();
718 // if Shamt-XLEN < 0: // Shamt < XLEN
719 // Lo = Lo << Shamt
720 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
721 // else:
722 // Lo = 0
723 // Hi = Lo << (Shamt-XLEN)
725 SDValue Zero = DAG.getConstant(0, DL, VT);
726 SDValue One = DAG.getConstant(1, DL, VT);
727 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
728 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
729 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
730 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
732 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
733 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
734 SDValue ShiftRightLo =
735 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
736 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
737 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
738 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
740 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
742 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
743 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
745 SDValue Parts[2] = {Lo, Hi};
746 return DAG.getMergeValues(Parts, DL);
749 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
750 bool IsSRA) const {
751 SDLoc DL(Op);
752 SDValue Lo = Op.getOperand(0);
753 SDValue Hi = Op.getOperand(1);
754 SDValue Shamt = Op.getOperand(2);
755 EVT VT = Lo.getValueType();
757 // SRA expansion:
758 // if Shamt-XLEN < 0: // Shamt < XLEN
759 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
760 // Hi = Hi >>s Shamt
761 // else:
762 // Lo = Hi >>s (Shamt-XLEN);
763 // Hi = Hi >>s (XLEN-1)
765 // SRL expansion:
766 // if Shamt-XLEN < 0: // Shamt < XLEN
767 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
768 // Hi = Hi >>u Shamt
769 // else:
770 // Lo = Hi >>u (Shamt-XLEN);
771 // Hi = 0;
773 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
775 SDValue Zero = DAG.getConstant(0, DL, VT);
776 SDValue One = DAG.getConstant(1, DL, VT);
777 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
778 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
779 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
780 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
782 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
783 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
784 SDValue ShiftLeftHi =
785 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
786 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
787 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
788 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
789 SDValue HiFalse =
790 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
792 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
794 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
795 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
797 SDValue Parts[2] = {Lo, Hi};
798 return DAG.getMergeValues(Parts, DL);
801 // Returns the opcode of the target-specific SDNode that implements the 32-bit
802 // form of the given Opcode.
803 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
804 switch (Opcode) {
805 default:
806 llvm_unreachable("Unexpected opcode");
807 case ISD::SHL:
808 return RISCVISD::SLLW;
809 case ISD::SRA:
810 return RISCVISD::SRAW;
811 case ISD::SRL:
812 return RISCVISD::SRLW;
813 case ISD::SDIV:
814 return RISCVISD::DIVW;
815 case ISD::UDIV:
816 return RISCVISD::DIVUW;
817 case ISD::UREM:
818 return RISCVISD::REMUW;
822 // Converts the given 32-bit operation to a target-specific SelectionDAG node.
823 // Because i32 isn't a legal type for RV64, these operations would otherwise
824 // be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
825 // later one because the fact the operation was originally of type i32 is
826 // lost.
827 static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG) {
828 SDLoc DL(N);
829 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
830 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
831 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
832 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
833 // ReplaceNodeResults requires we maintain the same type for the return value.
834 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
837 void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
838 SmallVectorImpl<SDValue> &Results,
839 SelectionDAG &DAG) const {
840 SDLoc DL(N);
841 switch (N->getOpcode()) {
842 default:
843 llvm_unreachable("Don't know how to custom type legalize this operation!");
844 case ISD::READCYCLECOUNTER: {
845 assert(!Subtarget.is64Bit() &&
846 "READCYCLECOUNTER only has custom type legalization on riscv32");
848 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
849 SDValue RCW =
850 DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
852 Results.push_back(RCW);
853 Results.push_back(RCW.getValue(1));
854 Results.push_back(RCW.getValue(2));
855 break;
857 case ISD::SHL:
858 case ISD::SRA:
859 case ISD::SRL:
860 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
861 "Unexpected custom legalisation");
862 if (N->getOperand(1).getOpcode() == ISD::Constant)
863 return;
864 Results.push_back(customLegalizeToWOp(N, DAG));
865 break;
866 case ISD::SDIV:
867 case ISD::UDIV:
868 case ISD::UREM:
869 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
870 Subtarget.hasStdExtM() && "Unexpected custom legalisation");
871 if (N->getOperand(0).getOpcode() == ISD::Constant ||
872 N->getOperand(1).getOpcode() == ISD::Constant)
873 return;
874 Results.push_back(customLegalizeToWOp(N, DAG));
875 break;
876 case ISD::BITCAST: {
877 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
878 Subtarget.hasStdExtF() && "Unexpected custom legalisation");
879 SDLoc DL(N);
880 SDValue Op0 = N->getOperand(0);
881 if (Op0.getValueType() != MVT::f32)
882 return;
883 SDValue FPConv =
884 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
885 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
886 break;
891 SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
892 DAGCombinerInfo &DCI) const {
893 SelectionDAG &DAG = DCI.DAG;
895 switch (N->getOpcode()) {
896 default:
897 break;
898 case RISCVISD::SplitF64: {
899 SDValue Op0 = N->getOperand(0);
900 // If the input to SplitF64 is just BuildPairF64 then the operation is
901 // redundant. Instead, use BuildPairF64's operands directly.
902 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
903 return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
905 SDLoc DL(N);
907 // It's cheaper to materialise two 32-bit integers than to load a double
908 // from the constant pool and transfer it to integer registers through the
909 // stack.
910 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
911 APInt V = C->getValueAPF().bitcastToAPInt();
912 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
913 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
914 return DCI.CombineTo(N, Lo, Hi);
917 // This is a target-specific version of a DAGCombine performed in
918 // DAGCombiner::visitBITCAST. It performs the equivalent of:
919 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
920 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
921 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
922 !Op0.getNode()->hasOneUse())
923 break;
924 SDValue NewSplitF64 =
925 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
926 Op0.getOperand(0));
927 SDValue Lo = NewSplitF64.getValue(0);
928 SDValue Hi = NewSplitF64.getValue(1);
929 APInt SignBit = APInt::getSignMask(32);
930 if (Op0.getOpcode() == ISD::FNEG) {
931 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
932 DAG.getConstant(SignBit, DL, MVT::i32));
933 return DCI.CombineTo(N, Lo, NewHi);
935 assert(Op0.getOpcode() == ISD::FABS);
936 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
937 DAG.getConstant(~SignBit, DL, MVT::i32));
938 return DCI.CombineTo(N, Lo, NewHi);
940 case RISCVISD::SLLW:
941 case RISCVISD::SRAW:
942 case RISCVISD::SRLW: {
943 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
944 SDValue LHS = N->getOperand(0);
945 SDValue RHS = N->getOperand(1);
946 APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
947 APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
948 if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) ||
949 (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)))
950 return SDValue();
951 break;
953 case RISCVISD::FMV_X_ANYEXTW_RV64: {
954 SDLoc DL(N);
955 SDValue Op0 = N->getOperand(0);
956 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
957 // conversion is unnecessary and can be replaced with an ANY_EXTEND
958 // of the FMV_W_X_RV64 operand.
959 if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
960 SDValue AExtOp =
961 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0));
962 return DCI.CombineTo(N, AExtOp);
965 // This is a target-specific version of a DAGCombine performed in
966 // DAGCombiner::visitBITCAST. It performs the equivalent of:
967 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
968 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
969 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
970 !Op0.getNode()->hasOneUse())
971 break;
972 SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64,
973 Op0.getOperand(0));
974 APInt SignBit = APInt::getSignMask(32).sext(64);
975 if (Op0.getOpcode() == ISD::FNEG) {
976 return DCI.CombineTo(N,
977 DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
978 DAG.getConstant(SignBit, DL, MVT::i64)));
980 assert(Op0.getOpcode() == ISD::FABS);
981 return DCI.CombineTo(N,
982 DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
983 DAG.getConstant(~SignBit, DL, MVT::i64)));
987 return SDValue();
990 bool RISCVTargetLowering::isDesirableToCommuteWithShift(
991 const SDNode *N, CombineLevel Level) const {
992 // The following folds are only desirable if `(OP _, c1 << c2)` can be
993 // materialised in fewer instructions than `(OP _, c1)`:
995 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
996 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
997 SDValue N0 = N->getOperand(0);
998 EVT Ty = N0.getValueType();
999 if (Ty.isScalarInteger() &&
1000 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
1001 auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
1002 auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
1003 if (C1 && C2) {
1004 APInt C1Int = C1->getAPIntValue();
1005 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
1007 // We can materialise `c1 << c2` into an add immediate, so it's "free",
1008 // and the combine should happen, to potentially allow further combines
1009 // later.
1010 if (isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
1011 return true;
1013 // We can materialise `c1` in an add immediate, so it's "free", and the
1014 // combine should be prevented.
1015 if (isLegalAddImmediate(C1Int.getSExtValue()))
1016 return false;
1018 // Neither constant will fit into an immediate, so find materialisation
1019 // costs.
1020 int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
1021 Subtarget.is64Bit());
1022 int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
1023 ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit());
1025 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
1026 // combine should be prevented.
1027 if (C1Cost < ShiftedC1Cost)
1028 return false;
1031 return true;
1034 unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
1035 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1036 unsigned Depth) const {
1037 switch (Op.getOpcode()) {
1038 default:
1039 break;
1040 case RISCVISD::SLLW:
1041 case RISCVISD::SRAW:
1042 case RISCVISD::SRLW:
1043 case RISCVISD::DIVW:
1044 case RISCVISD::DIVUW:
1045 case RISCVISD::REMUW:
1046 // TODO: As the result is sign-extended, this is conservatively correct. A
1047 // more precise answer could be calculated for SRAW depending on known
1048 // bits in the shift amount.
1049 return 33;
1052 return 1;
1055 MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
1056 MachineBasicBlock *BB) {
1057 assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
1059 // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
1060 // Should the count have wrapped while it was being read, we need to try
1061 // again.
1062 // ...
1063 // read:
1064 // rdcycleh x3 # load high word of cycle
1065 // rdcycle x2 # load low word of cycle
1066 // rdcycleh x4 # load high word of cycle
1067 // bne x3, x4, read # check if high word reads match, otherwise try again
1068 // ...
1070 MachineFunction &MF = *BB->getParent();
1071 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1072 MachineFunction::iterator It = ++BB->getIterator();
1074 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1075 MF.insert(It, LoopMBB);
1077 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1078 MF.insert(It, DoneMBB);
1080 // Transfer the remainder of BB and its successor edges to DoneMBB.
1081 DoneMBB->splice(DoneMBB->begin(), BB,
1082 std::next(MachineBasicBlock::iterator(MI)), BB->end());
1083 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
1085 BB->addSuccessor(LoopMBB);
1087 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1088 unsigned ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1089 unsigned LoReg = MI.getOperand(0).getReg();
1090 unsigned HiReg = MI.getOperand(1).getReg();
1091 DebugLoc DL = MI.getDebugLoc();
1093 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1094 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
1095 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1096 .addReg(RISCV::X0);
1097 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
1098 .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
1099 .addReg(RISCV::X0);
1100 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
1101 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1102 .addReg(RISCV::X0);
1104 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
1105 .addReg(HiReg)
1106 .addReg(ReadAgainReg)
1107 .addMBB(LoopMBB);
1109 LoopMBB->addSuccessor(LoopMBB);
1110 LoopMBB->addSuccessor(DoneMBB);
1112 MI.eraseFromParent();
1114 return DoneMBB;
1117 static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
1118 MachineBasicBlock *BB) {
1119 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
1121 MachineFunction &MF = *BB->getParent();
1122 DebugLoc DL = MI.getDebugLoc();
1123 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1124 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1125 unsigned LoReg = MI.getOperand(0).getReg();
1126 unsigned HiReg = MI.getOperand(1).getReg();
1127 unsigned SrcReg = MI.getOperand(2).getReg();
1128 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
1129 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1131 TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
1132 RI);
1133 MachineMemOperand *MMO =
1134 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1135 MachineMemOperand::MOLoad, 8, 8);
1136 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
1137 .addFrameIndex(FI)
1138 .addImm(0)
1139 .addMemOperand(MMO);
1140 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
1141 .addFrameIndex(FI)
1142 .addImm(4)
1143 .addMemOperand(MMO);
1144 MI.eraseFromParent(); // The pseudo instruction is gone now.
1145 return BB;
1148 static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
1149 MachineBasicBlock *BB) {
1150 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
1151 "Unexpected instruction");
1153 MachineFunction &MF = *BB->getParent();
1154 DebugLoc DL = MI.getDebugLoc();
1155 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1156 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1157 unsigned DstReg = MI.getOperand(0).getReg();
1158 unsigned LoReg = MI.getOperand(1).getReg();
1159 unsigned HiReg = MI.getOperand(2).getReg();
1160 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
1161 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1163 MachineMemOperand *MMO =
1164 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1165 MachineMemOperand::MOStore, 8, 8);
1166 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1167 .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
1168 .addFrameIndex(FI)
1169 .addImm(0)
1170 .addMemOperand(MMO);
1171 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1172 .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
1173 .addFrameIndex(FI)
1174 .addImm(4)
1175 .addMemOperand(MMO);
1176 TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
1177 MI.eraseFromParent(); // The pseudo instruction is gone now.
1178 return BB;
1181 static bool isSelectPseudo(MachineInstr &MI) {
1182 switch (MI.getOpcode()) {
1183 default:
1184 return false;
1185 case RISCV::Select_GPR_Using_CC_GPR:
1186 case RISCV::Select_FPR32_Using_CC_GPR:
1187 case RISCV::Select_FPR64_Using_CC_GPR:
1188 return true;
1192 static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
1193 MachineBasicBlock *BB) {
1194 // To "insert" Select_* instructions, we actually have to insert the triangle
1195 // control-flow pattern. The incoming instructions know the destination vreg
1196 // to set, the condition code register to branch on, the true/false values to
1197 // select between, and the condcode to use to select the appropriate branch.
1199 // We produce the following control flow:
1200 // HeadMBB
1201 // | \
1202 // | IfFalseMBB
1203 // | /
1204 // TailMBB
1206 // When we find a sequence of selects we attempt to optimize their emission
1207 // by sharing the control flow. Currently we only handle cases where we have
1208 // multiple selects with the exact same condition (same LHS, RHS and CC).
1209 // The selects may be interleaved with other instructions if the other
1210 // instructions meet some requirements we deem safe:
1211 // - They are debug instructions. Otherwise,
1212 // - They do not have side-effects, do not access memory and their inputs do
1213 // not depend on the results of the select pseudo-instructions.
1214 // The TrueV/FalseV operands of the selects cannot depend on the result of
1215 // previous selects in the sequence.
1216 // These conditions could be further relaxed. See the X86 target for a
1217 // related approach and more information.
1218 unsigned LHS = MI.getOperand(1).getReg();
1219 unsigned RHS = MI.getOperand(2).getReg();
1220 auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());
1222 SmallVector<MachineInstr *, 4> SelectDebugValues;
1223 SmallSet<unsigned, 4> SelectDests;
1224 SelectDests.insert(MI.getOperand(0).getReg());
1226 MachineInstr *LastSelectPseudo = &MI;
1228 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
1229 SequenceMBBI != E; ++SequenceMBBI) {
1230 if (SequenceMBBI->isDebugInstr())
1231 continue;
1232 else if (isSelectPseudo(*SequenceMBBI)) {
1233 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
1234 SequenceMBBI->getOperand(2).getReg() != RHS ||
1235 SequenceMBBI->getOperand(3).getImm() != CC ||
1236 SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
1237 SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
1238 break;
1239 LastSelectPseudo = &*SequenceMBBI;
1240 SequenceMBBI->collectDebugValues(SelectDebugValues);
1241 SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
1242 } else {
1243 if (SequenceMBBI->hasUnmodeledSideEffects() ||
1244 SequenceMBBI->mayLoadOrStore())
1245 break;
1246 if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
1247 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
1249 break;
1253 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1254 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1255 DebugLoc DL = MI.getDebugLoc();
1256 MachineFunction::iterator I = ++BB->getIterator();
1258 MachineBasicBlock *HeadMBB = BB;
1259 MachineFunction *F = BB->getParent();
1260 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
1261 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
1263 F->insert(I, IfFalseMBB);
1264 F->insert(I, TailMBB);
1266 // Transfer debug instructions associated with the selects to TailMBB.
1267 for (MachineInstr *DebugInstr : SelectDebugValues) {
1268 TailMBB->push_back(DebugInstr->removeFromParent());
1271 // Move all instructions after the sequence to TailMBB.
1272 TailMBB->splice(TailMBB->end(), HeadMBB,
1273 std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
1274 // Update machine-CFG edges by transferring all successors of the current
1275 // block to the new block which will contain the Phi nodes for the selects.
1276 TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
1277 // Set the successors for HeadMBB.
1278 HeadMBB->addSuccessor(IfFalseMBB);
1279 HeadMBB->addSuccessor(TailMBB);
1281 // Insert appropriate branch.
1282 unsigned Opcode = getBranchOpcodeForIntCondCode(CC);
1284 BuildMI(HeadMBB, DL, TII.get(Opcode))
1285 .addReg(LHS)
1286 .addReg(RHS)
1287 .addMBB(TailMBB);
1289 // IfFalseMBB just falls through to TailMBB.
1290 IfFalseMBB->addSuccessor(TailMBB);
1292 // Create PHIs for all of the select pseudo-instructions.
1293 auto SelectMBBI = MI.getIterator();
1294 auto SelectEnd = std::next(LastSelectPseudo->getIterator());
1295 auto InsertionPoint = TailMBB->begin();
1296 while (SelectMBBI != SelectEnd) {
1297 auto Next = std::next(SelectMBBI);
1298 if (isSelectPseudo(*SelectMBBI)) {
1299 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
1300 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
1301 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
1302 .addReg(SelectMBBI->getOperand(4).getReg())
1303 .addMBB(HeadMBB)
1304 .addReg(SelectMBBI->getOperand(5).getReg())
1305 .addMBB(IfFalseMBB);
1306 SelectMBBI->eraseFromParent();
1308 SelectMBBI = Next;
1311 F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
1312 return TailMBB;
1315 MachineBasicBlock *
1316 RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
1317 MachineBasicBlock *BB) const {
1318 switch (MI.getOpcode()) {
1319 default:
1320 llvm_unreachable("Unexpected instr type to insert");
1321 case RISCV::ReadCycleWide:
1322 assert(!Subtarget.is64Bit() &&
1323 "ReadCycleWrite is only to be used on riscv32");
1324 return emitReadCycleWidePseudo(MI, BB);
1325 case RISCV::Select_GPR_Using_CC_GPR:
1326 case RISCV::Select_FPR32_Using_CC_GPR:
1327 case RISCV::Select_FPR64_Using_CC_GPR:
1328 return emitSelectPseudo(MI, BB);
1329 case RISCV::BuildPairF64Pseudo:
1330 return emitBuildPairF64Pseudo(MI, BB);
1331 case RISCV::SplitF64Pseudo:
1332 return emitSplitF64Pseudo(MI, BB);
1336 // Calling Convention Implementation.
1337 // The expectations for frontend ABI lowering vary from target to target.
1338 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
1339 // details, but this is a longer term goal. For now, we simply try to keep the
1340 // role of the frontend as simple and well-defined as possible. The rules can
1341 // be summarised as:
1342 // * Never split up large scalar arguments. We handle them here.
1343 // * If a hardfloat calling convention is being used, and the struct may be
1344 // passed in a pair of registers (fp+fp, int+fp), and both registers are
1345 // available, then pass as two separate arguments. If either the GPRs or FPRs
1346 // are exhausted, then pass according to the rule below.
1347 // * If a struct could never be passed in registers or directly in a stack
1348 // slot (as it is larger than 2*XLEN and the floating point rules don't
1349 // apply), then pass it using a pointer with the byval attribute.
1350 // * If a struct is less than 2*XLEN, then coerce to either a two-element
1351 // word-sized array or a 2*XLEN scalar (depending on alignment).
1352 // * The frontend can determine whether a struct is returned by reference or
1353 // not based on its size and fields. If it will be returned by reference, the
1354 // frontend must modify the prototype so a pointer with the sret annotation is
1355 // passed as the first argument. This is not necessary for large scalar
1356 // returns.
1357 // * Struct return values and varargs should be coerced to structs containing
1358 // register-size fields in the same situations they would be for fixed
1359 // arguments.
1361 static const MCPhysReg ArgGPRs[] = {
1362 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
1363 RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
1365 static const MCPhysReg ArgFPR32s[] = {
1366 RISCV::F10_32, RISCV::F11_32, RISCV::F12_32, RISCV::F13_32,
1367 RISCV::F14_32, RISCV::F15_32, RISCV::F16_32, RISCV::F17_32
1369 static const MCPhysReg ArgFPR64s[] = {
1370 RISCV::F10_64, RISCV::F11_64, RISCV::F12_64, RISCV::F13_64,
1371 RISCV::F14_64, RISCV::F15_64, RISCV::F16_64, RISCV::F17_64
1374 // Pass a 2*XLEN argument that has been split into two XLEN values through
1375 // registers or the stack as necessary.
1376 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
1377 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
1378 MVT ValVT2, MVT LocVT2,
1379 ISD::ArgFlagsTy ArgFlags2) {
1380 unsigned XLenInBytes = XLen / 8;
1381 if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
1382 // At least one half can be passed via register.
1383 State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
1384 VA1.getLocVT(), CCValAssign::Full));
1385 } else {
1386 // Both halves must be passed on the stack, with proper alignment.
1387 unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
1388 State.addLoc(
1389 CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
1390 State.AllocateStack(XLenInBytes, StackAlign),
1391 VA1.getLocVT(), CCValAssign::Full));
1392 State.addLoc(CCValAssign::getMem(
1393 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1394 CCValAssign::Full));
1395 return false;
1398 if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
1399 // The second half can also be passed via register.
1400 State.addLoc(
1401 CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
1402 } else {
1403 // The second half is passed via the stack, without additional alignment.
1404 State.addLoc(CCValAssign::getMem(
1405 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1406 CCValAssign::Full));
1409 return false;
1412 // Implements the RISC-V calling convention. Returns true upon failure.
1413 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
1414 MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
1415 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
1416 bool IsRet, Type *OrigTy) {
1417 unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
1418 assert(XLen == 32 || XLen == 64);
1419 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
1421 // Any return value split in to more than two values can't be returned
1422 // directly.
1423 if (IsRet && ValNo > 1)
1424 return true;
1426 // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a
1427 // variadic argument, or if no F32 argument registers are available.
1428 bool UseGPRForF32 = true;
1429 // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
1430 // variadic argument, or if no F64 argument registers are available.
1431 bool UseGPRForF64 = true;
1433 switch (ABI) {
1434 default:
1435 llvm_unreachable("Unexpected ABI");
1436 case RISCVABI::ABI_ILP32:
1437 case RISCVABI::ABI_LP64:
1438 break;
1439 case RISCVABI::ABI_ILP32F:
1440 case RISCVABI::ABI_LP64F:
1441 UseGPRForF32 = !IsFixed;
1442 break;
1443 case RISCVABI::ABI_ILP32D:
1444 case RISCVABI::ABI_LP64D:
1445 UseGPRForF32 = !IsFixed;
1446 UseGPRForF64 = !IsFixed;
1447 break;
1450 if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s))
1451 UseGPRForF32 = true;
1452 if (State.getFirstUnallocated(ArgFPR64s) == array_lengthof(ArgFPR64s))
1453 UseGPRForF64 = true;
1455 // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local
1456 // variables rather than directly checking against the target ABI.
1458 if (UseGPRForF32 && ValVT == MVT::f32) {
1459 LocVT = XLenVT;
1460 LocInfo = CCValAssign::BCvt;
1461 } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
1462 LocVT = MVT::i64;
1463 LocInfo = CCValAssign::BCvt;
1466 // If this is a variadic argument, the RISC-V calling convention requires
1467 // that it is assigned an 'even' or 'aligned' register if it has 8-byte
1468 // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
1469 // be used regardless of whether the original argument was split during
1470 // legalisation or not. The argument will not be passed by registers if the
1471 // original type is larger than 2*XLEN, so the register alignment rule does
1472 // not apply.
1473 unsigned TwoXLenInBytes = (2 * XLen) / 8;
1474 if (!IsFixed && ArgFlags.getOrigAlign() == TwoXLenInBytes &&
1475 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
1476 unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
1477 // Skip 'odd' register if necessary.
1478 if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
1479 State.AllocateReg(ArgGPRs);
1482 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
1483 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
1484 State.getPendingArgFlags();
1486 assert(PendingLocs.size() == PendingArgFlags.size() &&
1487 "PendingLocs and PendingArgFlags out of sync");
1489 // Handle passing f64 on RV32D with a soft float ABI or when floating point
1490 // registers are exhausted.
1491 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
1492 assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
1493 "Can't lower f64 if it is split");
1494 // Depending on available argument GPRS, f64 may be passed in a pair of
1495 // GPRs, split between a GPR and the stack, or passed completely on the
1496 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
1497 // cases.
1498 unsigned Reg = State.AllocateReg(ArgGPRs);
1499 LocVT = MVT::i32;
1500 if (!Reg) {
1501 unsigned StackOffset = State.AllocateStack(8, 8);
1502 State.addLoc(
1503 CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1504 return false;
1506 if (!State.AllocateReg(ArgGPRs))
1507 State.AllocateStack(4, 4);
1508 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1509 return false;
1512 // Split arguments might be passed indirectly, so keep track of the pending
1513 // values.
1514 if (ArgFlags.isSplit() || !PendingLocs.empty()) {
1515 LocVT = XLenVT;
1516 LocInfo = CCValAssign::Indirect;
1517 PendingLocs.push_back(
1518 CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
1519 PendingArgFlags.push_back(ArgFlags);
1520 if (!ArgFlags.isSplitEnd()) {
1521 return false;
1525 // If the split argument only had two elements, it should be passed directly
1526 // in registers or on the stack.
1527 if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
1528 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
1529 // Apply the normal calling convention rules to the first half of the
1530 // split argument.
1531 CCValAssign VA = PendingLocs[0];
1532 ISD::ArgFlagsTy AF = PendingArgFlags[0];
1533 PendingLocs.clear();
1534 PendingArgFlags.clear();
1535 return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
1536 ArgFlags);
1539 // Allocate to a register if possible, or else a stack slot.
1540 unsigned Reg;
1541 if (ValVT == MVT::f32 && !UseGPRForF32)
1542 Reg = State.AllocateReg(ArgFPR32s, ArgFPR64s);
1543 else if (ValVT == MVT::f64 && !UseGPRForF64)
1544 Reg = State.AllocateReg(ArgFPR64s, ArgFPR32s);
1545 else
1546 Reg = State.AllocateReg(ArgGPRs);
1547 unsigned StackOffset = Reg ? 0 : State.AllocateStack(XLen / 8, XLen / 8);
1549 // If we reach this point and PendingLocs is non-empty, we must be at the
1550 // end of a split argument that must be passed indirectly.
1551 if (!PendingLocs.empty()) {
1552 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
1553 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
1555 for (auto &It : PendingLocs) {
1556 if (Reg)
1557 It.convertToReg(Reg);
1558 else
1559 It.convertToMem(StackOffset);
1560 State.addLoc(It);
1562 PendingLocs.clear();
1563 PendingArgFlags.clear();
1564 return false;
1567 assert((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&
1568 "Expected an XLenVT at this stage");
1570 if (Reg) {
1571 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1572 return false;
1575 // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
1576 if (ValVT == MVT::f32 || ValVT == MVT::f64) {
1577 LocVT = ValVT;
1578 LocInfo = CCValAssign::Full;
1580 State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1581 return false;
1584 void RISCVTargetLowering::analyzeInputArgs(
1585 MachineFunction &MF, CCState &CCInfo,
1586 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
1587 unsigned NumArgs = Ins.size();
1588 FunctionType *FType = MF.getFunction().getFunctionType();
1590 for (unsigned i = 0; i != NumArgs; ++i) {
1591 MVT ArgVT = Ins[i].VT;
1592 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
1594 Type *ArgTy = nullptr;
1595 if (IsRet)
1596 ArgTy = FType->getReturnType();
1597 else if (Ins[i].isOrigArg())
1598 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
1600 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1601 if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1602 ArgFlags, CCInfo, /*IsRet=*/true, IsRet, ArgTy)) {
1603 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
1604 << EVT(ArgVT).getEVTString() << '\n');
1605 llvm_unreachable(nullptr);
1610 void RISCVTargetLowering::analyzeOutputArgs(
1611 MachineFunction &MF, CCState &CCInfo,
1612 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
1613 CallLoweringInfo *CLI) const {
1614 unsigned NumArgs = Outs.size();
1616 for (unsigned i = 0; i != NumArgs; i++) {
1617 MVT ArgVT = Outs[i].VT;
1618 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
1619 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
1621 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1622 if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1623 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) {
1624 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
1625 << EVT(ArgVT).getEVTString() << "\n");
1626 llvm_unreachable(nullptr);
1631 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
1632 // values.
1633 static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
1634 const CCValAssign &VA, const SDLoc &DL) {
1635 switch (VA.getLocInfo()) {
1636 default:
1637 llvm_unreachable("Unexpected CCValAssign::LocInfo");
1638 case CCValAssign::Full:
1639 break;
1640 case CCValAssign::BCvt:
1641 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1642 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
1643 break;
1645 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
1646 break;
1648 return Val;
1651 // The caller is responsible for loading the full value if the argument is
1652 // passed with CCValAssign::Indirect.
1653 static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
1654 const CCValAssign &VA, const SDLoc &DL) {
1655 MachineFunction &MF = DAG.getMachineFunction();
1656 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1657 EVT LocVT = VA.getLocVT();
1658 SDValue Val;
1659 const TargetRegisterClass *RC;
1661 switch (LocVT.getSimpleVT().SimpleTy) {
1662 default:
1663 llvm_unreachable("Unexpected register type");
1664 case MVT::i32:
1665 case MVT::i64:
1666 RC = &RISCV::GPRRegClass;
1667 break;
1668 case MVT::f32:
1669 RC = &RISCV::FPR32RegClass;
1670 break;
1671 case MVT::f64:
1672 RC = &RISCV::FPR64RegClass;
1673 break;
1676 unsigned VReg = RegInfo.createVirtualRegister(RC);
1677 RegInfo.addLiveIn(VA.getLocReg(), VReg);
1678 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1680 if (VA.getLocInfo() == CCValAssign::Indirect)
1681 return Val;
1683 return convertLocVTToValVT(DAG, Val, VA, DL);
1686 static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
1687 const CCValAssign &VA, const SDLoc &DL) {
1688 EVT LocVT = VA.getLocVT();
1690 switch (VA.getLocInfo()) {
1691 default:
1692 llvm_unreachable("Unexpected CCValAssign::LocInfo");
1693 case CCValAssign::Full:
1694 break;
1695 case CCValAssign::BCvt:
1696 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1697 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
1698 break;
1700 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
1701 break;
1703 return Val;
1706 // The caller is responsible for loading the full value if the argument is
1707 // passed with CCValAssign::Indirect.
1708 static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
1709 const CCValAssign &VA, const SDLoc &DL) {
1710 MachineFunction &MF = DAG.getMachineFunction();
1711 MachineFrameInfo &MFI = MF.getFrameInfo();
1712 EVT LocVT = VA.getLocVT();
1713 EVT ValVT = VA.getValVT();
1714 EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
1715 int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
1716 VA.getLocMemOffset(), /*Immutable=*/true);
1717 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1718 SDValue Val;
1720 ISD::LoadExtType ExtType;
1721 switch (VA.getLocInfo()) {
1722 default:
1723 llvm_unreachable("Unexpected CCValAssign::LocInfo");
1724 case CCValAssign::Full:
1725 case CCValAssign::Indirect:
1726 case CCValAssign::BCvt:
1727 ExtType = ISD::NON_EXTLOAD;
1728 break;
1730 Val = DAG.getExtLoad(
1731 ExtType, DL, LocVT, Chain, FIN,
1732 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
1733 return Val;
1736 static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
1737 const CCValAssign &VA, const SDLoc &DL) {
1738 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
1739 "Unexpected VA");
1740 MachineFunction &MF = DAG.getMachineFunction();
1741 MachineFrameInfo &MFI = MF.getFrameInfo();
1742 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1744 if (VA.isMemLoc()) {
1745 // f64 is passed on the stack.
1746 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
1747 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1748 return DAG.getLoad(MVT::f64, DL, Chain, FIN,
1749 MachinePointerInfo::getFixedStack(MF, FI));
1752 assert(VA.isRegLoc() && "Expected register VA assignment");
1754 unsigned LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1755 RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
1756 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
1757 SDValue Hi;
1758 if (VA.getLocReg() == RISCV::X17) {
1759 // Second half of f64 is passed on the stack.
1760 int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
1761 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1762 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
1763 MachinePointerInfo::getFixedStack(MF, FI));
1764 } else {
1765 // Second half of f64 is passed in another GPR.
1766 unsigned HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1767 RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
1768 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
1770 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1773 // Transform physical registers into virtual registers.
1774 SDValue RISCVTargetLowering::LowerFormalArguments(
1775 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1776 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1777 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1779 switch (CallConv) {
1780 default:
1781 report_fatal_error("Unsupported calling convention");
1782 case CallingConv::C:
1783 case CallingConv::Fast:
1784 break;
1787 MachineFunction &MF = DAG.getMachineFunction();
1789 const Function &Func = MF.getFunction();
1790 if (Func.hasFnAttribute("interrupt")) {
1791 if (!Func.arg_empty())
1792 report_fatal_error(
1793 "Functions with the interrupt attribute cannot have arguments!");
1795 StringRef Kind =
1796 MF.getFunction().getFnAttribute("interrupt").getValueAsString();
1798 if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
1799 report_fatal_error(
1800 "Function interrupt attribute argument not supported!");
1803 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1804 MVT XLenVT = Subtarget.getXLenVT();
1805 unsigned XLenInBytes = Subtarget.getXLen() / 8;
1806 // Used with vargs to acumulate store chains.
1807 std::vector<SDValue> OutChains;
1809 // Assign locations to all of the incoming arguments.
1810 SmallVector<CCValAssign, 16> ArgLocs;
1811 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1812 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
1814 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1815 CCValAssign &VA = ArgLocs[i];
1816 SDValue ArgValue;
1817 // Passing f64 on RV32D with a soft float ABI must be handled as a special
1818 // case.
1819 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
1820 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
1821 else if (VA.isRegLoc())
1822 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
1823 else
1824 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
1826 if (VA.getLocInfo() == CCValAssign::Indirect) {
1827 // If the original argument was split and passed by reference (e.g. i128
1828 // on RV32), we need to load all parts of it here (using the same
1829 // address).
1830 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1831 MachinePointerInfo()));
1832 unsigned ArgIndex = Ins[i].OrigArgIndex;
1833 assert(Ins[i].PartOffset == 0);
1834 while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
1835 CCValAssign &PartVA = ArgLocs[i + 1];
1836 unsigned PartOffset = Ins[i + 1].PartOffset;
1837 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1838 DAG.getIntPtrConstant(PartOffset, DL));
1839 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1840 MachinePointerInfo()));
1841 ++i;
1843 continue;
1845 InVals.push_back(ArgValue);
1848 if (IsVarArg) {
1849 ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
1850 unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
1851 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1852 MachineFrameInfo &MFI = MF.getFrameInfo();
1853 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1854 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1856 // Offset of the first variable argument from stack pointer, and size of
1857 // the vararg save area. For now, the varargs save area is either zero or
1858 // large enough to hold a0-a7.
1859 int VaArgOffset, VarArgsSaveSize;
1861 // If all registers are allocated, then all varargs must be passed on the
1862 // stack and we don't need to save any argregs.
1863 if (ArgRegs.size() == Idx) {
1864 VaArgOffset = CCInfo.getNextStackOffset();
1865 VarArgsSaveSize = 0;
1866 } else {
1867 VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
1868 VaArgOffset = -VarArgsSaveSize;
1871 // Record the frame index of the first variable argument
1872 // which is a value necessary to VASTART.
1873 int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1874 RVFI->setVarArgsFrameIndex(FI);
1876 // If saving an odd number of registers then create an extra stack slot to
1877 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
1878 // offsets to even-numbered registered remain 2*XLEN-aligned.
1879 if (Idx % 2) {
1880 FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes,
1881 true);
1882 VarArgsSaveSize += XLenInBytes;
1885 // Copy the integer registers that may have been used for passing varargs
1886 // to the vararg save area.
1887 for (unsigned I = Idx; I < ArgRegs.size();
1888 ++I, VaArgOffset += XLenInBytes) {
1889 const unsigned Reg = RegInfo.createVirtualRegister(RC);
1890 RegInfo.addLiveIn(ArgRegs[I], Reg);
1891 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
1892 FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1893 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1894 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
1895 MachinePointerInfo::getFixedStack(MF, FI));
1896 cast<StoreSDNode>(Store.getNode())
1897 ->getMemOperand()
1898 ->setValue((Value *)nullptr);
1899 OutChains.push_back(Store);
1901 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
1904 // All stores are grouped in one node to allow the matching between
1905 // the size of Ins and InVals. This only happens for vararg functions.
1906 if (!OutChains.empty()) {
1907 OutChains.push_back(Chain);
1908 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
1911 return Chain;
1914 /// isEligibleForTailCallOptimization - Check whether the call is eligible
1915 /// for tail call optimization.
1916 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
1917 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
1918 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
1919 const SmallVector<CCValAssign, 16> &ArgLocs) const {
1921 auto &Callee = CLI.Callee;
1922 auto CalleeCC = CLI.CallConv;
1923 auto IsVarArg = CLI.IsVarArg;
1924 auto &Outs = CLI.Outs;
1925 auto &Caller = MF.getFunction();
1926 auto CallerCC = Caller.getCallingConv();
1928 // Do not tail call opt functions with "disable-tail-calls" attribute.
1929 if (Caller.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
1930 return false;
1932 // Exception-handling functions need a special set of instructions to
1933 // indicate a return to the hardware. Tail-calling another function would
1934 // probably break this.
1935 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
1936 // should be expanded as new function attributes are introduced.
1937 if (Caller.hasFnAttribute("interrupt"))
1938 return false;
1940 // Do not tail call opt functions with varargs.
1941 if (IsVarArg)
1942 return false;
1944 // Do not tail call opt if the stack is used to pass parameters.
1945 if (CCInfo.getNextStackOffset() != 0)
1946 return false;
1948 // Do not tail call opt if any parameters need to be passed indirectly.
1949 // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
1950 // passed indirectly. So the address of the value will be passed in a
1951 // register, or if not available, then the address is put on the stack. In
1952 // order to pass indirectly, space on the stack often needs to be allocated
1953 // in order to store the value. In this case the CCInfo.getNextStackOffset()
1954 // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
1955 // are passed CCValAssign::Indirect.
1956 for (auto &VA : ArgLocs)
1957 if (VA.getLocInfo() == CCValAssign::Indirect)
1958 return false;
1960 // Do not tail call opt if either caller or callee uses struct return
1961 // semantics.
1962 auto IsCallerStructRet = Caller.hasStructRetAttr();
1963 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
1964 if (IsCallerStructRet || IsCalleeStructRet)
1965 return false;
1967 // Externally-defined functions with weak linkage should not be
1968 // tail-called. The behaviour of branch instructions in this situation (as
1969 // used for tail calls) is implementation-defined, so we cannot rely on the
1970 // linker replacing the tail call with a return.
1971 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1972 const GlobalValue *GV = G->getGlobal();
1973 if (GV->hasExternalWeakLinkage())
1974 return false;
1977 // The callee has to preserve all registers the caller needs to preserve.
1978 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
1979 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1980 if (CalleeCC != CallerCC) {
1981 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
1982 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
1983 return false;
1986 // Byval parameters hand the function a pointer directly into the stack area
1987 // we want to reuse during a tail call. Working around this *is* possible
1988 // but less efficient and uglier in LowerCall.
1989 for (auto &Arg : Outs)
1990 if (Arg.Flags.isByVal())
1991 return false;
1993 return true;
1996 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
1997 // and output parameter nodes.
1998 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
1999 SmallVectorImpl<SDValue> &InVals) const {
2000 SelectionDAG &DAG = CLI.DAG;
2001 SDLoc &DL = CLI.DL;
2002 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2003 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2004 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2005 SDValue Chain = CLI.Chain;
2006 SDValue Callee = CLI.Callee;
2007 bool &IsTailCall = CLI.IsTailCall;
2008 CallingConv::ID CallConv = CLI.CallConv;
2009 bool IsVarArg = CLI.IsVarArg;
2010 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2011 MVT XLenVT = Subtarget.getXLenVT();
2013 MachineFunction &MF = DAG.getMachineFunction();
2015 // Analyze the operands of the call, assigning locations to each operand.
2016 SmallVector<CCValAssign, 16> ArgLocs;
2017 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2018 analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI);
2020 // Check if it's really possible to do a tail call.
2021 if (IsTailCall)
2022 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
2024 if (IsTailCall)
2025 ++NumTailCalls;
2026 else if (CLI.CS && CLI.CS.isMustTailCall())
2027 report_fatal_error("failed to perform tail call elimination on a call "
2028 "site marked musttail");
2030 // Get a count of how many bytes are to be pushed on the stack.
2031 unsigned NumBytes = ArgCCInfo.getNextStackOffset();
2033 // Create local copies for byval args
2034 SmallVector<SDValue, 8> ByValArgs;
2035 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2036 ISD::ArgFlagsTy Flags = Outs[i].Flags;
2037 if (!Flags.isByVal())
2038 continue;
2040 SDValue Arg = OutVals[i];
2041 unsigned Size = Flags.getByValSize();
2042 unsigned Align = Flags.getByValAlign();
2044 int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
2045 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2046 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
2048 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
2049 /*IsVolatile=*/false,
2050 /*AlwaysInline=*/false,
2051 IsTailCall, MachinePointerInfo(),
2052 MachinePointerInfo());
2053 ByValArgs.push_back(FIPtr);
2056 if (!IsTailCall)
2057 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
2059 // Copy argument values to their designated locations.
2060 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2061 SmallVector<SDValue, 8> MemOpChains;
2062 SDValue StackPtr;
2063 for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
2064 CCValAssign &VA = ArgLocs[i];
2065 SDValue ArgValue = OutVals[i];
2066 ISD::ArgFlagsTy Flags = Outs[i].Flags;
2068 // Handle passing f64 on RV32D with a soft float ABI as a special case.
2069 bool IsF64OnRV32DSoftABI =
2070 VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
2071 if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
2072 SDValue SplitF64 = DAG.getNode(
2073 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
2074 SDValue Lo = SplitF64.getValue(0);
2075 SDValue Hi = SplitF64.getValue(1);
2077 unsigned RegLo = VA.getLocReg();
2078 RegsToPass.push_back(std::make_pair(RegLo, Lo));
2080 if (RegLo == RISCV::X17) {
2081 // Second half of f64 is passed on the stack.
2082 // Work out the address of the stack slot.
2083 if (!StackPtr.getNode())
2084 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2085 // Emit the store.
2086 MemOpChains.push_back(
2087 DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
2088 } else {
2089 // Second half of f64 is passed in another GPR.
2090 unsigned RegHigh = RegLo + 1;
2091 RegsToPass.push_back(std::make_pair(RegHigh, Hi));
2093 continue;
2096 // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
2097 // as any other MemLoc.
2099 // Promote the value if needed.
2100 // For now, only handle fully promoted and indirect arguments.
2101 if (VA.getLocInfo() == CCValAssign::Indirect) {
2102 // Store the argument in a stack slot and pass its address.
2103 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
2104 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2105 MemOpChains.push_back(
2106 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
2107 MachinePointerInfo::getFixedStack(MF, FI)));
2108 // If the original argument was split (e.g. i128), we need
2109 // to store all parts of it here (and pass just one address).
2110 unsigned ArgIndex = Outs[i].OrigArgIndex;
2111 assert(Outs[i].PartOffset == 0);
2112 while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
2113 SDValue PartValue = OutVals[i + 1];
2114 unsigned PartOffset = Outs[i + 1].PartOffset;
2115 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2116 DAG.getIntPtrConstant(PartOffset, DL));
2117 MemOpChains.push_back(
2118 DAG.getStore(Chain, DL, PartValue, Address,
2119 MachinePointerInfo::getFixedStack(MF, FI)));
2120 ++i;
2122 ArgValue = SpillSlot;
2123 } else {
2124 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
2127 // Use local copy if it is a byval arg.
2128 if (Flags.isByVal())
2129 ArgValue = ByValArgs[j++];
2131 if (VA.isRegLoc()) {
2132 // Queue up the argument copies and emit them at the end.
2133 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
2134 } else {
2135 assert(VA.isMemLoc() && "Argument not register or memory");
2136 assert(!IsTailCall && "Tail call not allowed if stack is used "
2137 "for passing parameters");
2139 // Work out the address of the stack slot.
2140 if (!StackPtr.getNode())
2141 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2142 SDValue Address =
2143 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
2144 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
2146 // Emit the store.
2147 MemOpChains.push_back(
2148 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
2152 // Join the stores, which are independent of one another.
2153 if (!MemOpChains.empty())
2154 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2156 SDValue Glue;
2158 // Build a sequence of copy-to-reg nodes, chained and glued together.
2159 for (auto &Reg : RegsToPass) {
2160 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
2161 Glue = Chain.getValue(1);
2164 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
2165 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
2166 // split it and then direct call can be matched by PseudoCALL.
2167 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
2168 const GlobalValue *GV = S->getGlobal();
2170 unsigned OpFlags = RISCVII::MO_CALL;
2171 if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
2172 OpFlags = RISCVII::MO_PLT;
2174 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
2175 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2176 unsigned OpFlags = RISCVII::MO_CALL;
2178 if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
2179 nullptr))
2180 OpFlags = RISCVII::MO_PLT;
2182 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
2185 // The first call operand is the chain and the second is the target address.
2186 SmallVector<SDValue, 8> Ops;
2187 Ops.push_back(Chain);
2188 Ops.push_back(Callee);
2190 // Add argument registers to the end of the list so that they are
2191 // known live into the call.
2192 for (auto &Reg : RegsToPass)
2193 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
2195 if (!IsTailCall) {
2196 // Add a register mask operand representing the call-preserved registers.
2197 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2198 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2199 assert(Mask && "Missing call preserved mask for calling convention");
2200 Ops.push_back(DAG.getRegisterMask(Mask));
2203 // Glue the call to the argument copies, if any.
2204 if (Glue.getNode())
2205 Ops.push_back(Glue);
2207 // Emit the call.
2208 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2210 if (IsTailCall) {
2211 MF.getFrameInfo().setHasTailCall();
2212 return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
2215 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
2216 Glue = Chain.getValue(1);
2218 // Mark the end of the call, which is glued to the call itself.
2219 Chain = DAG.getCALLSEQ_END(Chain,
2220 DAG.getConstant(NumBytes, DL, PtrVT, true),
2221 DAG.getConstant(0, DL, PtrVT, true),
2222 Glue, DL);
2223 Glue = Chain.getValue(1);
2225 // Assign locations to each value returned by this call.
2226 SmallVector<CCValAssign, 16> RVLocs;
2227 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
2228 analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
2230 // Copy all of the result registers out of their specified physreg.
2231 for (auto &VA : RVLocs) {
2232 // Copy the value out
2233 SDValue RetValue =
2234 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
2235 // Glue the RetValue to the end of the call sequence
2236 Chain = RetValue.getValue(1);
2237 Glue = RetValue.getValue(2);
2239 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2240 assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
2241 SDValue RetValue2 =
2242 DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
2243 Chain = RetValue2.getValue(1);
2244 Glue = RetValue2.getValue(2);
2245 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
2246 RetValue2);
2249 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
2251 InVals.push_back(RetValue);
2254 return Chain;
2257 bool RISCVTargetLowering::CanLowerReturn(
2258 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2259 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2260 SmallVector<CCValAssign, 16> RVLocs;
2261 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2262 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2263 MVT VT = Outs[i].VT;
2264 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2265 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
2266 if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
2267 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr))
2268 return false;
2270 return true;
2273 SDValue
2274 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2275 bool IsVarArg,
2276 const SmallVectorImpl<ISD::OutputArg> &Outs,
2277 const SmallVectorImpl<SDValue> &OutVals,
2278 const SDLoc &DL, SelectionDAG &DAG) const {
2279 // Stores the assignment of the return value to a location.
2280 SmallVector<CCValAssign, 16> RVLocs;
2282 // Info about the registers and stack slot.
2283 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2284 *DAG.getContext());
2286 analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
2287 nullptr);
2289 SDValue Glue;
2290 SmallVector<SDValue, 4> RetOps(1, Chain);
2292 // Copy the result values into the output registers.
2293 for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
2294 SDValue Val = OutVals[i];
2295 CCValAssign &VA = RVLocs[i];
2296 assert(VA.isRegLoc() && "Can only return in registers!");
2298 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2299 // Handle returning f64 on RV32D with a soft float ABI.
2300 assert(VA.isRegLoc() && "Expected return via registers");
2301 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
2302 DAG.getVTList(MVT::i32, MVT::i32), Val);
2303 SDValue Lo = SplitF64.getValue(0);
2304 SDValue Hi = SplitF64.getValue(1);
2305 unsigned RegLo = VA.getLocReg();
2306 unsigned RegHi = RegLo + 1;
2307 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
2308 Glue = Chain.getValue(1);
2309 RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
2310 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
2311 Glue = Chain.getValue(1);
2312 RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
2313 } else {
2314 // Handle a 'normal' return.
2315 Val = convertValVTToLocVT(DAG, Val, VA, DL);
2316 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
2318 // Guarantee that all emitted copies are stuck together.
2319 Glue = Chain.getValue(1);
2320 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2324 RetOps[0] = Chain; // Update chain.
2326 // Add the glue node if we have it.
2327 if (Glue.getNode()) {
2328 RetOps.push_back(Glue);
2331 // Interrupt service routines use different return instructions.
2332 const Function &Func = DAG.getMachineFunction().getFunction();
2333 if (Func.hasFnAttribute("interrupt")) {
2334 if (!Func.getReturnType()->isVoidTy())
2335 report_fatal_error(
2336 "Functions with the interrupt attribute must have void return type!");
2338 MachineFunction &MF = DAG.getMachineFunction();
2339 StringRef Kind =
2340 MF.getFunction().getFnAttribute("interrupt").getValueAsString();
2342 unsigned RetOpc;
2343 if (Kind == "user")
2344 RetOpc = RISCVISD::URET_FLAG;
2345 else if (Kind == "supervisor")
2346 RetOpc = RISCVISD::SRET_FLAG;
2347 else
2348 RetOpc = RISCVISD::MRET_FLAG;
2350 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
2353 return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
2356 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
2357 switch ((RISCVISD::NodeType)Opcode) {
2358 case RISCVISD::FIRST_NUMBER:
2359 break;
2360 case RISCVISD::RET_FLAG:
2361 return "RISCVISD::RET_FLAG";
2362 case RISCVISD::URET_FLAG:
2363 return "RISCVISD::URET_FLAG";
2364 case RISCVISD::SRET_FLAG:
2365 return "RISCVISD::SRET_FLAG";
2366 case RISCVISD::MRET_FLAG:
2367 return "RISCVISD::MRET_FLAG";
2368 case RISCVISD::CALL:
2369 return "RISCVISD::CALL";
2370 case RISCVISD::SELECT_CC:
2371 return "RISCVISD::SELECT_CC";
2372 case RISCVISD::BuildPairF64:
2373 return "RISCVISD::BuildPairF64";
2374 case RISCVISD::SplitF64:
2375 return "RISCVISD::SplitF64";
2376 case RISCVISD::TAIL:
2377 return "RISCVISD::TAIL";
2378 case RISCVISD::SLLW:
2379 return "RISCVISD::SLLW";
2380 case RISCVISD::SRAW:
2381 return "RISCVISD::SRAW";
2382 case RISCVISD::SRLW:
2383 return "RISCVISD::SRLW";
2384 case RISCVISD::DIVW:
2385 return "RISCVISD::DIVW";
2386 case RISCVISD::DIVUW:
2387 return "RISCVISD::DIVUW";
2388 case RISCVISD::REMUW:
2389 return "RISCVISD::REMUW";
2390 case RISCVISD::FMV_W_X_RV64:
2391 return "RISCVISD::FMV_W_X_RV64";
2392 case RISCVISD::FMV_X_ANYEXTW_RV64:
2393 return "RISCVISD::FMV_X_ANYEXTW_RV64";
2394 case RISCVISD::READ_CYCLE_WIDE:
2395 return "RISCVISD::READ_CYCLE_WIDE";
2397 return nullptr;
2400 /// getConstraintType - Given a constraint letter, return the type of
2401 /// constraint it is for this target.
2402 RISCVTargetLowering::ConstraintType
2403 RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
2404 if (Constraint.size() == 1) {
2405 switch (Constraint[0]) {
2406 default:
2407 break;
2408 case 'f':
2409 return C_RegisterClass;
2410 case 'I':
2411 case 'J':
2412 case 'K':
2413 return C_Immediate;
2416 return TargetLowering::getConstraintType(Constraint);
2419 std::pair<unsigned, const TargetRegisterClass *>
2420 RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
2421 StringRef Constraint,
2422 MVT VT) const {
2423 // First, see if this is a constraint that directly corresponds to a
2424 // RISCV register class.
2425 if (Constraint.size() == 1) {
2426 switch (Constraint[0]) {
2427 case 'r':
2428 return std::make_pair(0U, &RISCV::GPRRegClass);
2429 case 'f':
2430 if (Subtarget.hasStdExtF() && VT == MVT::f32)
2431 return std::make_pair(0U, &RISCV::FPR32RegClass);
2432 if (Subtarget.hasStdExtD() && VT == MVT::f64)
2433 return std::make_pair(0U, &RISCV::FPR64RegClass);
2434 break;
2435 default:
2436 break;
2440 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
2441 // TableGen record rather than the AsmName to choose registers for InlineAsm
2442 // constraints, plus we want to match those names to the widest floating point
2443 // register type available, manually select floating point registers here.
2444 if (Subtarget.hasStdExtF() || Subtarget.hasStdExtD()) {
2445 std::pair<unsigned, unsigned> FReg =
2446 StringSwitch<std::pair<unsigned, unsigned>>(Constraint.lower())
2447 .Case("{f0}", {RISCV::F0_32, RISCV::F0_64})
2448 .Case("{f1}", {RISCV::F1_32, RISCV::F1_64})
2449 .Case("{f2}", {RISCV::F2_32, RISCV::F2_64})
2450 .Case("{f3}", {RISCV::F3_32, RISCV::F3_64})
2451 .Case("{f4}", {RISCV::F4_32, RISCV::F4_64})
2452 .Case("{f5}", {RISCV::F5_32, RISCV::F5_64})
2453 .Case("{f6}", {RISCV::F6_32, RISCV::F6_64})
2454 .Case("{f7}", {RISCV::F7_32, RISCV::F7_64})
2455 .Case("{f8}", {RISCV::F8_32, RISCV::F8_64})
2456 .Case("{f9}", {RISCV::F9_32, RISCV::F9_64})
2457 .Case("{f10}", {RISCV::F10_32, RISCV::F10_64})
2458 .Case("{f11}", {RISCV::F11_32, RISCV::F11_64})
2459 .Case("{f12}", {RISCV::F12_32, RISCV::F12_64})
2460 .Case("{f13}", {RISCV::F13_32, RISCV::F13_64})
2461 .Case("{f14}", {RISCV::F14_32, RISCV::F14_64})
2462 .Case("{f15}", {RISCV::F15_32, RISCV::F15_64})
2463 .Case("{f16}", {RISCV::F16_32, RISCV::F16_64})
2464 .Case("{f17}", {RISCV::F17_32, RISCV::F17_64})
2465 .Case("{f18}", {RISCV::F18_32, RISCV::F18_64})
2466 .Case("{f19}", {RISCV::F19_32, RISCV::F19_64})
2467 .Case("{f20}", {RISCV::F20_32, RISCV::F20_64})
2468 .Case("{f21}", {RISCV::F21_32, RISCV::F21_64})
2469 .Case("{f22}", {RISCV::F22_32, RISCV::F22_64})
2470 .Case("{f23}", {RISCV::F23_32, RISCV::F23_64})
2471 .Case("{f24}", {RISCV::F24_32, RISCV::F24_64})
2472 .Case("{f25}", {RISCV::F25_32, RISCV::F25_64})
2473 .Case("{f26}", {RISCV::F26_32, RISCV::F26_64})
2474 .Case("{f27}", {RISCV::F27_32, RISCV::F27_64})
2475 .Case("{f28}", {RISCV::F28_32, RISCV::F28_64})
2476 .Case("{f29}", {RISCV::F29_32, RISCV::F29_64})
2477 .Case("{f30}", {RISCV::F30_32, RISCV::F30_64})
2478 .Case("{f31}", {RISCV::F31_32, RISCV::F31_64})
2479 .Default({-1U, -1U});
2480 if (FReg.first != -1U)
2481 return Subtarget.hasStdExtD()
2482 ? std::make_pair(FReg.second, &RISCV::FPR64RegClass)
2483 : std::make_pair(FReg.first, &RISCV::FPR32RegClass);
2486 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
2489 void RISCVTargetLowering::LowerAsmOperandForConstraint(
2490 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2491 SelectionDAG &DAG) const {
2492 // Currently only support length 1 constraints.
2493 if (Constraint.length() == 1) {
2494 switch (Constraint[0]) {
2495 case 'I':
2496 // Validate & create a 12-bit signed immediate operand.
2497 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2498 uint64_t CVal = C->getSExtValue();
2499 if (isInt<12>(CVal))
2500 Ops.push_back(
2501 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2503 return;
2504 case 'J':
2505 // Validate & create an integer zero operand.
2506 if (auto *C = dyn_cast<ConstantSDNode>(Op))
2507 if (C->getZExtValue() == 0)
2508 Ops.push_back(
2509 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
2510 return;
2511 case 'K':
2512 // Validate & create a 5-bit unsigned immediate operand.
2513 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2514 uint64_t CVal = C->getZExtValue();
2515 if (isUInt<5>(CVal))
2516 Ops.push_back(
2517 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2519 return;
2520 default:
2521 break;
2524 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
2527 Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
2528 Instruction *Inst,
2529 AtomicOrdering Ord) const {
2530 if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
2531 return Builder.CreateFence(Ord);
2532 if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
2533 return Builder.CreateFence(AtomicOrdering::Release);
2534 return nullptr;
2537 Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
2538 Instruction *Inst,
2539 AtomicOrdering Ord) const {
2540 if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
2541 return Builder.CreateFence(AtomicOrdering::Acquire);
2542 return nullptr;
2545 TargetLowering::AtomicExpansionKind
2546 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
2547 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
2548 // point operations can't be used in an lr/sc sequence without breaking the
2549 // forward-progress guarantee.
2550 if (AI->isFloatingPointOperation())
2551 return AtomicExpansionKind::CmpXChg;
2553 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
2554 if (Size == 8 || Size == 16)
2555 return AtomicExpansionKind::MaskedIntrinsic;
2556 return AtomicExpansionKind::None;
2559 static Intrinsic::ID
2560 getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
2561 if (XLen == 32) {
2562 switch (BinOp) {
2563 default:
2564 llvm_unreachable("Unexpected AtomicRMW BinOp");
2565 case AtomicRMWInst::Xchg:
2566 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
2567 case AtomicRMWInst::Add:
2568 return Intrinsic::riscv_masked_atomicrmw_add_i32;
2569 case AtomicRMWInst::Sub:
2570 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
2571 case AtomicRMWInst::Nand:
2572 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
2573 case AtomicRMWInst::Max:
2574 return Intrinsic::riscv_masked_atomicrmw_max_i32;
2575 case AtomicRMWInst::Min:
2576 return Intrinsic::riscv_masked_atomicrmw_min_i32;
2577 case AtomicRMWInst::UMax:
2578 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
2579 case AtomicRMWInst::UMin:
2580 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
2584 if (XLen == 64) {
2585 switch (BinOp) {
2586 default:
2587 llvm_unreachable("Unexpected AtomicRMW BinOp");
2588 case AtomicRMWInst::Xchg:
2589 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
2590 case AtomicRMWInst::Add:
2591 return Intrinsic::riscv_masked_atomicrmw_add_i64;
2592 case AtomicRMWInst::Sub:
2593 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
2594 case AtomicRMWInst::Nand:
2595 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
2596 case AtomicRMWInst::Max:
2597 return Intrinsic::riscv_masked_atomicrmw_max_i64;
2598 case AtomicRMWInst::Min:
2599 return Intrinsic::riscv_masked_atomicrmw_min_i64;
2600 case AtomicRMWInst::UMax:
2601 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
2602 case AtomicRMWInst::UMin:
2603 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
2607 llvm_unreachable("Unexpected XLen\n");
2610 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
2611 IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
2612 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
2613 unsigned XLen = Subtarget.getXLen();
2614 Value *Ordering =
2615 Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
2616 Type *Tys[] = {AlignedAddr->getType()};
2617 Function *LrwOpScwLoop = Intrinsic::getDeclaration(
2618 AI->getModule(),
2619 getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
2621 if (XLen == 64) {
2622 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
2623 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2624 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
2627 Value *Result;
2629 // Must pass the shift amount needed to sign extend the loaded value prior
2630 // to performing a signed comparison for min/max. ShiftAmt is the number of
2631 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
2632 // is the number of bits to left+right shift the value in order to
2633 // sign-extend.
2634 if (AI->getOperation() == AtomicRMWInst::Min ||
2635 AI->getOperation() == AtomicRMWInst::Max) {
2636 const DataLayout &DL = AI->getModule()->getDataLayout();
2637 unsigned ValWidth =
2638 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
2639 Value *SextShamt =
2640 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
2641 Result = Builder.CreateCall(LrwOpScwLoop,
2642 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
2643 } else {
2644 Result =
2645 Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
2648 if (XLen == 64)
2649 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2650 return Result;
2653 TargetLowering::AtomicExpansionKind
2654 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
2655 AtomicCmpXchgInst *CI) const {
2656 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
2657 if (Size == 8 || Size == 16)
2658 return AtomicExpansionKind::MaskedIntrinsic;
2659 return AtomicExpansionKind::None;
2662 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
2663 IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2664 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2665 unsigned XLen = Subtarget.getXLen();
2666 Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
2667 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
2668 if (XLen == 64) {
2669 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
2670 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
2671 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2672 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
2674 Type *Tys[] = {AlignedAddr->getType()};
2675 Function *MaskedCmpXchg =
2676 Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
2677 Value *Result = Builder.CreateCall(
2678 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
2679 if (XLen == 64)
2680 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2681 return Result;
2684 unsigned RISCVTargetLowering::getExceptionPointerRegister(
2685 const Constant *PersonalityFn) const {
2686 return RISCV::X10;
2689 unsigned RISCVTargetLowering::getExceptionSelectorRegister(
2690 const Constant *PersonalityFn) const {
2691 return RISCV::X11;