1 //===--- AArch64CallLowering.cpp - Call lowering --------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file implements the lowering of LLVM calls to machine code calls for
13 //===----------------------------------------------------------------------===//
15 #include "AArch64CallLowering.h"
16 #include "AArch64ISelLowering.h"
17 #include "AArch64MachineFunctionInfo.h"
18 #include "AArch64Subtarget.h"
19 #include "llvm/ADT/ArrayRef.h"
20 #include "llvm/ADT/SmallVector.h"
21 #include "llvm/CodeGen/Analysis.h"
22 #include "llvm/CodeGen/CallingConvLower.h"
23 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
24 #include "llvm/CodeGen/GlobalISel/Utils.h"
25 #include "llvm/CodeGen/LowLevelType.h"
26 #include "llvm/CodeGen/MachineBasicBlock.h"
27 #include "llvm/CodeGen/MachineFrameInfo.h"
28 #include "llvm/CodeGen/MachineFunction.h"
29 #include "llvm/CodeGen/MachineInstrBuilder.h"
30 #include "llvm/CodeGen/MachineMemOperand.h"
31 #include "llvm/CodeGen/MachineOperand.h"
32 #include "llvm/CodeGen/MachineRegisterInfo.h"
33 #include "llvm/CodeGen/TargetRegisterInfo.h"
34 #include "llvm/CodeGen/TargetSubtargetInfo.h"
35 #include "llvm/CodeGen/ValueTypes.h"
36 #include "llvm/IR/Argument.h"
37 #include "llvm/IR/Attributes.h"
38 #include "llvm/IR/Function.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/IR/Value.h"
41 #include "llvm/Support/MachineValueType.h"
47 #define DEBUG_TYPE "aarch64-call-lowering"
51 AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering
&TLI
)
52 : CallLowering(&TLI
) {}
55 struct IncomingArgHandler
: public CallLowering::ValueHandler
{
56 IncomingArgHandler(MachineIRBuilder
&MIRBuilder
, MachineRegisterInfo
&MRI
,
58 : ValueHandler(MIRBuilder
, MRI
, AssignFn
), StackUsed(0) {}
60 Register
getStackAddress(uint64_t Size
, int64_t Offset
,
61 MachinePointerInfo
&MPO
) override
{
62 auto &MFI
= MIRBuilder
.getMF().getFrameInfo();
63 int FI
= MFI
.CreateFixedObject(Size
, Offset
, true);
64 MPO
= MachinePointerInfo::getFixedStack(MIRBuilder
.getMF(), FI
);
65 Register AddrReg
= MRI
.createGenericVirtualRegister(LLT::pointer(0, 64));
66 MIRBuilder
.buildFrameIndex(AddrReg
, FI
);
67 StackUsed
= std::max(StackUsed
, Size
+ Offset
);
71 void assignValueToReg(Register ValVReg
, Register PhysReg
,
72 CCValAssign
&VA
) override
{
73 markPhysRegUsed(PhysReg
);
74 switch (VA
.getLocInfo()) {
76 MIRBuilder
.buildCopy(ValVReg
, PhysReg
);
78 case CCValAssign::LocInfo::SExt
:
79 case CCValAssign::LocInfo::ZExt
:
80 case CCValAssign::LocInfo::AExt
: {
81 auto Copy
= MIRBuilder
.buildCopy(LLT
{VA
.getLocVT()}, PhysReg
);
82 MIRBuilder
.buildTrunc(ValVReg
, Copy
);
88 void assignValueToAddress(Register ValVReg
, Register Addr
, uint64_t Size
,
89 MachinePointerInfo
&MPO
, CCValAssign
&VA
) override
{
90 // FIXME: Get alignment
91 auto MMO
= MIRBuilder
.getMF().getMachineMemOperand(
92 MPO
, MachineMemOperand::MOLoad
| MachineMemOperand::MOInvariant
, Size
,
94 MIRBuilder
.buildLoad(ValVReg
, Addr
, *MMO
);
97 /// How the physical register gets marked varies between formal
98 /// parameters (it's a basic-block live-in), and a call instruction
99 /// (it's an implicit-def of the BL).
100 virtual void markPhysRegUsed(unsigned PhysReg
) = 0;
102 bool isIncomingArgumentHandler() const override
{ return true; }
107 struct FormalArgHandler
: public IncomingArgHandler
{
108 FormalArgHandler(MachineIRBuilder
&MIRBuilder
, MachineRegisterInfo
&MRI
,
109 CCAssignFn
*AssignFn
)
110 : IncomingArgHandler(MIRBuilder
, MRI
, AssignFn
) {}
112 void markPhysRegUsed(unsigned PhysReg
) override
{
113 MIRBuilder
.getMRI()->addLiveIn(PhysReg
);
114 MIRBuilder
.getMBB().addLiveIn(PhysReg
);
118 struct CallReturnHandler
: public IncomingArgHandler
{
119 CallReturnHandler(MachineIRBuilder
&MIRBuilder
, MachineRegisterInfo
&MRI
,
120 MachineInstrBuilder MIB
, CCAssignFn
*AssignFn
)
121 : IncomingArgHandler(MIRBuilder
, MRI
, AssignFn
), MIB(MIB
) {}
123 void markPhysRegUsed(unsigned PhysReg
) override
{
124 MIB
.addDef(PhysReg
, RegState::Implicit
);
127 MachineInstrBuilder MIB
;
130 struct OutgoingArgHandler
: public CallLowering::ValueHandler
{
131 OutgoingArgHandler(MachineIRBuilder
&MIRBuilder
, MachineRegisterInfo
&MRI
,
132 MachineInstrBuilder MIB
, CCAssignFn
*AssignFn
,
133 CCAssignFn
*AssignFnVarArg
, bool IsTailCall
= false,
135 : ValueHandler(MIRBuilder
, MRI
, AssignFn
), MIB(MIB
),
136 AssignFnVarArg(AssignFnVarArg
), IsTailCall(IsTailCall
), FPDiff(FPDiff
),
139 bool isIncomingArgumentHandler() const override
{ return false; }
141 Register
getStackAddress(uint64_t Size
, int64_t Offset
,
142 MachinePointerInfo
&MPO
) override
{
143 MachineFunction
&MF
= MIRBuilder
.getMF();
144 LLT p0
= LLT::pointer(0, 64);
145 LLT s64
= LLT::scalar(64);
149 int FI
= MF
.getFrameInfo().CreateFixedObject(Size
, Offset
, true);
150 Register FIReg
= MRI
.createGenericVirtualRegister(p0
);
151 MIRBuilder
.buildFrameIndex(FIReg
, FI
);
152 MPO
= MachinePointerInfo::getFixedStack(MF
, FI
);
156 Register SPReg
= MRI
.createGenericVirtualRegister(p0
);
157 MIRBuilder
.buildCopy(SPReg
, Register(AArch64::SP
));
159 Register OffsetReg
= MRI
.createGenericVirtualRegister(s64
);
160 MIRBuilder
.buildConstant(OffsetReg
, Offset
);
162 Register AddrReg
= MRI
.createGenericVirtualRegister(p0
);
163 MIRBuilder
.buildGEP(AddrReg
, SPReg
, OffsetReg
);
165 MPO
= MachinePointerInfo::getStack(MF
, Offset
);
169 void assignValueToReg(Register ValVReg
, Register PhysReg
,
170 CCValAssign
&VA
) override
{
171 MIB
.addUse(PhysReg
, RegState::Implicit
);
172 Register ExtReg
= extendRegister(ValVReg
, VA
);
173 MIRBuilder
.buildCopy(PhysReg
, ExtReg
);
176 void assignValueToAddress(Register ValVReg
, Register Addr
, uint64_t Size
,
177 MachinePointerInfo
&MPO
, CCValAssign
&VA
) override
{
178 if (VA
.getLocInfo() == CCValAssign::LocInfo::AExt
) {
179 Size
= VA
.getLocVT().getSizeInBits() / 8;
180 ValVReg
= MIRBuilder
.buildAnyExt(LLT::scalar(Size
* 8), ValVReg
)
184 auto MMO
= MIRBuilder
.getMF().getMachineMemOperand(
185 MPO
, MachineMemOperand::MOStore
, Size
, 1);
186 MIRBuilder
.buildStore(ValVReg
, Addr
, *MMO
);
189 bool assignArg(unsigned ValNo
, MVT ValVT
, MVT LocVT
,
190 CCValAssign::LocInfo LocInfo
,
191 const CallLowering::ArgInfo
&Info
,
192 ISD::ArgFlagsTy Flags
,
193 CCState
&State
) override
{
196 Res
= AssignFn(ValNo
, ValVT
, LocVT
, LocInfo
, Flags
, State
);
198 Res
= AssignFnVarArg(ValNo
, ValVT
, LocVT
, LocInfo
, Flags
, State
);
200 StackSize
= State
.getNextStackOffset();
204 MachineInstrBuilder MIB
;
205 CCAssignFn
*AssignFnVarArg
;
208 /// For tail calls, the byte offset of the call's argument area from the
209 /// callee's. Unused elsewhere.
215 static bool doesCalleeRestoreStack(CallingConv::ID CallConv
, bool TailCallOpt
) {
216 return CallConv
== CallingConv::Fast
&& TailCallOpt
;
219 void AArch64CallLowering::splitToValueTypes(
220 const ArgInfo
&OrigArg
, SmallVectorImpl
<ArgInfo
> &SplitArgs
,
221 const DataLayout
&DL
, MachineRegisterInfo
&MRI
, CallingConv::ID CallConv
) const {
222 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
223 LLVMContext
&Ctx
= OrigArg
.Ty
->getContext();
225 if (OrigArg
.Ty
->isVoidTy())
228 SmallVector
<EVT
, 4> SplitVTs
;
229 SmallVector
<uint64_t, 4> Offsets
;
230 ComputeValueVTs(TLI
, DL
, OrigArg
.Ty
, SplitVTs
, &Offsets
, 0);
232 if (SplitVTs
.size() == 1) {
233 // No splitting to do, but we want to replace the original type (e.g. [1 x
234 // double] -> double).
235 SplitArgs
.emplace_back(OrigArg
.Regs
[0], SplitVTs
[0].getTypeForEVT(Ctx
),
236 OrigArg
.Flags
[0], OrigArg
.IsFixed
);
240 // Create one ArgInfo for each virtual register in the original ArgInfo.
241 assert(OrigArg
.Regs
.size() == SplitVTs
.size() && "Regs / types mismatch");
243 bool NeedsRegBlock
= TLI
.functionArgumentNeedsConsecutiveRegisters(
244 OrigArg
.Ty
, CallConv
, false);
245 for (unsigned i
= 0, e
= SplitVTs
.size(); i
< e
; ++i
) {
246 Type
*SplitTy
= SplitVTs
[i
].getTypeForEVT(Ctx
);
247 SplitArgs
.emplace_back(OrigArg
.Regs
[i
], SplitTy
, OrigArg
.Flags
[0],
250 SplitArgs
.back().Flags
[0].setInConsecutiveRegs();
253 SplitArgs
.back().Flags
[0].setInConsecutiveRegsLast();
256 bool AArch64CallLowering::lowerReturn(MachineIRBuilder
&MIRBuilder
,
258 ArrayRef
<Register
> VRegs
,
259 Register SwiftErrorVReg
) const {
260 auto MIB
= MIRBuilder
.buildInstrNoInsert(AArch64::RET_ReallyLR
);
261 assert(((Val
&& !VRegs
.empty()) || (!Val
&& VRegs
.empty())) &&
262 "Return value without a vreg");
265 if (!VRegs
.empty()) {
266 MachineFunction
&MF
= MIRBuilder
.getMF();
267 const Function
&F
= MF
.getFunction();
269 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
270 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
271 CCAssignFn
*AssignFn
= TLI
.CCAssignFnForReturn(F
.getCallingConv());
272 auto &DL
= F
.getParent()->getDataLayout();
273 LLVMContext
&Ctx
= Val
->getType()->getContext();
275 SmallVector
<EVT
, 4> SplitEVTs
;
276 ComputeValueVTs(TLI
, DL
, Val
->getType(), SplitEVTs
);
277 assert(VRegs
.size() == SplitEVTs
.size() &&
278 "For each split Type there should be exactly one VReg.");
280 SmallVector
<ArgInfo
, 8> SplitArgs
;
281 CallingConv::ID CC
= F
.getCallingConv();
283 for (unsigned i
= 0; i
< SplitEVTs
.size(); ++i
) {
284 if (TLI
.getNumRegistersForCallingConv(Ctx
, CC
, SplitEVTs
[i
]) > 1) {
285 LLVM_DEBUG(dbgs() << "Can't handle extended arg types which need split");
289 Register CurVReg
= VRegs
[i
];
290 ArgInfo CurArgInfo
= ArgInfo
{CurVReg
, SplitEVTs
[i
].getTypeForEVT(Ctx
)};
291 setArgFlags(CurArgInfo
, AttributeList::ReturnIndex
, DL
, F
);
293 // i1 is a special case because SDAG i1 true is naturally zero extended
294 // when widened using ANYEXT. We need to do it explicitly here.
295 if (MRI
.getType(CurVReg
).getSizeInBits() == 1) {
296 CurVReg
= MIRBuilder
.buildZExt(LLT::scalar(8), CurVReg
).getReg(0);
298 // Some types will need extending as specified by the CC.
299 MVT NewVT
= TLI
.getRegisterTypeForCallingConv(Ctx
, CC
, SplitEVTs
[i
]);
300 if (EVT(NewVT
) != SplitEVTs
[i
]) {
301 unsigned ExtendOp
= TargetOpcode::G_ANYEXT
;
302 if (F
.getAttributes().hasAttribute(AttributeList::ReturnIndex
,
304 ExtendOp
= TargetOpcode::G_SEXT
;
305 else if (F
.getAttributes().hasAttribute(AttributeList::ReturnIndex
,
307 ExtendOp
= TargetOpcode::G_ZEXT
;
310 LLT
OldLLT(MVT::getVT(CurArgInfo
.Ty
));
311 CurArgInfo
.Ty
= EVT(NewVT
).getTypeForEVT(Ctx
);
312 // Instead of an extend, we might have a vector type which needs
313 // padding with more elements, e.g. <2 x half> -> <4 x half>.
314 if (NewVT
.isVector()) {
315 if (OldLLT
.isVector()) {
316 if (NewLLT
.getNumElements() > OldLLT
.getNumElements()) {
317 // We don't handle VA types which are not exactly twice the
318 // size, but can easily be done in future.
319 if (NewLLT
.getNumElements() != OldLLT
.getNumElements() * 2) {
320 LLVM_DEBUG(dbgs() << "Outgoing vector ret has too many elts");
323 auto Undef
= MIRBuilder
.buildUndef({OldLLT
});
325 MIRBuilder
.buildMerge({NewLLT
}, {CurVReg
, Undef
.getReg(0)})
328 // Just do a vector extend.
329 CurVReg
= MIRBuilder
.buildInstr(ExtendOp
, {NewLLT
}, {CurVReg
})
332 } else if (NewLLT
.getNumElements() == 2) {
333 // We need to pad a <1 x S> type to <2 x S>. Since we don't have
334 // <1 x S> vector types in GISel we use a build_vector instead
335 // of a vector merge/concat.
336 auto Undef
= MIRBuilder
.buildUndef({OldLLT
});
339 .buildBuildVector({NewLLT
}, {CurVReg
, Undef
.getReg(0)})
342 LLVM_DEBUG(dbgs() << "Could not handle ret ty");
348 MIRBuilder
.buildInstr(ExtendOp
, {NewLLT
}, {CurVReg
}).getReg(0);
352 if (CurVReg
!= CurArgInfo
.Regs
[0]) {
353 CurArgInfo
.Regs
[0] = CurVReg
;
354 // Reset the arg flags after modifying CurVReg.
355 setArgFlags(CurArgInfo
, AttributeList::ReturnIndex
, DL
, F
);
357 splitToValueTypes(CurArgInfo
, SplitArgs
, DL
, MRI
, CC
);
360 OutgoingArgHandler
Handler(MIRBuilder
, MRI
, MIB
, AssignFn
, AssignFn
);
361 Success
= handleAssignments(MIRBuilder
, SplitArgs
, Handler
);
364 if (SwiftErrorVReg
) {
365 MIB
.addUse(AArch64::X21
, RegState::Implicit
);
366 MIRBuilder
.buildCopy(AArch64::X21
, SwiftErrorVReg
);
369 MIRBuilder
.insertInstr(MIB
);
373 /// Helper function to compute forwarded registers for musttail calls. Computes
374 /// the forwarded registers, sets MBB liveness, and emits COPY instructions that
375 /// can be used to save + restore registers later.
376 static void handleMustTailForwardedRegisters(MachineIRBuilder
&MIRBuilder
,
377 CCAssignFn
*AssignFn
) {
378 MachineBasicBlock
&MBB
= MIRBuilder
.getMBB();
379 MachineFunction
&MF
= MIRBuilder
.getMF();
380 MachineFrameInfo
&MFI
= MF
.getFrameInfo();
382 if (!MFI
.hasMustTailInVarArgFunc())
385 AArch64FunctionInfo
*FuncInfo
= MF
.getInfo
<AArch64FunctionInfo
>();
386 const Function
&F
= MF
.getFunction();
387 assert(F
.isVarArg() && "Expected F to be vararg?");
389 // Compute the set of forwarded registers. The rest are scratch.
390 SmallVector
<CCValAssign
, 16> ArgLocs
;
391 CCState
CCInfo(F
.getCallingConv(), /*IsVarArg=*/true, MF
, ArgLocs
,
393 SmallVector
<MVT
, 2> RegParmTypes
;
394 RegParmTypes
.push_back(MVT::i64
);
395 RegParmTypes
.push_back(MVT::f128
);
397 // Later on, we can use this vector to restore the registers if necessary.
398 SmallVectorImpl
<ForwardedRegister
> &Forwards
=
399 FuncInfo
->getForwardedMustTailRegParms();
400 CCInfo
.analyzeMustTailForwardedRegisters(Forwards
, RegParmTypes
, AssignFn
);
402 // Conservatively forward X8, since it might be used for an aggregate
404 if (!CCInfo
.isAllocated(AArch64::X8
)) {
405 unsigned X8VReg
= MF
.addLiveIn(AArch64::X8
, &AArch64::GPR64RegClass
);
406 Forwards
.push_back(ForwardedRegister(X8VReg
, AArch64::X8
, MVT::i64
));
409 // Add the forwards to the MachineBasicBlock and MachineFunction.
410 for (const auto &F
: Forwards
) {
411 MBB
.addLiveIn(F
.PReg
);
412 MIRBuilder
.buildCopy(Register(F
.VReg
), Register(F
.PReg
));
416 bool AArch64CallLowering::lowerFormalArguments(
417 MachineIRBuilder
&MIRBuilder
, const Function
&F
,
418 ArrayRef
<ArrayRef
<Register
>> VRegs
) const {
419 MachineFunction
&MF
= MIRBuilder
.getMF();
420 MachineBasicBlock
&MBB
= MIRBuilder
.getMBB();
421 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
422 auto &DL
= F
.getParent()->getDataLayout();
424 SmallVector
<ArgInfo
, 8> SplitArgs
;
426 for (auto &Arg
: F
.args()) {
427 if (DL
.getTypeStoreSize(Arg
.getType()) == 0)
430 ArgInfo OrigArg
{VRegs
[i
], Arg
.getType()};
431 setArgFlags(OrigArg
, i
+ AttributeList::FirstArgIndex
, DL
, F
);
433 splitToValueTypes(OrigArg
, SplitArgs
, DL
, MRI
, F
.getCallingConv());
438 MIRBuilder
.setInstr(*MBB
.begin());
440 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
441 CCAssignFn
*AssignFn
=
442 TLI
.CCAssignFnForCall(F
.getCallingConv(), /*IsVarArg=*/false);
444 FormalArgHandler
Handler(MIRBuilder
, MRI
, AssignFn
);
445 if (!handleAssignments(MIRBuilder
, SplitArgs
, Handler
))
448 AArch64FunctionInfo
*FuncInfo
= MF
.getInfo
<AArch64FunctionInfo
>();
449 uint64_t StackOffset
= Handler
.StackUsed
;
451 auto &Subtarget
= MF
.getSubtarget
<AArch64Subtarget
>();
452 if (!Subtarget
.isTargetDarwin()) {
453 // FIXME: we need to reimplement saveVarArgsRegisters from
454 // AArch64ISelLowering.
458 // We currently pass all varargs at 8-byte alignment, or 4 in ILP32.
459 StackOffset
= alignTo(Handler
.StackUsed
, Subtarget
.isTargetILP32() ? 4 : 8);
461 auto &MFI
= MIRBuilder
.getMF().getFrameInfo();
462 FuncInfo
->setVarArgsStackIndex(MFI
.CreateFixedObject(4, StackOffset
, true));
465 if (doesCalleeRestoreStack(F
.getCallingConv(),
466 MF
.getTarget().Options
.GuaranteedTailCallOpt
)) {
467 // We have a non-standard ABI, so why not make full use of the stack that
468 // we're going to pop? It must be aligned to 16 B in any case.
469 StackOffset
= alignTo(StackOffset
, 16);
471 // If we're expected to restore the stack (e.g. fastcc), then we'll be
472 // adding a multiple of 16.
473 FuncInfo
->setArgumentStackToRestore(StackOffset
);
475 // Our own callers will guarantee that the space is free by giving an
476 // aligned value to CALLSEQ_START.
479 // When we tail call, we need to check if the callee's arguments
480 // will fit on the caller's stack. So, whenever we lower formal arguments,
481 // we should keep track of this information, since we might lower a tail call
482 // in this function later.
483 FuncInfo
->setBytesInStackArgArea(StackOffset
);
485 auto &Subtarget
= MF
.getSubtarget
<AArch64Subtarget
>();
486 if (Subtarget
.hasCustomCallingConv())
487 Subtarget
.getRegisterInfo()->UpdateCustomCalleeSavedRegs(MF
);
489 handleMustTailForwardedRegisters(MIRBuilder
, AssignFn
);
491 // Move back to the end of the basic block.
492 MIRBuilder
.setMBB(MBB
);
497 /// Return true if the calling convention is one that we can guarantee TCO for.
498 static bool canGuaranteeTCO(CallingConv::ID CC
) {
499 return CC
== CallingConv::Fast
;
502 /// Return true if we might ever do TCO for calls with this calling convention.
503 static bool mayTailCallThisCC(CallingConv::ID CC
) {
506 case CallingConv::PreserveMost
:
507 case CallingConv::Swift
:
510 return canGuaranteeTCO(CC
);
514 /// Returns a pair containing the fixed CCAssignFn and the vararg CCAssignFn for
516 static std::pair
<CCAssignFn
*, CCAssignFn
*>
517 getAssignFnsForCC(CallingConv::ID CC
, const AArch64TargetLowering
&TLI
) {
518 return {TLI
.CCAssignFnForCall(CC
, false), TLI
.CCAssignFnForCall(CC
, true)};
521 bool AArch64CallLowering::doCallerAndCalleePassArgsTheSameWay(
522 CallLoweringInfo
&Info
, MachineFunction
&MF
,
523 SmallVectorImpl
<ArgInfo
> &InArgs
) const {
524 const Function
&CallerF
= MF
.getFunction();
525 CallingConv::ID CalleeCC
= Info
.CallConv
;
526 CallingConv::ID CallerCC
= CallerF
.getCallingConv();
528 // If the calling conventions match, then everything must be the same.
529 if (CalleeCC
== CallerCC
)
532 // Check if the caller and callee will handle arguments in the same way.
533 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
534 CCAssignFn
*CalleeAssignFnFixed
;
535 CCAssignFn
*CalleeAssignFnVarArg
;
536 std::tie(CalleeAssignFnFixed
, CalleeAssignFnVarArg
) =
537 getAssignFnsForCC(CalleeCC
, TLI
);
539 CCAssignFn
*CallerAssignFnFixed
;
540 CCAssignFn
*CallerAssignFnVarArg
;
541 std::tie(CallerAssignFnFixed
, CallerAssignFnVarArg
) =
542 getAssignFnsForCC(CallerCC
, TLI
);
544 if (!resultsCompatible(Info
, MF
, InArgs
, *CalleeAssignFnFixed
,
545 *CalleeAssignFnVarArg
, *CallerAssignFnFixed
,
546 *CallerAssignFnVarArg
))
549 // Make sure that the caller and callee preserve all of the same registers.
550 auto TRI
= MF
.getSubtarget
<AArch64Subtarget
>().getRegisterInfo();
551 const uint32_t *CallerPreserved
= TRI
->getCallPreservedMask(MF
, CallerCC
);
552 const uint32_t *CalleePreserved
= TRI
->getCallPreservedMask(MF
, CalleeCC
);
553 if (MF
.getSubtarget
<AArch64Subtarget
>().hasCustomCallingConv()) {
554 TRI
->UpdateCustomCallPreservedMask(MF
, &CallerPreserved
);
555 TRI
->UpdateCustomCallPreservedMask(MF
, &CalleePreserved
);
558 return TRI
->regmaskSubsetEqual(CallerPreserved
, CalleePreserved
);
561 bool AArch64CallLowering::areCalleeOutgoingArgsTailCallable(
562 CallLoweringInfo
&Info
, MachineFunction
&MF
,
563 SmallVectorImpl
<ArgInfo
> &OutArgs
) const {
564 // If there are no outgoing arguments, then we are done.
568 const Function
&CallerF
= MF
.getFunction();
569 CallingConv::ID CalleeCC
= Info
.CallConv
;
570 CallingConv::ID CallerCC
= CallerF
.getCallingConv();
571 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
573 CCAssignFn
*AssignFnFixed
;
574 CCAssignFn
*AssignFnVarArg
;
575 std::tie(AssignFnFixed
, AssignFnVarArg
) = getAssignFnsForCC(CalleeCC
, TLI
);
577 // We have outgoing arguments. Make sure that we can tail call with them.
578 SmallVector
<CCValAssign
, 16> OutLocs
;
579 CCState
OutInfo(CalleeCC
, false, MF
, OutLocs
, CallerF
.getContext());
581 if (!analyzeArgInfo(OutInfo
, OutArgs
, *AssignFnFixed
, *AssignFnVarArg
)) {
582 LLVM_DEBUG(dbgs() << "... Could not analyze call operands.\n");
586 // Make sure that they can fit on the caller's stack.
587 const AArch64FunctionInfo
*FuncInfo
= MF
.getInfo
<AArch64FunctionInfo
>();
588 if (OutInfo
.getNextStackOffset() > FuncInfo
->getBytesInStackArgArea()) {
589 LLVM_DEBUG(dbgs() << "... Cannot fit call operands on caller's stack.\n");
593 // Verify that the parameters in callee-saved registers match.
594 // TODO: Port this over to CallLowering as general code once swiftself is
596 auto TRI
= MF
.getSubtarget
<AArch64Subtarget
>().getRegisterInfo();
597 const uint32_t *CallerPreservedMask
= TRI
->getCallPreservedMask(MF
, CallerCC
);
598 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
600 for (unsigned i
= 0; i
< OutLocs
.size(); ++i
) {
601 auto &ArgLoc
= OutLocs
[i
];
602 // If it's not a register, it's fine.
603 if (!ArgLoc
.isRegLoc()) {
605 // Be conservative and disallow variadic memory operands to match SDAG's
607 // FIXME: If the caller's calling convention is C, then we can
608 // potentially use its argument area. However, for cases like fastcc,
609 // we can't do anything.
612 << "... Cannot tail call vararg function with stack arguments\n");
618 Register Reg
= ArgLoc
.getLocReg();
620 // Only look at callee-saved registers.
621 if (MachineOperand::clobbersPhysReg(CallerPreservedMask
, Reg
))
626 << "... Call has an argument passed in a callee-saved register.\n");
628 // Check if it was copied from.
629 ArgInfo
&OutInfo
= OutArgs
[i
];
631 if (OutInfo
.Regs
.size() > 1) {
633 dbgs() << "... Cannot handle arguments in multiple registers.\n");
637 // Check if we copy the register, walking through copies from virtual
638 // registers. Note that getDefIgnoringCopies does not ignore copies from
639 // physical registers.
640 MachineInstr
*RegDef
= getDefIgnoringCopies(OutInfo
.Regs
[0], MRI
);
641 if (!RegDef
|| RegDef
->getOpcode() != TargetOpcode::COPY
) {
644 << "... Parameter was not copied into a VReg, cannot tail call.\n");
648 // Got a copy. Verify that it's the same as the register we want.
649 Register CopyRHS
= RegDef
->getOperand(1).getReg();
650 if (CopyRHS
!= Reg
) {
651 LLVM_DEBUG(dbgs() << "... Callee-saved register was not copied into "
652 "VReg, cannot tail call.\n");
660 bool AArch64CallLowering::isEligibleForTailCallOptimization(
661 MachineIRBuilder
&MIRBuilder
, CallLoweringInfo
&Info
,
662 SmallVectorImpl
<ArgInfo
> &InArgs
,
663 SmallVectorImpl
<ArgInfo
> &OutArgs
) const {
665 // Must pass all target-independent checks in order to tail call optimize.
666 if (!Info
.IsTailCall
)
669 CallingConv::ID CalleeCC
= Info
.CallConv
;
670 MachineFunction
&MF
= MIRBuilder
.getMF();
671 const Function
&CallerF
= MF
.getFunction();
673 LLVM_DEBUG(dbgs() << "Attempting to lower call as tail call\n");
675 if (Info
.SwiftErrorVReg
) {
676 // TODO: We should handle this.
677 // Note that this is also handled by the check for no outgoing arguments.
678 // Proactively disabling this though, because the swifterror handling in
679 // lowerCall inserts a COPY *after* the location of the call.
680 LLVM_DEBUG(dbgs() << "... Cannot handle tail calls with swifterror yet.\n");
684 if (!mayTailCallThisCC(CalleeCC
)) {
685 LLVM_DEBUG(dbgs() << "... Calling convention cannot be tail called.\n");
689 // Byval parameters hand the function a pointer directly into the stack area
690 // we want to reuse during a tail call. Working around this *is* possible (see
693 // FIXME: In AArch64ISelLowering, this isn't worked around. Can/should we try
696 // On Windows, "inreg" attributes signify non-aggregate indirect returns.
697 // In this case, it is necessary to save/restore X0 in the callee. Tail
698 // call opt interferes with this. So we disable tail call opt when the
699 // caller has an argument with "inreg" attribute.
701 // FIXME: Check whether the callee also has an "inreg" argument.
703 // When the caller has a swifterror argument, we don't want to tail call
704 // because would have to move into the swifterror register before the
706 if (any_of(CallerF
.args(), [](const Argument
&A
) {
707 return A
.hasByValAttr() || A
.hasInRegAttr() || A
.hasSwiftErrorAttr();
709 LLVM_DEBUG(dbgs() << "... Cannot tail call from callers with byval, "
710 "inreg, or swifterror arguments\n");
714 // Externally-defined functions with weak linkage should not be
715 // tail-called on AArch64 when the OS does not support dynamic
716 // pre-emption of symbols, as the AAELF spec requires normal calls
717 // to undefined weak functions to be replaced with a NOP or jump to the
718 // next instruction. The behaviour of branch instructions in this
719 // situation (as used for tail calls) is implementation-defined, so we
720 // cannot rely on the linker replacing the tail call with a return.
721 if (Info
.Callee
.isGlobal()) {
722 const GlobalValue
*GV
= Info
.Callee
.getGlobal();
723 const Triple
&TT
= MF
.getTarget().getTargetTriple();
724 if (GV
->hasExternalWeakLinkage() &&
725 (!TT
.isOSWindows() || TT
.isOSBinFormatELF() ||
726 TT
.isOSBinFormatMachO())) {
727 LLVM_DEBUG(dbgs() << "... Cannot tail call externally-defined function "
728 "with weak linkage for this OS.\n");
733 // If we have -tailcallopt, then we're done.
734 if (MF
.getTarget().Options
.GuaranteedTailCallOpt
)
735 return canGuaranteeTCO(CalleeCC
) && CalleeCC
== CallerF
.getCallingConv();
737 // We don't have -tailcallopt, so we're allowed to change the ABI (sibcall).
738 // Try to find cases where we can do that.
740 // I want anyone implementing a new calling convention to think long and hard
741 // about this assert.
742 assert((!Info
.IsVarArg
|| CalleeCC
== CallingConv::C
) &&
743 "Unexpected variadic calling convention");
745 // Verify that the incoming and outgoing arguments from the callee are
746 // safe to tail call.
747 if (!doCallerAndCalleePassArgsTheSameWay(Info
, MF
, InArgs
)) {
750 << "... Caller and callee have incompatible calling conventions.\n");
754 if (!areCalleeOutgoingArgsTailCallable(Info
, MF
, OutArgs
))
758 dbgs() << "... Call is eligible for tail call optimization.\n");
762 static unsigned getCallOpcode(const Function
&CallerF
, bool IsIndirect
,
765 return IsIndirect
? AArch64::BLR
: AArch64::BL
;
768 return AArch64::TCRETURNdi
;
770 // When BTI is enabled, we need to use TCRETURNriBTI to make sure that we use
772 if (CallerF
.hasFnAttribute("branch-target-enforcement"))
773 return AArch64::TCRETURNriBTI
;
775 return AArch64::TCRETURNri
;
778 bool AArch64CallLowering::lowerTailCall(
779 MachineIRBuilder
&MIRBuilder
, CallLoweringInfo
&Info
,
780 SmallVectorImpl
<ArgInfo
> &OutArgs
) const {
781 MachineFunction
&MF
= MIRBuilder
.getMF();
782 const Function
&F
= MF
.getFunction();
783 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
784 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
785 AArch64FunctionInfo
*FuncInfo
= MF
.getInfo
<AArch64FunctionInfo
>();
787 // True when we're tail calling, but without -tailcallopt.
788 bool IsSibCall
= !MF
.getTarget().Options
.GuaranteedTailCallOpt
;
790 // TODO: Right now, regbankselect doesn't know how to handle the rtcGPR64
791 // register class. Until we can do that, we should fall back here.
792 if (F
.hasFnAttribute("branch-target-enforcement")) {
794 dbgs() << "Cannot lower indirect tail calls with BTI enabled yet.\n");
798 // Find out which ABI gets to decide where things go.
799 CallingConv::ID CalleeCC
= Info
.CallConv
;
800 CCAssignFn
*AssignFnFixed
;
801 CCAssignFn
*AssignFnVarArg
;
802 std::tie(AssignFnFixed
, AssignFnVarArg
) = getAssignFnsForCC(CalleeCC
, TLI
);
804 MachineInstrBuilder CallSeqStart
;
806 CallSeqStart
= MIRBuilder
.buildInstr(AArch64::ADJCALLSTACKDOWN
);
808 unsigned Opc
= getCallOpcode(F
, Info
.Callee
.isReg(), true);
809 auto MIB
= MIRBuilder
.buildInstrNoInsert(Opc
);
810 MIB
.add(Info
.Callee
);
812 // Byte offset for the tail call. When we are sibcalling, this will always
816 // Tell the call which registers are clobbered.
817 auto TRI
= MF
.getSubtarget
<AArch64Subtarget
>().getRegisterInfo();
818 const uint32_t *Mask
= TRI
->getCallPreservedMask(MF
, F
.getCallingConv());
819 if (MF
.getSubtarget
<AArch64Subtarget
>().hasCustomCallingConv())
820 TRI
->UpdateCustomCallPreservedMask(MF
, &Mask
);
821 MIB
.addRegMask(Mask
);
823 if (TRI
->isAnyArgRegReserved(MF
))
824 TRI
->emitReservedArgRegCallError(MF
);
826 // FPDiff is the byte offset of the call's argument area from the callee's.
827 // Stores to callee stack arguments will be placed in FixedStackSlots offset
828 // by this amount for a tail call. In a sibling call it must be 0 because the
829 // caller will deallocate the entire stack and the callee still expects its
830 // arguments to begin at SP+0.
833 // This will be 0 for sibcalls, potentially nonzero for tail calls produced
834 // by -tailcallopt. For sibcalls, the memory operands for the call are
835 // already available in the caller's incoming argument space.
836 unsigned NumBytes
= 0;
838 // We aren't sibcalling, so we need to compute FPDiff. We need to do this
839 // before handling assignments, because FPDiff must be known for memory
841 unsigned NumReusableBytes
= FuncInfo
->getBytesInStackArgArea();
842 SmallVector
<CCValAssign
, 16> OutLocs
;
843 CCState
OutInfo(CalleeCC
, false, MF
, OutLocs
, F
.getContext());
844 analyzeArgInfo(OutInfo
, OutArgs
, *AssignFnFixed
, *AssignFnVarArg
);
846 // The callee will pop the argument stack as a tail call. Thus, we must
847 // keep it 16-byte aligned.
848 NumBytes
= alignTo(OutInfo
.getNextStackOffset(), 16);
850 // FPDiff will be negative if this tail call requires more space than we
851 // would automatically have in our incoming argument space. Positive if we
852 // actually shrink the stack.
853 FPDiff
= NumReusableBytes
- NumBytes
;
855 // The stack pointer must be 16-byte aligned at all times it's used for a
856 // memory operation, which in practice means at *all* times and in
857 // particular across call boundaries. Therefore our own arguments started at
858 // a 16-byte aligned SP and the delta applied for the tail call should
859 // satisfy the same constraint.
860 assert(FPDiff
% 16 == 0 && "unaligned stack on tail call");
863 const auto &Forwards
= FuncInfo
->getForwardedMustTailRegParms();
865 // Do the actual argument marshalling.
866 SmallVector
<unsigned, 8> PhysRegs
;
867 OutgoingArgHandler
Handler(MIRBuilder
, MRI
, MIB
, AssignFnFixed
,
868 AssignFnVarArg
, true, FPDiff
);
869 if (!handleAssignments(MIRBuilder
, OutArgs
, Handler
))
872 if (Info
.IsVarArg
&& Info
.IsMustTailCall
) {
873 // Now we know what's being passed to the function. Add uses to the call for
874 // the forwarded registers that we *aren't* passing as parameters. This will
875 // preserve the copies we build earlier.
876 for (const auto &F
: Forwards
) {
877 Register ForwardedReg
= F
.PReg
;
878 // If the register is already passed, or aliases a register which is
879 // already being passed, then skip it.
880 if (any_of(MIB
->uses(), [&ForwardedReg
, &TRI
](const MachineOperand
&Use
) {
883 return TRI
->regsOverlap(Use
.getReg(), ForwardedReg
);
887 // We aren't passing it already, so we should add it to the call.
888 MIRBuilder
.buildCopy(ForwardedReg
, Register(F
.VReg
));
889 MIB
.addReg(ForwardedReg
, RegState::Implicit
);
893 // If we have -tailcallopt, we need to adjust the stack. We'll do the call
894 // sequence start and end here.
896 MIB
->getOperand(1).setImm(FPDiff
);
897 CallSeqStart
.addImm(NumBytes
).addImm(0);
898 // End the call sequence *before* emitting the call. Normally, we would
899 // tidy the frame up after the call. However, here, we've laid out the
900 // parameters so that when SP is reset, they will be in the correct
902 MIRBuilder
.buildInstr(AArch64::ADJCALLSTACKUP
).addImm(NumBytes
).addImm(0);
905 // Now we can add the actual call instruction to the correct basic block.
906 MIRBuilder
.insertInstr(MIB
);
908 // If Callee is a reg, since it is used by a target specific instruction,
909 // it must have a register class matching the constraint of that instruction.
910 if (Info
.Callee
.isReg())
911 MIB
->getOperand(0).setReg(constrainOperandRegClass(
912 MF
, *TRI
, MRI
, *MF
.getSubtarget().getInstrInfo(),
913 *MF
.getSubtarget().getRegBankInfo(), *MIB
, MIB
->getDesc(), Info
.Callee
,
916 MF
.getFrameInfo().setHasTailCall();
917 Info
.LoweredTailCall
= true;
921 bool AArch64CallLowering::lowerCall(MachineIRBuilder
&MIRBuilder
,
922 CallLoweringInfo
&Info
) const {
923 MachineFunction
&MF
= MIRBuilder
.getMF();
924 const Function
&F
= MF
.getFunction();
925 MachineRegisterInfo
&MRI
= MF
.getRegInfo();
926 auto &DL
= F
.getParent()->getDataLayout();
927 const AArch64TargetLowering
&TLI
= *getTLI
<AArch64TargetLowering
>();
929 SmallVector
<ArgInfo
, 8> OutArgs
;
930 for (auto &OrigArg
: Info
.OrigArgs
) {
931 splitToValueTypes(OrigArg
, OutArgs
, DL
, MRI
, Info
.CallConv
);
932 // AAPCS requires that we zero-extend i1 to 8 bits by the caller.
933 if (OrigArg
.Ty
->isIntegerTy(1))
934 OutArgs
.back().Flags
[0].setZExt();
937 SmallVector
<ArgInfo
, 8> InArgs
;
938 if (!Info
.OrigRet
.Ty
->isVoidTy())
939 splitToValueTypes(Info
.OrigRet
, InArgs
, DL
, MRI
, F
.getCallingConv());
941 // If we can lower as a tail call, do that instead.
942 bool CanTailCallOpt
=
943 isEligibleForTailCallOptimization(MIRBuilder
, Info
, InArgs
, OutArgs
);
945 // We must emit a tail call if we have musttail.
946 if (Info
.IsMustTailCall
&& !CanTailCallOpt
) {
947 // There are types of incoming/outgoing arguments we can't handle yet, so
948 // it doesn't make sense to actually die here like in ISelLowering. Instead,
949 // fall back to SelectionDAG and let it try to handle this.
950 LLVM_DEBUG(dbgs() << "Failed to lower musttail call as tail call\n");
955 return lowerTailCall(MIRBuilder
, Info
, OutArgs
);
957 // Find out which ABI gets to decide where things go.
958 CCAssignFn
*AssignFnFixed
;
959 CCAssignFn
*AssignFnVarArg
;
960 std::tie(AssignFnFixed
, AssignFnVarArg
) =
961 getAssignFnsForCC(Info
.CallConv
, TLI
);
963 MachineInstrBuilder CallSeqStart
;
964 CallSeqStart
= MIRBuilder
.buildInstr(AArch64::ADJCALLSTACKDOWN
);
966 // Create a temporarily-floating call instruction so we can add the implicit
967 // uses of arg registers.
968 unsigned Opc
= getCallOpcode(F
, Info
.Callee
.isReg(), false);
970 auto MIB
= MIRBuilder
.buildInstrNoInsert(Opc
);
971 MIB
.add(Info
.Callee
);
973 // Tell the call which registers are clobbered.
974 auto TRI
= MF
.getSubtarget
<AArch64Subtarget
>().getRegisterInfo();
975 const uint32_t *Mask
= TRI
->getCallPreservedMask(MF
, F
.getCallingConv());
976 if (MF
.getSubtarget
<AArch64Subtarget
>().hasCustomCallingConv())
977 TRI
->UpdateCustomCallPreservedMask(MF
, &Mask
);
978 MIB
.addRegMask(Mask
);
980 if (TRI
->isAnyArgRegReserved(MF
))
981 TRI
->emitReservedArgRegCallError(MF
);
983 // Do the actual argument marshalling.
984 SmallVector
<unsigned, 8> PhysRegs
;
985 OutgoingArgHandler
Handler(MIRBuilder
, MRI
, MIB
, AssignFnFixed
,
986 AssignFnVarArg
, false);
987 if (!handleAssignments(MIRBuilder
, OutArgs
, Handler
))
990 // Now we can add the actual call instruction to the correct basic block.
991 MIRBuilder
.insertInstr(MIB
);
993 // If Callee is a reg, since it is used by a target specific
994 // instruction, it must have a register class matching the
995 // constraint of that instruction.
996 if (Info
.Callee
.isReg())
997 MIB
->getOperand(0).setReg(constrainOperandRegClass(
998 MF
, *TRI
, MRI
, *MF
.getSubtarget().getInstrInfo(),
999 *MF
.getSubtarget().getRegBankInfo(), *MIB
, MIB
->getDesc(), Info
.Callee
,
1002 // Finally we can copy the returned value back into its virtual-register. In
1003 // symmetry with the arugments, the physical register must be an
1004 // implicit-define of the call instruction.
1005 if (!Info
.OrigRet
.Ty
->isVoidTy()) {
1006 CCAssignFn
*RetAssignFn
= TLI
.CCAssignFnForReturn(F
.getCallingConv());
1007 CallReturnHandler
Handler(MIRBuilder
, MRI
, MIB
, RetAssignFn
);
1008 if (!handleAssignments(MIRBuilder
, InArgs
, Handler
))
1012 if (Info
.SwiftErrorVReg
) {
1013 MIB
.addDef(AArch64::X21
, RegState::Implicit
);
1014 MIRBuilder
.buildCopy(Info
.SwiftErrorVReg
, Register(AArch64::X21
));
1017 uint64_t CalleePopBytes
=
1018 doesCalleeRestoreStack(Info
.CallConv
,
1019 MF
.getTarget().Options
.GuaranteedTailCallOpt
)
1020 ? alignTo(Handler
.StackSize
, 16)
1023 CallSeqStart
.addImm(Handler
.StackSize
).addImm(0);
1024 MIRBuilder
.buildInstr(AArch64::ADJCALLSTACKUP
)
1025 .addImm(Handler
.StackSize
)
1026 .addImm(CalleePopBytes
);