1 //===-- X86FastISel.cpp - X86 FastISel implementation ---------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file defines the X86-specific support for the FastISel class. Much
10 // of the target-specific code is generated by tablegen in the file
11 // X86GenFastISel.inc, which is #included here.
13 //===----------------------------------------------------------------------===//
16 #include "X86CallingConv.h"
17 #include "X86InstrBuilder.h"
18 #include "X86InstrInfo.h"
19 #include "X86MachineFunctionInfo.h"
20 #include "X86RegisterInfo.h"
21 #include "X86Subtarget.h"
22 #include "X86TargetMachine.h"
23 #include "llvm/Analysis/BranchProbabilityInfo.h"
24 #include "llvm/CodeGen/FastISel.h"
25 #include "llvm/CodeGen/FunctionLoweringInfo.h"
26 #include "llvm/CodeGen/MachineConstantPool.h"
27 #include "llvm/CodeGen/MachineFrameInfo.h"
28 #include "llvm/CodeGen/MachineRegisterInfo.h"
29 #include "llvm/IR/CallSite.h"
30 #include "llvm/IR/CallingConv.h"
31 #include "llvm/IR/DebugInfo.h"
32 #include "llvm/IR/DerivedTypes.h"
33 #include "llvm/IR/GetElementPtrTypeIterator.h"
34 #include "llvm/IR/GlobalAlias.h"
35 #include "llvm/IR/GlobalVariable.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/IntrinsicInst.h"
38 #include "llvm/IR/Operator.h"
39 #include "llvm/MC/MCAsmInfo.h"
40 #include "llvm/MC/MCSymbol.h"
41 #include "llvm/Support/ErrorHandling.h"
42 #include "llvm/Target/TargetOptions.h"
47 class X86FastISel final
: public FastISel
{
48 /// Subtarget - Keep a pointer to the X86Subtarget around so that we can
49 /// make the right decision when generating code for different targets.
50 const X86Subtarget
*Subtarget
;
52 /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87
53 /// floating point ops.
54 /// When SSE is available, use it for f32 operations.
55 /// When SSE2 is available, use it for f64 operations.
60 explicit X86FastISel(FunctionLoweringInfo
&funcInfo
,
61 const TargetLibraryInfo
*libInfo
)
62 : FastISel(funcInfo
, libInfo
) {
63 Subtarget
= &funcInfo
.MF
->getSubtarget
<X86Subtarget
>();
64 X86ScalarSSEf64
= Subtarget
->hasSSE2();
65 X86ScalarSSEf32
= Subtarget
->hasSSE1();
68 bool fastSelectInstruction(const Instruction
*I
) override
;
70 /// The specified machine instr operand is a vreg, and that
71 /// vreg is being provided by the specified load instruction. If possible,
72 /// try to fold the load as an operand to the instruction, returning true if
74 bool tryToFoldLoadIntoMI(MachineInstr
*MI
, unsigned OpNo
,
75 const LoadInst
*LI
) override
;
77 bool fastLowerArguments() override
;
78 bool fastLowerCall(CallLoweringInfo
&CLI
) override
;
79 bool fastLowerIntrinsicCall(const IntrinsicInst
*II
) override
;
81 #include "X86GenFastISel.inc"
84 bool X86FastEmitCompare(const Value
*LHS
, const Value
*RHS
, EVT VT
,
87 bool X86FastEmitLoad(MVT VT
, X86AddressMode
&AM
, MachineMemOperand
*MMO
,
88 unsigned &ResultReg
, unsigned Alignment
= 1);
90 bool X86FastEmitStore(EVT VT
, const Value
*Val
, X86AddressMode
&AM
,
91 MachineMemOperand
*MMO
= nullptr, bool Aligned
= false);
92 bool X86FastEmitStore(EVT VT
, unsigned ValReg
, bool ValIsKill
,
94 MachineMemOperand
*MMO
= nullptr, bool Aligned
= false);
96 bool X86FastEmitExtend(ISD::NodeType Opc
, EVT DstVT
, unsigned Src
, EVT SrcVT
,
99 bool X86SelectAddress(const Value
*V
, X86AddressMode
&AM
);
100 bool X86SelectCallAddress(const Value
*V
, X86AddressMode
&AM
);
102 bool X86SelectLoad(const Instruction
*I
);
104 bool X86SelectStore(const Instruction
*I
);
106 bool X86SelectRet(const Instruction
*I
);
108 bool X86SelectCmp(const Instruction
*I
);
110 bool X86SelectZExt(const Instruction
*I
);
112 bool X86SelectSExt(const Instruction
*I
);
114 bool X86SelectBranch(const Instruction
*I
);
116 bool X86SelectShift(const Instruction
*I
);
118 bool X86SelectDivRem(const Instruction
*I
);
120 bool X86FastEmitCMoveSelect(MVT RetVT
, const Instruction
*I
);
122 bool X86FastEmitSSESelect(MVT RetVT
, const Instruction
*I
);
124 bool X86FastEmitPseudoSelect(MVT RetVT
, const Instruction
*I
);
126 bool X86SelectSelect(const Instruction
*I
);
128 bool X86SelectTrunc(const Instruction
*I
);
130 bool X86SelectFPExtOrFPTrunc(const Instruction
*I
, unsigned Opc
,
131 const TargetRegisterClass
*RC
);
133 bool X86SelectFPExt(const Instruction
*I
);
134 bool X86SelectFPTrunc(const Instruction
*I
);
135 bool X86SelectSIToFP(const Instruction
*I
);
136 bool X86SelectUIToFP(const Instruction
*I
);
137 bool X86SelectIntToFP(const Instruction
*I
, bool IsSigned
);
139 const X86InstrInfo
*getInstrInfo() const {
140 return Subtarget
->getInstrInfo();
142 const X86TargetMachine
*getTargetMachine() const {
143 return static_cast<const X86TargetMachine
*>(&TM
);
146 bool handleConstantAddresses(const Value
*V
, X86AddressMode
&AM
);
148 unsigned X86MaterializeInt(const ConstantInt
*CI
, MVT VT
);
149 unsigned X86MaterializeFP(const ConstantFP
*CFP
, MVT VT
);
150 unsigned X86MaterializeGV(const GlobalValue
*GV
, MVT VT
);
151 unsigned fastMaterializeConstant(const Constant
*C
) override
;
153 unsigned fastMaterializeAlloca(const AllocaInst
*C
) override
;
155 unsigned fastMaterializeFloatZero(const ConstantFP
*CF
) override
;
157 /// isScalarFPTypeInSSEReg - Return true if the specified scalar FP type is
158 /// computed in an SSE register, not on the X87 floating point stack.
159 bool isScalarFPTypeInSSEReg(EVT VT
) const {
160 return (VT
== MVT::f64
&& X86ScalarSSEf64
) || // f64 is when SSE2
161 (VT
== MVT::f32
&& X86ScalarSSEf32
); // f32 is when SSE1
164 bool isTypeLegal(Type
*Ty
, MVT
&VT
, bool AllowI1
= false);
166 bool IsMemcpySmall(uint64_t Len
);
168 bool TryEmitSmallMemcpy(X86AddressMode DestAM
,
169 X86AddressMode SrcAM
, uint64_t Len
);
171 bool foldX86XALUIntrinsic(X86::CondCode
&CC
, const Instruction
*I
,
174 const MachineInstrBuilder
&addFullAddress(const MachineInstrBuilder
&MIB
,
177 unsigned fastEmitInst_rrrr(unsigned MachineInstOpcode
,
178 const TargetRegisterClass
*RC
, unsigned Op0
,
179 bool Op0IsKill
, unsigned Op1
, bool Op1IsKill
,
180 unsigned Op2
, bool Op2IsKill
, unsigned Op3
,
184 } // end anonymous namespace.
186 static std::pair
<unsigned, bool>
187 getX86SSEConditionCode(CmpInst::Predicate Predicate
) {
189 bool NeedSwap
= false;
191 // SSE Condition code mapping:
201 default: llvm_unreachable("Unexpected predicate");
202 case CmpInst::FCMP_OEQ
: CC
= 0; break;
203 case CmpInst::FCMP_OGT
: NeedSwap
= true; LLVM_FALLTHROUGH
;
204 case CmpInst::FCMP_OLT
: CC
= 1; break;
205 case CmpInst::FCMP_OGE
: NeedSwap
= true; LLVM_FALLTHROUGH
;
206 case CmpInst::FCMP_OLE
: CC
= 2; break;
207 case CmpInst::FCMP_UNO
: CC
= 3; break;
208 case CmpInst::FCMP_UNE
: CC
= 4; break;
209 case CmpInst::FCMP_ULE
: NeedSwap
= true; LLVM_FALLTHROUGH
;
210 case CmpInst::FCMP_UGE
: CC
= 5; break;
211 case CmpInst::FCMP_ULT
: NeedSwap
= true; LLVM_FALLTHROUGH
;
212 case CmpInst::FCMP_UGT
: CC
= 6; break;
213 case CmpInst::FCMP_ORD
: CC
= 7; break;
214 case CmpInst::FCMP_UEQ
: CC
= 8; break;
215 case CmpInst::FCMP_ONE
: CC
= 12; break;
218 return std::make_pair(CC
, NeedSwap
);
221 /// Adds a complex addressing mode to the given machine instr builder.
222 /// Note, this will constrain the index register. If its not possible to
223 /// constrain the given index register, then a new one will be created. The
224 /// IndexReg field of the addressing mode will be updated to match in this case.
225 const MachineInstrBuilder
&
226 X86FastISel::addFullAddress(const MachineInstrBuilder
&MIB
,
227 X86AddressMode
&AM
) {
228 // First constrain the index register. It needs to be a GR64_NOSP.
229 AM
.IndexReg
= constrainOperandRegClass(MIB
->getDesc(), AM
.IndexReg
,
230 MIB
->getNumOperands() +
232 return ::addFullAddress(MIB
, AM
);
235 /// Check if it is possible to fold the condition from the XALU intrinsic
236 /// into the user. The condition code will only be updated on success.
237 bool X86FastISel::foldX86XALUIntrinsic(X86::CondCode
&CC
, const Instruction
*I
,
239 if (!isa
<ExtractValueInst
>(Cond
))
242 const auto *EV
= cast
<ExtractValueInst
>(Cond
);
243 if (!isa
<IntrinsicInst
>(EV
->getAggregateOperand()))
246 const auto *II
= cast
<IntrinsicInst
>(EV
->getAggregateOperand());
248 const Function
*Callee
= II
->getCalledFunction();
250 cast
<StructType
>(Callee
->getReturnType())->getTypeAtIndex(0U);
251 if (!isTypeLegal(RetTy
, RetVT
))
254 if (RetVT
!= MVT::i32
&& RetVT
!= MVT::i64
)
258 switch (II
->getIntrinsicID()) {
259 default: return false;
260 case Intrinsic::sadd_with_overflow
:
261 case Intrinsic::ssub_with_overflow
:
262 case Intrinsic::smul_with_overflow
:
263 case Intrinsic::umul_with_overflow
: TmpCC
= X86::COND_O
; break;
264 case Intrinsic::uadd_with_overflow
:
265 case Intrinsic::usub_with_overflow
: TmpCC
= X86::COND_B
; break;
268 // Check if both instructions are in the same basic block.
269 if (II
->getParent() != I
->getParent())
272 // Make sure nothing is in the way
273 BasicBlock::const_iterator
Start(I
);
274 BasicBlock::const_iterator
End(II
);
275 for (auto Itr
= std::prev(Start
); Itr
!= End
; --Itr
) {
276 // We only expect extractvalue instructions between the intrinsic and the
277 // instruction to be selected.
278 if (!isa
<ExtractValueInst
>(Itr
))
281 // Check that the extractvalue operand comes from the intrinsic.
282 const auto *EVI
= cast
<ExtractValueInst
>(Itr
);
283 if (EVI
->getAggregateOperand() != II
)
291 bool X86FastISel::isTypeLegal(Type
*Ty
, MVT
&VT
, bool AllowI1
) {
292 EVT evt
= TLI
.getValueType(DL
, Ty
, /*HandleUnknown=*/true);
293 if (evt
== MVT::Other
|| !evt
.isSimple())
294 // Unhandled type. Halt "fast" selection and bail.
297 VT
= evt
.getSimpleVT();
298 // For now, require SSE/SSE2 for performing floating-point operations,
299 // since x87 requires additional work.
300 if (VT
== MVT::f64
&& !X86ScalarSSEf64
)
302 if (VT
== MVT::f32
&& !X86ScalarSSEf32
)
304 // Similarly, no f80 support yet.
307 // We only handle legal types. For example, on x86-32 the instruction
308 // selector contains all of the 64-bit instructions from x86-64,
309 // under the assumption that i64 won't be used if the target doesn't
311 return (AllowI1
&& VT
== MVT::i1
) || TLI
.isTypeLegal(VT
);
314 /// X86FastEmitLoad - Emit a machine instruction to load a value of type VT.
315 /// The address is either pre-computed, i.e. Ptr, or a GlobalAddress, i.e. GV.
316 /// Return true and the result register by reference if it is possible.
317 bool X86FastISel::X86FastEmitLoad(MVT VT
, X86AddressMode
&AM
,
318 MachineMemOperand
*MMO
, unsigned &ResultReg
,
319 unsigned Alignment
) {
320 bool HasSSE41
= Subtarget
->hasSSE41();
321 bool HasAVX
= Subtarget
->hasAVX();
322 bool HasAVX2
= Subtarget
->hasAVX2();
323 bool HasAVX512
= Subtarget
->hasAVX512();
324 bool HasVLX
= Subtarget
->hasVLX();
325 bool IsNonTemporal
= MMO
&& MMO
->isNonTemporal();
327 // Treat i1 loads the same as i8 loads. Masking will be done when storing.
331 // Get opcode and regclass of the output for the given load instruction.
333 switch (VT
.SimpleTy
) {
334 default: return false;
345 // Must be in x86-64 mode.
350 Opc
= HasAVX512
? X86::VMOVSSZrm_alt
:
351 HasAVX
? X86::VMOVSSrm_alt
:
358 Opc
= HasAVX512
? X86::VMOVSDZrm_alt
:
359 HasAVX
? X86::VMOVSDrm_alt
:
365 // No f80 support yet.
368 if (IsNonTemporal
&& Alignment
>= 16 && HasSSE41
)
369 Opc
= HasVLX
? X86::VMOVNTDQAZ128rm
:
370 HasAVX
? X86::VMOVNTDQArm
: X86::MOVNTDQArm
;
371 else if (Alignment
>= 16)
372 Opc
= HasVLX
? X86::VMOVAPSZ128rm
:
373 HasAVX
? X86::VMOVAPSrm
: X86::MOVAPSrm
;
375 Opc
= HasVLX
? X86::VMOVUPSZ128rm
:
376 HasAVX
? X86::VMOVUPSrm
: X86::MOVUPSrm
;
379 if (IsNonTemporal
&& Alignment
>= 16 && HasSSE41
)
380 Opc
= HasVLX
? X86::VMOVNTDQAZ128rm
:
381 HasAVX
? X86::VMOVNTDQArm
: X86::MOVNTDQArm
;
382 else if (Alignment
>= 16)
383 Opc
= HasVLX
? X86::VMOVAPDZ128rm
:
384 HasAVX
? X86::VMOVAPDrm
: X86::MOVAPDrm
;
386 Opc
= HasVLX
? X86::VMOVUPDZ128rm
:
387 HasAVX
? X86::VMOVUPDrm
: X86::MOVUPDrm
;
393 if (IsNonTemporal
&& Alignment
>= 16 && HasSSE41
)
394 Opc
= HasVLX
? X86::VMOVNTDQAZ128rm
:
395 HasAVX
? X86::VMOVNTDQArm
: X86::MOVNTDQArm
;
396 else if (Alignment
>= 16)
397 Opc
= HasVLX
? X86::VMOVDQA64Z128rm
:
398 HasAVX
? X86::VMOVDQArm
: X86::MOVDQArm
;
400 Opc
= HasVLX
? X86::VMOVDQU64Z128rm
:
401 HasAVX
? X86::VMOVDQUrm
: X86::MOVDQUrm
;
405 if (IsNonTemporal
&& Alignment
>= 32 && HasAVX2
)
406 Opc
= HasVLX
? X86::VMOVNTDQAZ256rm
: X86::VMOVNTDQAYrm
;
407 else if (IsNonTemporal
&& Alignment
>= 16)
408 return false; // Force split for X86::VMOVNTDQArm
409 else if (Alignment
>= 32)
410 Opc
= HasVLX
? X86::VMOVAPSZ256rm
: X86::VMOVAPSYrm
;
412 Opc
= HasVLX
? X86::VMOVUPSZ256rm
: X86::VMOVUPSYrm
;
416 if (IsNonTemporal
&& Alignment
>= 32 && HasAVX2
)
417 Opc
= HasVLX
? X86::VMOVNTDQAZ256rm
: X86::VMOVNTDQAYrm
;
418 else if (IsNonTemporal
&& Alignment
>= 16)
419 return false; // Force split for X86::VMOVNTDQArm
420 else if (Alignment
>= 32)
421 Opc
= HasVLX
? X86::VMOVAPDZ256rm
: X86::VMOVAPDYrm
;
423 Opc
= HasVLX
? X86::VMOVUPDZ256rm
: X86::VMOVUPDYrm
;
430 if (IsNonTemporal
&& Alignment
>= 32 && HasAVX2
)
431 Opc
= HasVLX
? X86::VMOVNTDQAZ256rm
: X86::VMOVNTDQAYrm
;
432 else if (IsNonTemporal
&& Alignment
>= 16)
433 return false; // Force split for X86::VMOVNTDQArm
434 else if (Alignment
>= 32)
435 Opc
= HasVLX
? X86::VMOVDQA64Z256rm
: X86::VMOVDQAYrm
;
437 Opc
= HasVLX
? X86::VMOVDQU64Z256rm
: X86::VMOVDQUYrm
;
441 if (IsNonTemporal
&& Alignment
>= 64)
442 Opc
= X86::VMOVNTDQAZrm
;
444 Opc
= (Alignment
>= 64) ? X86::VMOVAPSZrm
: X86::VMOVUPSZrm
;
448 if (IsNonTemporal
&& Alignment
>= 64)
449 Opc
= X86::VMOVNTDQAZrm
;
451 Opc
= (Alignment
>= 64) ? X86::VMOVAPDZrm
: X86::VMOVUPDZrm
;
458 // Note: There are a lot more choices based on type with AVX-512, but
459 // there's really no advantage when the load isn't masked.
460 if (IsNonTemporal
&& Alignment
>= 64)
461 Opc
= X86::VMOVNTDQAZrm
;
463 Opc
= (Alignment
>= 64) ? X86::VMOVDQA64Zrm
: X86::VMOVDQU64Zrm
;
467 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(VT
);
469 ResultReg
= createResultReg(RC
);
470 MachineInstrBuilder MIB
=
471 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Opc
), ResultReg
);
472 addFullAddress(MIB
, AM
);
474 MIB
->addMemOperand(*FuncInfo
.MF
, MMO
);
478 /// X86FastEmitStore - Emit a machine instruction to store a value Val of
479 /// type VT. The address is either pre-computed, consisted of a base ptr, Ptr
480 /// and a displacement offset, or a GlobalAddress,
481 /// i.e. V. Return true if it is possible.
482 bool X86FastISel::X86FastEmitStore(EVT VT
, unsigned ValReg
, bool ValIsKill
,
484 MachineMemOperand
*MMO
, bool Aligned
) {
485 bool HasSSE1
= Subtarget
->hasSSE1();
486 bool HasSSE2
= Subtarget
->hasSSE2();
487 bool HasSSE4A
= Subtarget
->hasSSE4A();
488 bool HasAVX
= Subtarget
->hasAVX();
489 bool HasAVX512
= Subtarget
->hasAVX512();
490 bool HasVLX
= Subtarget
->hasVLX();
491 bool IsNonTemporal
= MMO
&& MMO
->isNonTemporal();
493 // Get opcode and regclass of the output for the given store instruction.
495 switch (VT
.getSimpleVT().SimpleTy
) {
496 case MVT::f80
: // No f80 support yet.
497 default: return false;
499 // Mask out all but lowest bit.
500 unsigned AndResult
= createResultReg(&X86::GR8RegClass
);
501 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
502 TII
.get(X86::AND8ri
), AndResult
)
503 .addReg(ValReg
, getKillRegState(ValIsKill
)).addImm(1);
505 LLVM_FALLTHROUGH
; // handle i1 as i8.
507 case MVT::i8
: Opc
= X86::MOV8mr
; break;
508 case MVT::i16
: Opc
= X86::MOV16mr
; break;
510 Opc
= (IsNonTemporal
&& HasSSE2
) ? X86::MOVNTImr
: X86::MOV32mr
;
513 // Must be in x86-64 mode.
514 Opc
= (IsNonTemporal
&& HasSSE2
) ? X86::MOVNTI_64mr
: X86::MOV64mr
;
517 if (X86ScalarSSEf32
) {
518 if (IsNonTemporal
&& HasSSE4A
)
521 Opc
= HasAVX512
? X86::VMOVSSZmr
:
522 HasAVX
? X86::VMOVSSmr
: X86::MOVSSmr
;
527 if (X86ScalarSSEf32
) {
528 if (IsNonTemporal
&& HasSSE4A
)
531 Opc
= HasAVX512
? X86::VMOVSDZmr
:
532 HasAVX
? X86::VMOVSDmr
: X86::MOVSDmr
;
537 Opc
= (IsNonTemporal
&& HasSSE1
) ? X86::MMX_MOVNTQmr
: X86::MMX_MOVQ64mr
;
542 Opc
= HasVLX
? X86::VMOVNTPSZ128mr
:
543 HasAVX
? X86::VMOVNTPSmr
: X86::MOVNTPSmr
;
545 Opc
= HasVLX
? X86::VMOVAPSZ128mr
:
546 HasAVX
? X86::VMOVAPSmr
: X86::MOVAPSmr
;
548 Opc
= HasVLX
? X86::VMOVUPSZ128mr
:
549 HasAVX
? X86::VMOVUPSmr
: X86::MOVUPSmr
;
554 Opc
= HasVLX
? X86::VMOVNTPDZ128mr
:
555 HasAVX
? X86::VMOVNTPDmr
: X86::MOVNTPDmr
;
557 Opc
= HasVLX
? X86::VMOVAPDZ128mr
:
558 HasAVX
? X86::VMOVAPDmr
: X86::MOVAPDmr
;
560 Opc
= HasVLX
? X86::VMOVUPDZ128mr
:
561 HasAVX
? X86::VMOVUPDmr
: X86::MOVUPDmr
;
569 Opc
= HasVLX
? X86::VMOVNTDQZ128mr
:
570 HasAVX
? X86::VMOVNTDQmr
: X86::MOVNTDQmr
;
572 Opc
= HasVLX
? X86::VMOVDQA64Z128mr
:
573 HasAVX
? X86::VMOVDQAmr
: X86::MOVDQAmr
;
575 Opc
= HasVLX
? X86::VMOVDQU64Z128mr
:
576 HasAVX
? X86::VMOVDQUmr
: X86::MOVDQUmr
;
582 Opc
= HasVLX
? X86::VMOVNTPSZ256mr
: X86::VMOVNTPSYmr
;
584 Opc
= HasVLX
? X86::VMOVAPSZ256mr
: X86::VMOVAPSYmr
;
586 Opc
= HasVLX
? X86::VMOVUPSZ256mr
: X86::VMOVUPSYmr
;
592 Opc
= HasVLX
? X86::VMOVNTPDZ256mr
: X86::VMOVNTPDYmr
;
594 Opc
= HasVLX
? X86::VMOVAPDZ256mr
: X86::VMOVAPDYmr
;
596 Opc
= HasVLX
? X86::VMOVUPDZ256mr
: X86::VMOVUPDYmr
;
605 Opc
= HasVLX
? X86::VMOVNTDQZ256mr
: X86::VMOVNTDQYmr
;
607 Opc
= HasVLX
? X86::VMOVDQA64Z256mr
: X86::VMOVDQAYmr
;
609 Opc
= HasVLX
? X86::VMOVDQU64Z256mr
: X86::VMOVDQUYmr
;
614 Opc
= IsNonTemporal
? X86::VMOVNTPSZmr
: X86::VMOVAPSZmr
;
616 Opc
= X86::VMOVUPSZmr
;
621 Opc
= IsNonTemporal
? X86::VMOVNTPDZmr
: X86::VMOVAPDZmr
;
623 Opc
= X86::VMOVUPDZmr
;
630 // Note: There are a lot more choices based on type with AVX-512, but
631 // there's really no advantage when the store isn't masked.
633 Opc
= IsNonTemporal
? X86::VMOVNTDQZmr
: X86::VMOVDQA64Zmr
;
635 Opc
= X86::VMOVDQU64Zmr
;
639 const MCInstrDesc
&Desc
= TII
.get(Opc
);
640 // Some of the instructions in the previous switch use FR128 instead
641 // of FR32 for ValReg. Make sure the register we feed the instruction
642 // matches its register class constraints.
643 // Note: This is fine to do a copy from FR32 to FR128, this is the
644 // same registers behind the scene and actually why it did not trigger
646 ValReg
= constrainOperandRegClass(Desc
, ValReg
, Desc
.getNumOperands() - 1);
647 MachineInstrBuilder MIB
=
648 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, Desc
);
649 addFullAddress(MIB
, AM
).addReg(ValReg
, getKillRegState(ValIsKill
));
651 MIB
->addMemOperand(*FuncInfo
.MF
, MMO
);
656 bool X86FastISel::X86FastEmitStore(EVT VT
, const Value
*Val
,
658 MachineMemOperand
*MMO
, bool Aligned
) {
659 // Handle 'null' like i32/i64 0.
660 if (isa
<ConstantPointerNull
>(Val
))
661 Val
= Constant::getNullValue(DL
.getIntPtrType(Val
->getContext()));
663 // If this is a store of a simple constant, fold the constant into the store.
664 if (const ConstantInt
*CI
= dyn_cast
<ConstantInt
>(Val
)) {
667 switch (VT
.getSimpleVT().SimpleTy
) {
671 LLVM_FALLTHROUGH
; // Handle as i8.
672 case MVT::i8
: Opc
= X86::MOV8mi
; break;
673 case MVT::i16
: Opc
= X86::MOV16mi
; break;
674 case MVT::i32
: Opc
= X86::MOV32mi
; break;
676 // Must be a 32-bit sign extended value.
677 if (isInt
<32>(CI
->getSExtValue()))
678 Opc
= X86::MOV64mi32
;
683 MachineInstrBuilder MIB
=
684 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Opc
));
685 addFullAddress(MIB
, AM
).addImm(Signed
? (uint64_t) CI
->getSExtValue()
686 : CI
->getZExtValue());
688 MIB
->addMemOperand(*FuncInfo
.MF
, MMO
);
693 unsigned ValReg
= getRegForValue(Val
);
697 bool ValKill
= hasTrivialKill(Val
);
698 return X86FastEmitStore(VT
, ValReg
, ValKill
, AM
, MMO
, Aligned
);
701 /// X86FastEmitExtend - Emit a machine instruction to extend a value Src of
702 /// type SrcVT to type DstVT using the specified extension opcode Opc (e.g.
703 /// ISD::SIGN_EXTEND).
704 bool X86FastISel::X86FastEmitExtend(ISD::NodeType Opc
, EVT DstVT
,
705 unsigned Src
, EVT SrcVT
,
706 unsigned &ResultReg
) {
707 unsigned RR
= fastEmit_r(SrcVT
.getSimpleVT(), DstVT
.getSimpleVT(), Opc
,
708 Src
, /*TODO: Kill=*/false);
716 bool X86FastISel::handleConstantAddresses(const Value
*V
, X86AddressMode
&AM
) {
717 // Handle constant address.
718 if (const GlobalValue
*GV
= dyn_cast
<GlobalValue
>(V
)) {
719 // Can't handle alternate code models yet.
720 if (TM
.getCodeModel() != CodeModel::Small
)
723 // Can't handle TLS yet.
724 if (GV
->isThreadLocal())
727 // Can't handle !absolute_symbol references yet.
728 if (GV
->isAbsoluteSymbolRef())
731 // RIP-relative addresses can't have additional register operands, so if
732 // we've already folded stuff into the addressing mode, just force the
733 // global value into its own register, which we can use as the basereg.
734 if (!Subtarget
->isPICStyleRIPRel() ||
735 (AM
.Base
.Reg
== 0 && AM
.IndexReg
== 0)) {
736 // Okay, we've committed to selecting this global. Set up the address.
739 // Allow the subtarget to classify the global.
740 unsigned char GVFlags
= Subtarget
->classifyGlobalReference(GV
);
742 // If this reference is relative to the pic base, set it now.
743 if (isGlobalRelativeToPICBase(GVFlags
)) {
744 // FIXME: How do we know Base.Reg is free??
745 AM
.Base
.Reg
= getInstrInfo()->getGlobalBaseReg(FuncInfo
.MF
);
748 // Unless the ABI requires an extra load, return a direct reference to
750 if (!isGlobalStubReference(GVFlags
)) {
751 if (Subtarget
->isPICStyleRIPRel()) {
752 // Use rip-relative addressing if we can. Above we verified that the
753 // base and index registers are unused.
754 assert(AM
.Base
.Reg
== 0 && AM
.IndexReg
== 0);
755 AM
.Base
.Reg
= X86::RIP
;
757 AM
.GVOpFlags
= GVFlags
;
761 // Ok, we need to do a load from a stub. If we've already loaded from
762 // this stub, reuse the loaded pointer, otherwise emit the load now.
763 DenseMap
<const Value
*, unsigned>::iterator I
= LocalValueMap
.find(V
);
765 if (I
!= LocalValueMap
.end() && I
->second
!= 0) {
768 // Issue load from stub.
770 const TargetRegisterClass
*RC
= nullptr;
771 X86AddressMode StubAM
;
772 StubAM
.Base
.Reg
= AM
.Base
.Reg
;
774 StubAM
.GVOpFlags
= GVFlags
;
776 // Prepare for inserting code in the local-value area.
777 SavePoint SaveInsertPt
= enterLocalValueArea();
779 if (TLI
.getPointerTy(DL
) == MVT::i64
) {
781 RC
= &X86::GR64RegClass
;
783 if (Subtarget
->isPICStyleRIPRel())
784 StubAM
.Base
.Reg
= X86::RIP
;
787 RC
= &X86::GR32RegClass
;
790 LoadReg
= createResultReg(RC
);
791 MachineInstrBuilder LoadMI
=
792 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Opc
), LoadReg
);
793 addFullAddress(LoadMI
, StubAM
);
795 // Ok, back to normal mode.
796 leaveLocalValueArea(SaveInsertPt
);
798 // Prevent loading GV stub multiple times in same MBB.
799 LocalValueMap
[V
] = LoadReg
;
802 // Now construct the final address. Note that the Disp, Scale,
803 // and Index values may already be set here.
804 AM
.Base
.Reg
= LoadReg
;
810 // If all else fails, try to materialize the value in a register.
811 if (!AM
.GV
|| !Subtarget
->isPICStyleRIPRel()) {
812 if (AM
.Base
.Reg
== 0) {
813 AM
.Base
.Reg
= getRegForValue(V
);
814 return AM
.Base
.Reg
!= 0;
816 if (AM
.IndexReg
== 0) {
817 assert(AM
.Scale
== 1 && "Scale with no index!");
818 AM
.IndexReg
= getRegForValue(V
);
819 return AM
.IndexReg
!= 0;
826 /// X86SelectAddress - Attempt to fill in an address from the given value.
828 bool X86FastISel::X86SelectAddress(const Value
*V
, X86AddressMode
&AM
) {
829 SmallVector
<const Value
*, 32> GEPs
;
831 const User
*U
= nullptr;
832 unsigned Opcode
= Instruction::UserOp1
;
833 if (const Instruction
*I
= dyn_cast
<Instruction
>(V
)) {
834 // Don't walk into other basic blocks; it's possible we haven't
835 // visited them yet, so the instructions may not yet be assigned
836 // virtual registers.
837 if (FuncInfo
.StaticAllocaMap
.count(static_cast<const AllocaInst
*>(V
)) ||
838 FuncInfo
.MBBMap
[I
->getParent()] == FuncInfo
.MBB
) {
839 Opcode
= I
->getOpcode();
842 } else if (const ConstantExpr
*C
= dyn_cast
<ConstantExpr
>(V
)) {
843 Opcode
= C
->getOpcode();
847 if (PointerType
*Ty
= dyn_cast
<PointerType
>(V
->getType()))
848 if (Ty
->getAddressSpace() > 255)
849 // Fast instruction selection doesn't support the special
855 case Instruction::BitCast
:
856 // Look past bitcasts.
857 return X86SelectAddress(U
->getOperand(0), AM
);
859 case Instruction::IntToPtr
:
860 // Look past no-op inttoptrs.
861 if (TLI
.getValueType(DL
, U
->getOperand(0)->getType()) ==
862 TLI
.getPointerTy(DL
))
863 return X86SelectAddress(U
->getOperand(0), AM
);
866 case Instruction::PtrToInt
:
867 // Look past no-op ptrtoints.
868 if (TLI
.getValueType(DL
, U
->getType()) == TLI
.getPointerTy(DL
))
869 return X86SelectAddress(U
->getOperand(0), AM
);
872 case Instruction::Alloca
: {
873 // Do static allocas.
874 const AllocaInst
*A
= cast
<AllocaInst
>(V
);
875 DenseMap
<const AllocaInst
*, int>::iterator SI
=
876 FuncInfo
.StaticAllocaMap
.find(A
);
877 if (SI
!= FuncInfo
.StaticAllocaMap
.end()) {
878 AM
.BaseType
= X86AddressMode::FrameIndexBase
;
879 AM
.Base
.FrameIndex
= SI
->second
;
885 case Instruction::Add
: {
886 // Adds of constants are common and easy enough.
887 if (const ConstantInt
*CI
= dyn_cast
<ConstantInt
>(U
->getOperand(1))) {
888 uint64_t Disp
= (int32_t)AM
.Disp
+ (uint64_t)CI
->getSExtValue();
889 // They have to fit in the 32-bit signed displacement field though.
890 if (isInt
<32>(Disp
)) {
891 AM
.Disp
= (uint32_t)Disp
;
892 return X86SelectAddress(U
->getOperand(0), AM
);
898 case Instruction::GetElementPtr
: {
899 X86AddressMode SavedAM
= AM
;
901 // Pattern-match simple GEPs.
902 uint64_t Disp
= (int32_t)AM
.Disp
;
903 unsigned IndexReg
= AM
.IndexReg
;
904 unsigned Scale
= AM
.Scale
;
905 gep_type_iterator GTI
= gep_type_begin(U
);
906 // Iterate through the indices, folding what we can. Constants can be
907 // folded, and one dynamic index can be handled, if the scale is supported.
908 for (User::const_op_iterator i
= U
->op_begin() + 1, e
= U
->op_end();
909 i
!= e
; ++i
, ++GTI
) {
910 const Value
*Op
= *i
;
911 if (StructType
*STy
= GTI
.getStructTypeOrNull()) {
912 const StructLayout
*SL
= DL
.getStructLayout(STy
);
913 Disp
+= SL
->getElementOffset(cast
<ConstantInt
>(Op
)->getZExtValue());
917 // A array/variable index is always of the form i*S where S is the
918 // constant scale size. See if we can push the scale into immediates.
919 uint64_t S
= DL
.getTypeAllocSize(GTI
.getIndexedType());
921 if (const ConstantInt
*CI
= dyn_cast
<ConstantInt
>(Op
)) {
922 // Constant-offset addressing.
923 Disp
+= CI
->getSExtValue() * S
;
926 if (canFoldAddIntoGEP(U
, Op
)) {
927 // A compatible add with a constant operand. Fold the constant.
929 cast
<ConstantInt
>(cast
<AddOperator
>(Op
)->getOperand(1));
930 Disp
+= CI
->getSExtValue() * S
;
931 // Iterate on the other operand.
932 Op
= cast
<AddOperator
>(Op
)->getOperand(0);
936 (!AM
.GV
|| !Subtarget
->isPICStyleRIPRel()) &&
937 (S
== 1 || S
== 2 || S
== 4 || S
== 8)) {
938 // Scaled-index addressing.
940 IndexReg
= getRegForGEPIndex(Op
).first
;
946 goto unsupported_gep
;
950 // Check for displacement overflow.
951 if (!isInt
<32>(Disp
))
954 AM
.IndexReg
= IndexReg
;
956 AM
.Disp
= (uint32_t)Disp
;
959 if (const GetElementPtrInst
*GEP
=
960 dyn_cast
<GetElementPtrInst
>(U
->getOperand(0))) {
961 // Ok, the GEP indices were covered by constant-offset and scaled-index
962 // addressing. Update the address state and move on to examining the base.
965 } else if (X86SelectAddress(U
->getOperand(0), AM
)) {
969 // If we couldn't merge the gep value into this addr mode, revert back to
970 // our address and just match the value instead of completely failing.
973 for (const Value
*I
: reverse(GEPs
))
974 if (handleConstantAddresses(I
, AM
))
979 // Ok, the GEP indices weren't all covered.
984 return handleConstantAddresses(V
, AM
);
987 /// X86SelectCallAddress - Attempt to fill in an address from the given value.
989 bool X86FastISel::X86SelectCallAddress(const Value
*V
, X86AddressMode
&AM
) {
990 const User
*U
= nullptr;
991 unsigned Opcode
= Instruction::UserOp1
;
992 const Instruction
*I
= dyn_cast
<Instruction
>(V
);
993 // Record if the value is defined in the same basic block.
995 // This information is crucial to know whether or not folding an
997 // Indeed, FastISel generates or reuses a virtual register for all
998 // operands of all instructions it selects. Obviously, the definition and
999 // its uses must use the same virtual register otherwise the produced
1000 // code is incorrect.
1001 // Before instruction selection, FunctionLoweringInfo::set sets the virtual
1002 // registers for values that are alive across basic blocks. This ensures
1003 // that the values are consistently set between across basic block, even
1004 // if different instruction selection mechanisms are used (e.g., a mix of
1005 // SDISel and FastISel).
1006 // For values local to a basic block, the instruction selection process
1007 // generates these virtual registers with whatever method is appropriate
1008 // for its needs. In particular, FastISel and SDISel do not share the way
1009 // local virtual registers are set.
1010 // Therefore, this is impossible (or at least unsafe) to share values
1011 // between basic blocks unless they use the same instruction selection
1012 // method, which is not guarantee for X86.
1013 // Moreover, things like hasOneUse could not be used accurately, if we
1014 // allow to reference values across basic blocks whereas they are not
1015 // alive across basic blocks initially.
1018 Opcode
= I
->getOpcode();
1020 InMBB
= I
->getParent() == FuncInfo
.MBB
->getBasicBlock();
1021 } else if (const ConstantExpr
*C
= dyn_cast
<ConstantExpr
>(V
)) {
1022 Opcode
= C
->getOpcode();
1028 case Instruction::BitCast
:
1029 // Look past bitcasts if its operand is in the same BB.
1031 return X86SelectCallAddress(U
->getOperand(0), AM
);
1034 case Instruction::IntToPtr
:
1035 // Look past no-op inttoptrs if its operand is in the same BB.
1037 TLI
.getValueType(DL
, U
->getOperand(0)->getType()) ==
1038 TLI
.getPointerTy(DL
))
1039 return X86SelectCallAddress(U
->getOperand(0), AM
);
1042 case Instruction::PtrToInt
:
1043 // Look past no-op ptrtoints if its operand is in the same BB.
1044 if (InMBB
&& TLI
.getValueType(DL
, U
->getType()) == TLI
.getPointerTy(DL
))
1045 return X86SelectCallAddress(U
->getOperand(0), AM
);
1049 // Handle constant address.
1050 if (const GlobalValue
*GV
= dyn_cast
<GlobalValue
>(V
)) {
1051 // Can't handle alternate code models yet.
1052 if (TM
.getCodeModel() != CodeModel::Small
)
1055 // RIP-relative addresses can't have additional register operands.
1056 if (Subtarget
->isPICStyleRIPRel() &&
1057 (AM
.Base
.Reg
!= 0 || AM
.IndexReg
!= 0))
1060 // Can't handle TLS.
1061 if (const GlobalVariable
*GVar
= dyn_cast
<GlobalVariable
>(GV
))
1062 if (GVar
->isThreadLocal())
1065 // Okay, we've committed to selecting this global. Set up the basic address.
1068 // Return a direct reference to the global. Fastisel can handle calls to
1069 // functions that require loads, such as dllimport and nonlazybind
1071 if (Subtarget
->isPICStyleRIPRel()) {
1072 // Use rip-relative addressing if we can. Above we verified that the
1073 // base and index registers are unused.
1074 assert(AM
.Base
.Reg
== 0 && AM
.IndexReg
== 0);
1075 AM
.Base
.Reg
= X86::RIP
;
1077 AM
.GVOpFlags
= Subtarget
->classifyLocalReference(nullptr);
1083 // If all else fails, try to materialize the value in a register.
1084 if (!AM
.GV
|| !Subtarget
->isPICStyleRIPRel()) {
1085 if (AM
.Base
.Reg
== 0) {
1086 AM
.Base
.Reg
= getRegForValue(V
);
1087 return AM
.Base
.Reg
!= 0;
1089 if (AM
.IndexReg
== 0) {
1090 assert(AM
.Scale
== 1 && "Scale with no index!");
1091 AM
.IndexReg
= getRegForValue(V
);
1092 return AM
.IndexReg
!= 0;
1100 /// X86SelectStore - Select and emit code to implement store instructions.
1101 bool X86FastISel::X86SelectStore(const Instruction
*I
) {
1102 // Atomic stores need special handling.
1103 const StoreInst
*S
= cast
<StoreInst
>(I
);
1108 const Value
*PtrV
= I
->getOperand(1);
1109 if (TLI
.supportSwiftError()) {
1110 // Swifterror values can come from either a function parameter with
1111 // swifterror attribute or an alloca with swifterror attribute.
1112 if (const Argument
*Arg
= dyn_cast
<Argument
>(PtrV
)) {
1113 if (Arg
->hasSwiftErrorAttr())
1117 if (const AllocaInst
*Alloca
= dyn_cast
<AllocaInst
>(PtrV
)) {
1118 if (Alloca
->isSwiftError())
1123 const Value
*Val
= S
->getValueOperand();
1124 const Value
*Ptr
= S
->getPointerOperand();
1127 if (!isTypeLegal(Val
->getType(), VT
, /*AllowI1=*/true))
1130 unsigned Alignment
= S
->getAlignment();
1131 unsigned ABIAlignment
= DL
.getABITypeAlignment(Val
->getType());
1132 if (Alignment
== 0) // Ensure that codegen never sees alignment 0
1133 Alignment
= ABIAlignment
;
1134 bool Aligned
= Alignment
>= ABIAlignment
;
1137 if (!X86SelectAddress(Ptr
, AM
))
1140 return X86FastEmitStore(VT
, Val
, AM
, createMachineMemOperandFor(I
), Aligned
);
1143 /// X86SelectRet - Select and emit code to implement ret instructions.
1144 bool X86FastISel::X86SelectRet(const Instruction
*I
) {
1145 const ReturnInst
*Ret
= cast
<ReturnInst
>(I
);
1146 const Function
&F
= *I
->getParent()->getParent();
1147 const X86MachineFunctionInfo
*X86MFInfo
=
1148 FuncInfo
.MF
->getInfo
<X86MachineFunctionInfo
>();
1150 if (!FuncInfo
.CanLowerReturn
)
1153 if (TLI
.supportSwiftError() &&
1154 F
.getAttributes().hasAttrSomewhere(Attribute::SwiftError
))
1157 if (TLI
.supportSplitCSR(FuncInfo
.MF
))
1160 CallingConv::ID CC
= F
.getCallingConv();
1161 if (CC
!= CallingConv::C
&&
1162 CC
!= CallingConv::Fast
&&
1163 CC
!= CallingConv::X86_FastCall
&&
1164 CC
!= CallingConv::X86_StdCall
&&
1165 CC
!= CallingConv::X86_ThisCall
&&
1166 CC
!= CallingConv::X86_64_SysV
&&
1167 CC
!= CallingConv::Win64
)
1170 // Don't handle popping bytes if they don't fit the ret's immediate.
1171 if (!isUInt
<16>(X86MFInfo
->getBytesToPopOnReturn()))
1174 // fastcc with -tailcallopt is intended to provide a guaranteed
1175 // tail call optimization. Fastisel doesn't know how to do that.
1176 if (CC
== CallingConv::Fast
&& TM
.Options
.GuaranteedTailCallOpt
)
1179 // Let SDISel handle vararg functions.
1183 // Build a list of return value registers.
1184 SmallVector
<unsigned, 4> RetRegs
;
1186 if (Ret
->getNumOperands() > 0) {
1187 SmallVector
<ISD::OutputArg
, 4> Outs
;
1188 GetReturnInfo(CC
, F
.getReturnType(), F
.getAttributes(), Outs
, TLI
, DL
);
1190 // Analyze operands of the call, assigning locations to each operand.
1191 SmallVector
<CCValAssign
, 16> ValLocs
;
1192 CCState
CCInfo(CC
, F
.isVarArg(), *FuncInfo
.MF
, ValLocs
, I
->getContext());
1193 CCInfo
.AnalyzeReturn(Outs
, RetCC_X86
);
1195 const Value
*RV
= Ret
->getOperand(0);
1196 unsigned Reg
= getRegForValue(RV
);
1200 // Only handle a single return value for now.
1201 if (ValLocs
.size() != 1)
1204 CCValAssign
&VA
= ValLocs
[0];
1206 // Don't bother handling odd stuff for now.
1207 if (VA
.getLocInfo() != CCValAssign::Full
)
1209 // Only handle register returns for now.
1213 // The calling-convention tables for x87 returns don't tell
1215 if (VA
.getLocReg() == X86::FP0
|| VA
.getLocReg() == X86::FP1
)
1218 unsigned SrcReg
= Reg
+ VA
.getValNo();
1219 EVT SrcVT
= TLI
.getValueType(DL
, RV
->getType());
1220 EVT DstVT
= VA
.getValVT();
1221 // Special handling for extended integers.
1222 if (SrcVT
!= DstVT
) {
1223 if (SrcVT
!= MVT::i1
&& SrcVT
!= MVT::i8
&& SrcVT
!= MVT::i16
)
1226 if (!Outs
[0].Flags
.isZExt() && !Outs
[0].Flags
.isSExt())
1229 assert(DstVT
== MVT::i32
&& "X86 should always ext to i32");
1231 if (SrcVT
== MVT::i1
) {
1232 if (Outs
[0].Flags
.isSExt())
1234 SrcReg
= fastEmitZExtFromI1(MVT::i8
, SrcReg
, /*TODO: Kill=*/false);
1237 unsigned Op
= Outs
[0].Flags
.isZExt() ? ISD::ZERO_EXTEND
:
1239 SrcReg
= fastEmit_r(SrcVT
.getSimpleVT(), DstVT
.getSimpleVT(), Op
,
1240 SrcReg
, /*TODO: Kill=*/false);
1244 unsigned DstReg
= VA
.getLocReg();
1245 const TargetRegisterClass
*SrcRC
= MRI
.getRegClass(SrcReg
);
1246 // Avoid a cross-class copy. This is very unlikely.
1247 if (!SrcRC
->contains(DstReg
))
1249 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1250 TII
.get(TargetOpcode::COPY
), DstReg
).addReg(SrcReg
);
1252 // Add register to return instruction.
1253 RetRegs
.push_back(VA
.getLocReg());
1256 // Swift calling convention does not require we copy the sret argument
1257 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
1259 // All x86 ABIs require that for returning structs by value we copy
1260 // the sret argument into %rax/%eax (depending on ABI) for the return.
1261 // We saved the argument into a virtual register in the entry block,
1262 // so now we copy the value out and into %rax/%eax.
1263 if (F
.hasStructRetAttr() && CC
!= CallingConv::Swift
) {
1264 unsigned Reg
= X86MFInfo
->getSRetReturnReg();
1266 "SRetReturnReg should have been set in LowerFormalArguments()!");
1267 unsigned RetReg
= Subtarget
->isTarget64BitLP64() ? X86::RAX
: X86::EAX
;
1268 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1269 TII
.get(TargetOpcode::COPY
), RetReg
).addReg(Reg
);
1270 RetRegs
.push_back(RetReg
);
1273 // Now emit the RET.
1274 MachineInstrBuilder MIB
;
1275 if (X86MFInfo
->getBytesToPopOnReturn()) {
1276 MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1277 TII
.get(Subtarget
->is64Bit() ? X86::RETIQ
: X86::RETIL
))
1278 .addImm(X86MFInfo
->getBytesToPopOnReturn());
1280 MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1281 TII
.get(Subtarget
->is64Bit() ? X86::RETQ
: X86::RETL
));
1283 for (unsigned i
= 0, e
= RetRegs
.size(); i
!= e
; ++i
)
1284 MIB
.addReg(RetRegs
[i
], RegState::Implicit
);
1288 /// X86SelectLoad - Select and emit code to implement load instructions.
1290 bool X86FastISel::X86SelectLoad(const Instruction
*I
) {
1291 const LoadInst
*LI
= cast
<LoadInst
>(I
);
1293 // Atomic loads need special handling.
1297 const Value
*SV
= I
->getOperand(0);
1298 if (TLI
.supportSwiftError()) {
1299 // Swifterror values can come from either a function parameter with
1300 // swifterror attribute or an alloca with swifterror attribute.
1301 if (const Argument
*Arg
= dyn_cast
<Argument
>(SV
)) {
1302 if (Arg
->hasSwiftErrorAttr())
1306 if (const AllocaInst
*Alloca
= dyn_cast
<AllocaInst
>(SV
)) {
1307 if (Alloca
->isSwiftError())
1313 if (!isTypeLegal(LI
->getType(), VT
, /*AllowI1=*/true))
1316 const Value
*Ptr
= LI
->getPointerOperand();
1319 if (!X86SelectAddress(Ptr
, AM
))
1322 unsigned Alignment
= LI
->getAlignment();
1323 unsigned ABIAlignment
= DL
.getABITypeAlignment(LI
->getType());
1324 if (Alignment
== 0) // Ensure that codegen never sees alignment 0
1325 Alignment
= ABIAlignment
;
1327 unsigned ResultReg
= 0;
1328 if (!X86FastEmitLoad(VT
, AM
, createMachineMemOperandFor(LI
), ResultReg
,
1332 updateValueMap(I
, ResultReg
);
1336 static unsigned X86ChooseCmpOpcode(EVT VT
, const X86Subtarget
*Subtarget
) {
1337 bool HasAVX512
= Subtarget
->hasAVX512();
1338 bool HasAVX
= Subtarget
->hasAVX();
1339 bool X86ScalarSSEf32
= Subtarget
->hasSSE1();
1340 bool X86ScalarSSEf64
= Subtarget
->hasSSE2();
1342 switch (VT
.getSimpleVT().SimpleTy
) {
1344 case MVT::i8
: return X86::CMP8rr
;
1345 case MVT::i16
: return X86::CMP16rr
;
1346 case MVT::i32
: return X86::CMP32rr
;
1347 case MVT::i64
: return X86::CMP64rr
;
1349 return X86ScalarSSEf32
1350 ? (HasAVX512
? X86::VUCOMISSZrr
1351 : HasAVX
? X86::VUCOMISSrr
: X86::UCOMISSrr
)
1354 return X86ScalarSSEf64
1355 ? (HasAVX512
? X86::VUCOMISDZrr
1356 : HasAVX
? X86::VUCOMISDrr
: X86::UCOMISDrr
)
1361 /// If we have a comparison with RHS as the RHS of the comparison, return an
1362 /// opcode that works for the compare (e.g. CMP32ri) otherwise return 0.
1363 static unsigned X86ChooseCmpImmediateOpcode(EVT VT
, const ConstantInt
*RHSC
) {
1364 int64_t Val
= RHSC
->getSExtValue();
1365 switch (VT
.getSimpleVT().SimpleTy
) {
1366 // Otherwise, we can't fold the immediate into this comparison.
1373 return X86::CMP16ri8
;
1374 return X86::CMP16ri
;
1377 return X86::CMP32ri8
;
1378 return X86::CMP32ri
;
1381 return X86::CMP64ri8
;
1382 // 64-bit comparisons are only valid if the immediate fits in a 32-bit sext
1385 return X86::CMP64ri32
;
1390 bool X86FastISel::X86FastEmitCompare(const Value
*Op0
, const Value
*Op1
, EVT VT
,
1391 const DebugLoc
&CurDbgLoc
) {
1392 unsigned Op0Reg
= getRegForValue(Op0
);
1393 if (Op0Reg
== 0) return false;
1395 // Handle 'null' like i32/i64 0.
1396 if (isa
<ConstantPointerNull
>(Op1
))
1397 Op1
= Constant::getNullValue(DL
.getIntPtrType(Op0
->getContext()));
1399 // We have two options: compare with register or immediate. If the RHS of
1400 // the compare is an immediate that we can fold into this compare, use
1401 // CMPri, otherwise use CMPrr.
1402 if (const ConstantInt
*Op1C
= dyn_cast
<ConstantInt
>(Op1
)) {
1403 if (unsigned CompareImmOpc
= X86ChooseCmpImmediateOpcode(VT
, Op1C
)) {
1404 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, CurDbgLoc
, TII
.get(CompareImmOpc
))
1406 .addImm(Op1C
->getSExtValue());
1411 unsigned CompareOpc
= X86ChooseCmpOpcode(VT
, Subtarget
);
1412 if (CompareOpc
== 0) return false;
1414 unsigned Op1Reg
= getRegForValue(Op1
);
1415 if (Op1Reg
== 0) return false;
1416 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, CurDbgLoc
, TII
.get(CompareOpc
))
1423 bool X86FastISel::X86SelectCmp(const Instruction
*I
) {
1424 const CmpInst
*CI
= cast
<CmpInst
>(I
);
1427 if (!isTypeLegal(I
->getOperand(0)->getType(), VT
))
1430 // Try to optimize or fold the cmp.
1431 CmpInst::Predicate Predicate
= optimizeCmpPredicate(CI
);
1432 unsigned ResultReg
= 0;
1433 switch (Predicate
) {
1435 case CmpInst::FCMP_FALSE
: {
1436 ResultReg
= createResultReg(&X86::GR32RegClass
);
1437 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOV32r0
),
1439 ResultReg
= fastEmitInst_extractsubreg(MVT::i8
, ResultReg
, /*Kill=*/true,
1445 case CmpInst::FCMP_TRUE
: {
1446 ResultReg
= createResultReg(&X86::GR8RegClass
);
1447 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOV8ri
),
1448 ResultReg
).addImm(1);
1454 updateValueMap(I
, ResultReg
);
1458 const Value
*LHS
= CI
->getOperand(0);
1459 const Value
*RHS
= CI
->getOperand(1);
1461 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
1462 // We don't have to materialize a zero constant for this case and can just use
1463 // %x again on the RHS.
1464 if (Predicate
== CmpInst::FCMP_ORD
|| Predicate
== CmpInst::FCMP_UNO
) {
1465 const auto *RHSC
= dyn_cast
<ConstantFP
>(RHS
);
1466 if (RHSC
&& RHSC
->isNullValue())
1470 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1471 static const uint16_t SETFOpcTable
[2][3] = {
1472 { X86::COND_E
, X86::COND_NP
, X86::AND8rr
},
1473 { X86::COND_NE
, X86::COND_P
, X86::OR8rr
}
1475 const uint16_t *SETFOpc
= nullptr;
1476 switch (Predicate
) {
1478 case CmpInst::FCMP_OEQ
: SETFOpc
= &SETFOpcTable
[0][0]; break;
1479 case CmpInst::FCMP_UNE
: SETFOpc
= &SETFOpcTable
[1][0]; break;
1482 ResultReg
= createResultReg(&X86::GR8RegClass
);
1484 if (!X86FastEmitCompare(LHS
, RHS
, VT
, I
->getDebugLoc()))
1487 unsigned FlagReg1
= createResultReg(&X86::GR8RegClass
);
1488 unsigned FlagReg2
= createResultReg(&X86::GR8RegClass
);
1489 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SETCCr
),
1490 FlagReg1
).addImm(SETFOpc
[0]);
1491 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SETCCr
),
1492 FlagReg2
).addImm(SETFOpc
[1]);
1493 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(SETFOpc
[2]),
1494 ResultReg
).addReg(FlagReg1
).addReg(FlagReg2
);
1495 updateValueMap(I
, ResultReg
);
1501 std::tie(CC
, SwapArgs
) = X86::getX86ConditionCode(Predicate
);
1502 assert(CC
<= X86::LAST_VALID_COND
&& "Unexpected condition code.");
1505 std::swap(LHS
, RHS
);
1507 // Emit a compare of LHS/RHS.
1508 if (!X86FastEmitCompare(LHS
, RHS
, VT
, I
->getDebugLoc()))
1511 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SETCCr
),
1512 ResultReg
).addImm(CC
);
1513 updateValueMap(I
, ResultReg
);
1517 bool X86FastISel::X86SelectZExt(const Instruction
*I
) {
1518 EVT DstVT
= TLI
.getValueType(DL
, I
->getType());
1519 if (!TLI
.isTypeLegal(DstVT
))
1522 unsigned ResultReg
= getRegForValue(I
->getOperand(0));
1526 // Handle zero-extension from i1 to i8, which is common.
1527 MVT SrcVT
= TLI
.getSimpleValueType(DL
, I
->getOperand(0)->getType());
1528 if (SrcVT
== MVT::i1
) {
1529 // Set the high bits to zero.
1530 ResultReg
= fastEmitZExtFromI1(MVT::i8
, ResultReg
, /*TODO: Kill=*/false);
1537 if (DstVT
== MVT::i64
) {
1538 // Handle extension to 64-bits via sub-register shenanigans.
1541 switch (SrcVT
.SimpleTy
) {
1542 case MVT::i8
: MovInst
= X86::MOVZX32rr8
; break;
1543 case MVT::i16
: MovInst
= X86::MOVZX32rr16
; break;
1544 case MVT::i32
: MovInst
= X86::MOV32rr
; break;
1545 default: llvm_unreachable("Unexpected zext to i64 source type");
1548 unsigned Result32
= createResultReg(&X86::GR32RegClass
);
1549 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(MovInst
), Result32
)
1552 ResultReg
= createResultReg(&X86::GR64RegClass
);
1553 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(TargetOpcode::SUBREG_TO_REG
),
1555 .addImm(0).addReg(Result32
).addImm(X86::sub_32bit
);
1556 } else if (DstVT
== MVT::i16
) {
1557 // i8->i16 doesn't exist in the autogenerated isel table. Need to zero
1558 // extend to 32-bits and then extract down to 16-bits.
1559 unsigned Result32
= createResultReg(&X86::GR32RegClass
);
1560 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOVZX32rr8
),
1561 Result32
).addReg(ResultReg
);
1563 ResultReg
= fastEmitInst_extractsubreg(MVT::i16
, Result32
, /*Kill=*/true,
1565 } else if (DstVT
!= MVT::i8
) {
1566 ResultReg
= fastEmit_r(MVT::i8
, DstVT
.getSimpleVT(), ISD::ZERO_EXTEND
,
1567 ResultReg
, /*Kill=*/true);
1572 updateValueMap(I
, ResultReg
);
1576 bool X86FastISel::X86SelectSExt(const Instruction
*I
) {
1577 EVT DstVT
= TLI
.getValueType(DL
, I
->getType());
1578 if (!TLI
.isTypeLegal(DstVT
))
1581 unsigned ResultReg
= getRegForValue(I
->getOperand(0));
1585 // Handle sign-extension from i1 to i8.
1586 MVT SrcVT
= TLI
.getSimpleValueType(DL
, I
->getOperand(0)->getType());
1587 if (SrcVT
== MVT::i1
) {
1588 // Set the high bits to zero.
1589 unsigned ZExtReg
= fastEmitZExtFromI1(MVT::i8
, ResultReg
,
1590 /*TODO: Kill=*/false);
1594 // Negate the result to make an 8-bit sign extended value.
1595 ResultReg
= createResultReg(&X86::GR8RegClass
);
1596 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::NEG8r
),
1597 ResultReg
).addReg(ZExtReg
);
1602 if (DstVT
== MVT::i16
) {
1603 // i8->i16 doesn't exist in the autogenerated isel table. Need to sign
1604 // extend to 32-bits and then extract down to 16-bits.
1605 unsigned Result32
= createResultReg(&X86::GR32RegClass
);
1606 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOVSX32rr8
),
1607 Result32
).addReg(ResultReg
);
1609 ResultReg
= fastEmitInst_extractsubreg(MVT::i16
, Result32
, /*Kill=*/true,
1611 } else if (DstVT
!= MVT::i8
) {
1612 ResultReg
= fastEmit_r(MVT::i8
, DstVT
.getSimpleVT(), ISD::SIGN_EXTEND
,
1613 ResultReg
, /*Kill=*/true);
1618 updateValueMap(I
, ResultReg
);
1622 bool X86FastISel::X86SelectBranch(const Instruction
*I
) {
1623 // Unconditional branches are selected by tablegen-generated code.
1624 // Handle a conditional branch.
1625 const BranchInst
*BI
= cast
<BranchInst
>(I
);
1626 MachineBasicBlock
*TrueMBB
= FuncInfo
.MBBMap
[BI
->getSuccessor(0)];
1627 MachineBasicBlock
*FalseMBB
= FuncInfo
.MBBMap
[BI
->getSuccessor(1)];
1629 // Fold the common case of a conditional branch with a comparison
1630 // in the same block (values defined on other blocks may not have
1631 // initialized registers).
1633 if (const CmpInst
*CI
= dyn_cast
<CmpInst
>(BI
->getCondition())) {
1634 if (CI
->hasOneUse() && CI
->getParent() == I
->getParent()) {
1635 EVT VT
= TLI
.getValueType(DL
, CI
->getOperand(0)->getType());
1637 // Try to optimize or fold the cmp.
1638 CmpInst::Predicate Predicate
= optimizeCmpPredicate(CI
);
1639 switch (Predicate
) {
1641 case CmpInst::FCMP_FALSE
: fastEmitBranch(FalseMBB
, DbgLoc
); return true;
1642 case CmpInst::FCMP_TRUE
: fastEmitBranch(TrueMBB
, DbgLoc
); return true;
1645 const Value
*CmpLHS
= CI
->getOperand(0);
1646 const Value
*CmpRHS
= CI
->getOperand(1);
1648 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x,
1650 // We don't have to materialize a zero constant for this case and can just
1651 // use %x again on the RHS.
1652 if (Predicate
== CmpInst::FCMP_ORD
|| Predicate
== CmpInst::FCMP_UNO
) {
1653 const auto *CmpRHSC
= dyn_cast
<ConstantFP
>(CmpRHS
);
1654 if (CmpRHSC
&& CmpRHSC
->isNullValue())
1658 // Try to take advantage of fallthrough opportunities.
1659 if (FuncInfo
.MBB
->isLayoutSuccessor(TrueMBB
)) {
1660 std::swap(TrueMBB
, FalseMBB
);
1661 Predicate
= CmpInst::getInversePredicate(Predicate
);
1664 // FCMP_OEQ and FCMP_UNE cannot be expressed with a single flag/condition
1665 // code check. Instead two branch instructions are required to check all
1666 // the flags. First we change the predicate to a supported condition code,
1667 // which will be the first branch. Later one we will emit the second
1669 bool NeedExtraBranch
= false;
1670 switch (Predicate
) {
1672 case CmpInst::FCMP_OEQ
:
1673 std::swap(TrueMBB
, FalseMBB
);
1675 case CmpInst::FCMP_UNE
:
1676 NeedExtraBranch
= true;
1677 Predicate
= CmpInst::FCMP_ONE
;
1682 std::tie(CC
, SwapArgs
) = X86::getX86ConditionCode(Predicate
);
1683 assert(CC
<= X86::LAST_VALID_COND
&& "Unexpected condition code.");
1686 std::swap(CmpLHS
, CmpRHS
);
1688 // Emit a compare of the LHS and RHS, setting the flags.
1689 if (!X86FastEmitCompare(CmpLHS
, CmpRHS
, VT
, CI
->getDebugLoc()))
1692 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::JCC_1
))
1693 .addMBB(TrueMBB
).addImm(CC
);
1695 // X86 requires a second branch to handle UNE (and OEQ, which is mapped
1697 if (NeedExtraBranch
) {
1698 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::JCC_1
))
1699 .addMBB(TrueMBB
).addImm(X86::COND_P
);
1702 finishCondBranch(BI
->getParent(), TrueMBB
, FalseMBB
);
1705 } else if (TruncInst
*TI
= dyn_cast
<TruncInst
>(BI
->getCondition())) {
1706 // Handle things like "%cond = trunc i32 %X to i1 / br i1 %cond", which
1707 // typically happen for _Bool and C++ bools.
1709 if (TI
->hasOneUse() && TI
->getParent() == I
->getParent() &&
1710 isTypeLegal(TI
->getOperand(0)->getType(), SourceVT
)) {
1711 unsigned TestOpc
= 0;
1712 switch (SourceVT
.SimpleTy
) {
1714 case MVT::i8
: TestOpc
= X86::TEST8ri
; break;
1715 case MVT::i16
: TestOpc
= X86::TEST16ri
; break;
1716 case MVT::i32
: TestOpc
= X86::TEST32ri
; break;
1717 case MVT::i64
: TestOpc
= X86::TEST64ri32
; break;
1720 unsigned OpReg
= getRegForValue(TI
->getOperand(0));
1721 if (OpReg
== 0) return false;
1723 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(TestOpc
))
1724 .addReg(OpReg
).addImm(1);
1726 unsigned JmpCond
= X86::COND_NE
;
1727 if (FuncInfo
.MBB
->isLayoutSuccessor(TrueMBB
)) {
1728 std::swap(TrueMBB
, FalseMBB
);
1729 JmpCond
= X86::COND_E
;
1732 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::JCC_1
))
1733 .addMBB(TrueMBB
).addImm(JmpCond
);
1735 finishCondBranch(BI
->getParent(), TrueMBB
, FalseMBB
);
1739 } else if (foldX86XALUIntrinsic(CC
, BI
, BI
->getCondition())) {
1740 // Fake request the condition, otherwise the intrinsic might be completely
1742 unsigned TmpReg
= getRegForValue(BI
->getCondition());
1746 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::JCC_1
))
1747 .addMBB(TrueMBB
).addImm(CC
);
1748 finishCondBranch(BI
->getParent(), TrueMBB
, FalseMBB
);
1752 // Otherwise do a clumsy setcc and re-test it.
1753 // Note that i1 essentially gets ANY_EXTEND'ed to i8 where it isn't used
1754 // in an explicit cast, so make sure to handle that correctly.
1755 unsigned OpReg
= getRegForValue(BI
->getCondition());
1756 if (OpReg
== 0) return false;
1758 // In case OpReg is a K register, COPY to a GPR
1759 if (MRI
.getRegClass(OpReg
) == &X86::VK1RegClass
) {
1760 unsigned KOpReg
= OpReg
;
1761 OpReg
= createResultReg(&X86::GR32RegClass
);
1762 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1763 TII
.get(TargetOpcode::COPY
), OpReg
)
1765 OpReg
= fastEmitInst_extractsubreg(MVT::i8
, OpReg
, /*Kill=*/true,
1768 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::TEST8ri
))
1771 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::JCC_1
))
1772 .addMBB(TrueMBB
).addImm(X86::COND_NE
);
1773 finishCondBranch(BI
->getParent(), TrueMBB
, FalseMBB
);
1777 bool X86FastISel::X86SelectShift(const Instruction
*I
) {
1778 unsigned CReg
= 0, OpReg
= 0;
1779 const TargetRegisterClass
*RC
= nullptr;
1780 if (I
->getType()->isIntegerTy(8)) {
1782 RC
= &X86::GR8RegClass
;
1783 switch (I
->getOpcode()) {
1784 case Instruction::LShr
: OpReg
= X86::SHR8rCL
; break;
1785 case Instruction::AShr
: OpReg
= X86::SAR8rCL
; break;
1786 case Instruction::Shl
: OpReg
= X86::SHL8rCL
; break;
1787 default: return false;
1789 } else if (I
->getType()->isIntegerTy(16)) {
1791 RC
= &X86::GR16RegClass
;
1792 switch (I
->getOpcode()) {
1793 default: llvm_unreachable("Unexpected shift opcode");
1794 case Instruction::LShr
: OpReg
= X86::SHR16rCL
; break;
1795 case Instruction::AShr
: OpReg
= X86::SAR16rCL
; break;
1796 case Instruction::Shl
: OpReg
= X86::SHL16rCL
; break;
1798 } else if (I
->getType()->isIntegerTy(32)) {
1800 RC
= &X86::GR32RegClass
;
1801 switch (I
->getOpcode()) {
1802 default: llvm_unreachable("Unexpected shift opcode");
1803 case Instruction::LShr
: OpReg
= X86::SHR32rCL
; break;
1804 case Instruction::AShr
: OpReg
= X86::SAR32rCL
; break;
1805 case Instruction::Shl
: OpReg
= X86::SHL32rCL
; break;
1807 } else if (I
->getType()->isIntegerTy(64)) {
1809 RC
= &X86::GR64RegClass
;
1810 switch (I
->getOpcode()) {
1811 default: llvm_unreachable("Unexpected shift opcode");
1812 case Instruction::LShr
: OpReg
= X86::SHR64rCL
; break;
1813 case Instruction::AShr
: OpReg
= X86::SAR64rCL
; break;
1814 case Instruction::Shl
: OpReg
= X86::SHL64rCL
; break;
1821 if (!isTypeLegal(I
->getType(), VT
))
1824 unsigned Op0Reg
= getRegForValue(I
->getOperand(0));
1825 if (Op0Reg
== 0) return false;
1827 unsigned Op1Reg
= getRegForValue(I
->getOperand(1));
1828 if (Op1Reg
== 0) return false;
1829 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(TargetOpcode::COPY
),
1830 CReg
).addReg(Op1Reg
);
1832 // The shift instruction uses X86::CL. If we defined a super-register
1833 // of X86::CL, emit a subreg KILL to precisely describe what we're doing here.
1834 if (CReg
!= X86::CL
)
1835 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1836 TII
.get(TargetOpcode::KILL
), X86::CL
)
1837 .addReg(CReg
, RegState::Kill
);
1839 unsigned ResultReg
= createResultReg(RC
);
1840 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(OpReg
), ResultReg
)
1842 updateValueMap(I
, ResultReg
);
1846 bool X86FastISel::X86SelectDivRem(const Instruction
*I
) {
1847 const static unsigned NumTypes
= 4; // i8, i16, i32, i64
1848 const static unsigned NumOps
= 4; // SDiv, SRem, UDiv, URem
1849 const static bool S
= true; // IsSigned
1850 const static bool U
= false; // !IsSigned
1851 const static unsigned Copy
= TargetOpcode::COPY
;
1852 // For the X86 DIV/IDIV instruction, in most cases the dividend
1853 // (numerator) must be in a specific register pair highreg:lowreg,
1854 // producing the quotient in lowreg and the remainder in highreg.
1855 // For most data types, to set up the instruction, the dividend is
1856 // copied into lowreg, and lowreg is sign-extended or zero-extended
1857 // into highreg. The exception is i8, where the dividend is defined
1858 // as a single register rather than a register pair, and we
1859 // therefore directly sign-extend or zero-extend the dividend into
1860 // lowreg, instead of copying, and ignore the highreg.
1861 const static struct DivRemEntry
{
1862 // The following portion depends only on the data type.
1863 const TargetRegisterClass
*RC
;
1864 unsigned LowInReg
; // low part of the register pair
1865 unsigned HighInReg
; // high part of the register pair
1866 // The following portion depends on both the data type and the operation.
1867 struct DivRemResult
{
1868 unsigned OpDivRem
; // The specific DIV/IDIV opcode to use.
1869 unsigned OpSignExtend
; // Opcode for sign-extending lowreg into
1870 // highreg, or copying a zero into highreg.
1871 unsigned OpCopy
; // Opcode for copying dividend into lowreg, or
1872 // zero/sign-extending into lowreg for i8.
1873 unsigned DivRemResultReg
; // Register containing the desired result.
1874 bool IsOpSigned
; // Whether to use signed or unsigned form.
1875 } ResultTable
[NumOps
];
1876 } OpTable
[NumTypes
] = {
1877 { &X86::GR8RegClass
, X86::AX
, 0, {
1878 { X86::IDIV8r
, 0, X86::MOVSX16rr8
, X86::AL
, S
}, // SDiv
1879 { X86::IDIV8r
, 0, X86::MOVSX16rr8
, X86::AH
, S
}, // SRem
1880 { X86::DIV8r
, 0, X86::MOVZX16rr8
, X86::AL
, U
}, // UDiv
1881 { X86::DIV8r
, 0, X86::MOVZX16rr8
, X86::AH
, U
}, // URem
1884 { &X86::GR16RegClass
, X86::AX
, X86::DX
, {
1885 { X86::IDIV16r
, X86::CWD
, Copy
, X86::AX
, S
}, // SDiv
1886 { X86::IDIV16r
, X86::CWD
, Copy
, X86::DX
, S
}, // SRem
1887 { X86::DIV16r
, X86::MOV32r0
, Copy
, X86::AX
, U
}, // UDiv
1888 { X86::DIV16r
, X86::MOV32r0
, Copy
, X86::DX
, U
}, // URem
1891 { &X86::GR32RegClass
, X86::EAX
, X86::EDX
, {
1892 { X86::IDIV32r
, X86::CDQ
, Copy
, X86::EAX
, S
}, // SDiv
1893 { X86::IDIV32r
, X86::CDQ
, Copy
, X86::EDX
, S
}, // SRem
1894 { X86::DIV32r
, X86::MOV32r0
, Copy
, X86::EAX
, U
}, // UDiv
1895 { X86::DIV32r
, X86::MOV32r0
, Copy
, X86::EDX
, U
}, // URem
1898 { &X86::GR64RegClass
, X86::RAX
, X86::RDX
, {
1899 { X86::IDIV64r
, X86::CQO
, Copy
, X86::RAX
, S
}, // SDiv
1900 { X86::IDIV64r
, X86::CQO
, Copy
, X86::RDX
, S
}, // SRem
1901 { X86::DIV64r
, X86::MOV32r0
, Copy
, X86::RAX
, U
}, // UDiv
1902 { X86::DIV64r
, X86::MOV32r0
, Copy
, X86::RDX
, U
}, // URem
1908 if (!isTypeLegal(I
->getType(), VT
))
1911 unsigned TypeIndex
, OpIndex
;
1912 switch (VT
.SimpleTy
) {
1913 default: return false;
1914 case MVT::i8
: TypeIndex
= 0; break;
1915 case MVT::i16
: TypeIndex
= 1; break;
1916 case MVT::i32
: TypeIndex
= 2; break;
1917 case MVT::i64
: TypeIndex
= 3;
1918 if (!Subtarget
->is64Bit())
1923 switch (I
->getOpcode()) {
1924 default: llvm_unreachable("Unexpected div/rem opcode");
1925 case Instruction::SDiv
: OpIndex
= 0; break;
1926 case Instruction::SRem
: OpIndex
= 1; break;
1927 case Instruction::UDiv
: OpIndex
= 2; break;
1928 case Instruction::URem
: OpIndex
= 3; break;
1931 const DivRemEntry
&TypeEntry
= OpTable
[TypeIndex
];
1932 const DivRemEntry::DivRemResult
&OpEntry
= TypeEntry
.ResultTable
[OpIndex
];
1933 unsigned Op0Reg
= getRegForValue(I
->getOperand(0));
1936 unsigned Op1Reg
= getRegForValue(I
->getOperand(1));
1940 // Move op0 into low-order input register.
1941 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1942 TII
.get(OpEntry
.OpCopy
), TypeEntry
.LowInReg
).addReg(Op0Reg
);
1943 // Zero-extend or sign-extend into high-order input register.
1944 if (OpEntry
.OpSignExtend
) {
1945 if (OpEntry
.IsOpSigned
)
1946 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1947 TII
.get(OpEntry
.OpSignExtend
));
1949 unsigned Zero32
= createResultReg(&X86::GR32RegClass
);
1950 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1951 TII
.get(X86::MOV32r0
), Zero32
);
1953 // Copy the zero into the appropriate sub/super/identical physical
1954 // register. Unfortunately the operations needed are not uniform enough
1955 // to fit neatly into the table above.
1956 if (VT
== MVT::i16
) {
1957 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1958 TII
.get(Copy
), TypeEntry
.HighInReg
)
1959 .addReg(Zero32
, 0, X86::sub_16bit
);
1960 } else if (VT
== MVT::i32
) {
1961 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1962 TII
.get(Copy
), TypeEntry
.HighInReg
)
1964 } else if (VT
== MVT::i64
) {
1965 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1966 TII
.get(TargetOpcode::SUBREG_TO_REG
), TypeEntry
.HighInReg
)
1967 .addImm(0).addReg(Zero32
).addImm(X86::sub_32bit
);
1971 // Generate the DIV/IDIV instruction.
1972 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1973 TII
.get(OpEntry
.OpDivRem
)).addReg(Op1Reg
);
1974 // For i8 remainder, we can't reference ah directly, as we'll end
1975 // up with bogus copies like %r9b = COPY %ah. Reference ax
1976 // instead to prevent ah references in a rex instruction.
1978 // The current assumption of the fast register allocator is that isel
1979 // won't generate explicit references to the GR8_NOREX registers. If
1980 // the allocator and/or the backend get enhanced to be more robust in
1981 // that regard, this can be, and should be, removed.
1982 unsigned ResultReg
= 0;
1983 if ((I
->getOpcode() == Instruction::SRem
||
1984 I
->getOpcode() == Instruction::URem
) &&
1985 OpEntry
.DivRemResultReg
== X86::AH
&& Subtarget
->is64Bit()) {
1986 unsigned SourceSuperReg
= createResultReg(&X86::GR16RegClass
);
1987 unsigned ResultSuperReg
= createResultReg(&X86::GR16RegClass
);
1988 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
1989 TII
.get(Copy
), SourceSuperReg
).addReg(X86::AX
);
1991 // Shift AX right by 8 bits instead of using AH.
1992 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SHR16ri
),
1993 ResultSuperReg
).addReg(SourceSuperReg
).addImm(8);
1995 // Now reference the 8-bit subreg of the result.
1996 ResultReg
= fastEmitInst_extractsubreg(MVT::i8
, ResultSuperReg
,
1997 /*Kill=*/true, X86::sub_8bit
);
1999 // Copy the result out of the physreg if we haven't already.
2001 ResultReg
= createResultReg(TypeEntry
.RC
);
2002 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Copy
), ResultReg
)
2003 .addReg(OpEntry
.DivRemResultReg
);
2005 updateValueMap(I
, ResultReg
);
2010 /// Emit a conditional move instruction (if the are supported) to lower
2012 bool X86FastISel::X86FastEmitCMoveSelect(MVT RetVT
, const Instruction
*I
) {
2013 // Check if the subtarget supports these instructions.
2014 if (!Subtarget
->hasCMov())
2017 // FIXME: Add support for i8.
2018 if (RetVT
< MVT::i16
|| RetVT
> MVT::i64
)
2021 const Value
*Cond
= I
->getOperand(0);
2022 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(RetVT
);
2023 bool NeedTest
= true;
2024 X86::CondCode CC
= X86::COND_NE
;
2026 // Optimize conditions coming from a compare if both instructions are in the
2027 // same basic block (values defined in other basic blocks may not have
2028 // initialized registers).
2029 const auto *CI
= dyn_cast
<CmpInst
>(Cond
);
2030 if (CI
&& (CI
->getParent() == I
->getParent())) {
2031 CmpInst::Predicate Predicate
= optimizeCmpPredicate(CI
);
2033 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
2034 static const uint16_t SETFOpcTable
[2][3] = {
2035 { X86::COND_NP
, X86::COND_E
, X86::TEST8rr
},
2036 { X86::COND_P
, X86::COND_NE
, X86::OR8rr
}
2038 const uint16_t *SETFOpc
= nullptr;
2039 switch (Predicate
) {
2041 case CmpInst::FCMP_OEQ
:
2042 SETFOpc
= &SETFOpcTable
[0][0];
2043 Predicate
= CmpInst::ICMP_NE
;
2045 case CmpInst::FCMP_UNE
:
2046 SETFOpc
= &SETFOpcTable
[1][0];
2047 Predicate
= CmpInst::ICMP_NE
;
2052 std::tie(CC
, NeedSwap
) = X86::getX86ConditionCode(Predicate
);
2053 assert(CC
<= X86::LAST_VALID_COND
&& "Unexpected condition code.");
2055 const Value
*CmpLHS
= CI
->getOperand(0);
2056 const Value
*CmpRHS
= CI
->getOperand(1);
2058 std::swap(CmpLHS
, CmpRHS
);
2060 EVT CmpVT
= TLI
.getValueType(DL
, CmpLHS
->getType());
2061 // Emit a compare of the LHS and RHS, setting the flags.
2062 if (!X86FastEmitCompare(CmpLHS
, CmpRHS
, CmpVT
, CI
->getDebugLoc()))
2066 unsigned FlagReg1
= createResultReg(&X86::GR8RegClass
);
2067 unsigned FlagReg2
= createResultReg(&X86::GR8RegClass
);
2068 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SETCCr
),
2069 FlagReg1
).addImm(SETFOpc
[0]);
2070 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SETCCr
),
2071 FlagReg2
).addImm(SETFOpc
[1]);
2072 auto const &II
= TII
.get(SETFOpc
[2]);
2073 if (II
.getNumDefs()) {
2074 unsigned TmpReg
= createResultReg(&X86::GR8RegClass
);
2075 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, II
, TmpReg
)
2076 .addReg(FlagReg2
).addReg(FlagReg1
);
2078 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, II
)
2079 .addReg(FlagReg2
).addReg(FlagReg1
);
2083 } else if (foldX86XALUIntrinsic(CC
, I
, Cond
)) {
2084 // Fake request the condition, otherwise the intrinsic might be completely
2086 unsigned TmpReg
= getRegForValue(Cond
);
2094 // Selects operate on i1, however, CondReg is 8 bits width and may contain
2095 // garbage. Indeed, only the less significant bit is supposed to be
2096 // accurate. If we read more than the lsb, we may see non-zero values
2097 // whereas lsb is zero. Therefore, we have to truncate Op0Reg to i1 for
2098 // the select. This is achieved by performing TEST against 1.
2099 unsigned CondReg
= getRegForValue(Cond
);
2102 bool CondIsKill
= hasTrivialKill(Cond
);
2104 // In case OpReg is a K register, COPY to a GPR
2105 if (MRI
.getRegClass(CondReg
) == &X86::VK1RegClass
) {
2106 unsigned KCondReg
= CondReg
;
2107 CondReg
= createResultReg(&X86::GR32RegClass
);
2108 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2109 TII
.get(TargetOpcode::COPY
), CondReg
)
2110 .addReg(KCondReg
, getKillRegState(CondIsKill
));
2111 CondReg
= fastEmitInst_extractsubreg(MVT::i8
, CondReg
, /*Kill=*/true,
2114 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::TEST8ri
))
2115 .addReg(CondReg
, getKillRegState(CondIsKill
))
2119 const Value
*LHS
= I
->getOperand(1);
2120 const Value
*RHS
= I
->getOperand(2);
2122 unsigned RHSReg
= getRegForValue(RHS
);
2123 bool RHSIsKill
= hasTrivialKill(RHS
);
2125 unsigned LHSReg
= getRegForValue(LHS
);
2126 bool LHSIsKill
= hasTrivialKill(LHS
);
2128 if (!LHSReg
|| !RHSReg
)
2131 const TargetRegisterInfo
&TRI
= *Subtarget
->getRegisterInfo();
2132 unsigned Opc
= X86::getCMovOpcode(TRI
.getRegSizeInBits(*RC
)/8);
2133 unsigned ResultReg
= fastEmitInst_rri(Opc
, RC
, RHSReg
, RHSIsKill
,
2134 LHSReg
, LHSIsKill
, CC
);
2135 updateValueMap(I
, ResultReg
);
2139 /// Emit SSE or AVX instructions to lower the select.
2141 /// Try to use SSE1/SSE2 instructions to simulate a select without branches.
2142 /// This lowers fp selects into a CMP/AND/ANDN/OR sequence when the necessary
2143 /// SSE instructions are available. If AVX is available, try to use a VBLENDV.
2144 bool X86FastISel::X86FastEmitSSESelect(MVT RetVT
, const Instruction
*I
) {
2145 // Optimize conditions coming from a compare if both instructions are in the
2146 // same basic block (values defined in other basic blocks may not have
2147 // initialized registers).
2148 const auto *CI
= dyn_cast
<FCmpInst
>(I
->getOperand(0));
2149 if (!CI
|| (CI
->getParent() != I
->getParent()))
2152 if (I
->getType() != CI
->getOperand(0)->getType() ||
2153 !((Subtarget
->hasSSE1() && RetVT
== MVT::f32
) ||
2154 (Subtarget
->hasSSE2() && RetVT
== MVT::f64
)))
2157 const Value
*CmpLHS
= CI
->getOperand(0);
2158 const Value
*CmpRHS
= CI
->getOperand(1);
2159 CmpInst::Predicate Predicate
= optimizeCmpPredicate(CI
);
2161 // The optimizer might have replaced fcmp oeq %x, %x with fcmp ord %x, 0.0.
2162 // We don't have to materialize a zero constant for this case and can just use
2163 // %x again on the RHS.
2164 if (Predicate
== CmpInst::FCMP_ORD
|| Predicate
== CmpInst::FCMP_UNO
) {
2165 const auto *CmpRHSC
= dyn_cast
<ConstantFP
>(CmpRHS
);
2166 if (CmpRHSC
&& CmpRHSC
->isNullValue())
2172 std::tie(CC
, NeedSwap
) = getX86SSEConditionCode(Predicate
);
2173 if (CC
> 7 && !Subtarget
->hasAVX())
2177 std::swap(CmpLHS
, CmpRHS
);
2179 const Value
*LHS
= I
->getOperand(1);
2180 const Value
*RHS
= I
->getOperand(2);
2182 unsigned LHSReg
= getRegForValue(LHS
);
2183 bool LHSIsKill
= hasTrivialKill(LHS
);
2185 unsigned RHSReg
= getRegForValue(RHS
);
2186 bool RHSIsKill
= hasTrivialKill(RHS
);
2188 unsigned CmpLHSReg
= getRegForValue(CmpLHS
);
2189 bool CmpLHSIsKill
= hasTrivialKill(CmpLHS
);
2191 unsigned CmpRHSReg
= getRegForValue(CmpRHS
);
2192 bool CmpRHSIsKill
= hasTrivialKill(CmpRHS
);
2194 if (!LHSReg
|| !RHSReg
|| !CmpLHS
|| !CmpRHS
)
2197 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(RetVT
);
2200 if (Subtarget
->hasAVX512()) {
2201 // If we have AVX512 we can use a mask compare and masked movss/sd.
2202 const TargetRegisterClass
*VR128X
= &X86::VR128XRegClass
;
2203 const TargetRegisterClass
*VK1
= &X86::VK1RegClass
;
2205 unsigned CmpOpcode
=
2206 (RetVT
== MVT::f32
) ? X86::VCMPSSZrr
: X86::VCMPSDZrr
;
2207 unsigned CmpReg
= fastEmitInst_rri(CmpOpcode
, VK1
, CmpLHSReg
, CmpLHSIsKill
,
2208 CmpRHSReg
, CmpRHSIsKill
, CC
);
2210 // Need an IMPLICIT_DEF for the input that is used to generate the upper
2211 // bits of the result register since its not based on any of the inputs.
2212 unsigned ImplicitDefReg
= createResultReg(VR128X
);
2213 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2214 TII
.get(TargetOpcode::IMPLICIT_DEF
), ImplicitDefReg
);
2216 // Place RHSReg is the passthru of the masked movss/sd operation and put
2217 // LHS in the input. The mask input comes from the compare.
2218 unsigned MovOpcode
=
2219 (RetVT
== MVT::f32
) ? X86::VMOVSSZrrk
: X86::VMOVSDZrrk
;
2220 unsigned MovReg
= fastEmitInst_rrrr(MovOpcode
, VR128X
, RHSReg
, RHSIsKill
,
2221 CmpReg
, true, ImplicitDefReg
, true,
2224 ResultReg
= createResultReg(RC
);
2225 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2226 TII
.get(TargetOpcode::COPY
), ResultReg
).addReg(MovReg
);
2228 } else if (Subtarget
->hasAVX()) {
2229 const TargetRegisterClass
*VR128
= &X86::VR128RegClass
;
2231 // If we have AVX, create 1 blendv instead of 3 logic instructions.
2232 // Blendv was introduced with SSE 4.1, but the 2 register form implicitly
2233 // uses XMM0 as the selection register. That may need just as many
2234 // instructions as the AND/ANDN/OR sequence due to register moves, so
2236 unsigned CmpOpcode
=
2237 (RetVT
== MVT::f32
) ? X86::VCMPSSrr
: X86::VCMPSDrr
;
2238 unsigned BlendOpcode
=
2239 (RetVT
== MVT::f32
) ? X86::VBLENDVPSrr
: X86::VBLENDVPDrr
;
2241 unsigned CmpReg
= fastEmitInst_rri(CmpOpcode
, RC
, CmpLHSReg
, CmpLHSIsKill
,
2242 CmpRHSReg
, CmpRHSIsKill
, CC
);
2243 unsigned VBlendReg
= fastEmitInst_rrr(BlendOpcode
, VR128
, RHSReg
, RHSIsKill
,
2244 LHSReg
, LHSIsKill
, CmpReg
, true);
2245 ResultReg
= createResultReg(RC
);
2246 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2247 TII
.get(TargetOpcode::COPY
), ResultReg
).addReg(VBlendReg
);
2249 // Choose the SSE instruction sequence based on data type (float or double).
2250 static const uint16_t OpcTable
[2][4] = {
2251 { X86::CMPSSrr
, X86::ANDPSrr
, X86::ANDNPSrr
, X86::ORPSrr
},
2252 { X86::CMPSDrr
, X86::ANDPDrr
, X86::ANDNPDrr
, X86::ORPDrr
}
2255 const uint16_t *Opc
= nullptr;
2256 switch (RetVT
.SimpleTy
) {
2257 default: return false;
2258 case MVT::f32
: Opc
= &OpcTable
[0][0]; break;
2259 case MVT::f64
: Opc
= &OpcTable
[1][0]; break;
2262 const TargetRegisterClass
*VR128
= &X86::VR128RegClass
;
2263 unsigned CmpReg
= fastEmitInst_rri(Opc
[0], RC
, CmpLHSReg
, CmpLHSIsKill
,
2264 CmpRHSReg
, CmpRHSIsKill
, CC
);
2265 unsigned AndReg
= fastEmitInst_rr(Opc
[1], VR128
, CmpReg
, /*IsKill=*/false,
2267 unsigned AndNReg
= fastEmitInst_rr(Opc
[2], VR128
, CmpReg
, /*IsKill=*/true,
2269 unsigned OrReg
= fastEmitInst_rr(Opc
[3], VR128
, AndNReg
, /*IsKill=*/true,
2270 AndReg
, /*IsKill=*/true);
2271 ResultReg
= createResultReg(RC
);
2272 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2273 TII
.get(TargetOpcode::COPY
), ResultReg
).addReg(OrReg
);
2275 updateValueMap(I
, ResultReg
);
2279 bool X86FastISel::X86FastEmitPseudoSelect(MVT RetVT
, const Instruction
*I
) {
2280 // These are pseudo CMOV instructions and will be later expanded into control-
2283 switch (RetVT
.SimpleTy
) {
2284 default: return false;
2285 case MVT::i8
: Opc
= X86::CMOV_GR8
; break;
2286 case MVT::i16
: Opc
= X86::CMOV_GR16
; break;
2287 case MVT::i32
: Opc
= X86::CMOV_GR32
; break;
2288 case MVT::f32
: Opc
= Subtarget
->hasAVX512() ? X86::CMOV_FR32X
2289 : X86::CMOV_FR32
; break;
2290 case MVT::f64
: Opc
= Subtarget
->hasAVX512() ? X86::CMOV_FR64X
2291 : X86::CMOV_FR64
; break;
2294 const Value
*Cond
= I
->getOperand(0);
2295 X86::CondCode CC
= X86::COND_NE
;
2297 // Optimize conditions coming from a compare if both instructions are in the
2298 // same basic block (values defined in other basic blocks may not have
2299 // initialized registers).
2300 const auto *CI
= dyn_cast
<CmpInst
>(Cond
);
2301 if (CI
&& (CI
->getParent() == I
->getParent())) {
2303 std::tie(CC
, NeedSwap
) = X86::getX86ConditionCode(CI
->getPredicate());
2304 if (CC
> X86::LAST_VALID_COND
)
2307 const Value
*CmpLHS
= CI
->getOperand(0);
2308 const Value
*CmpRHS
= CI
->getOperand(1);
2311 std::swap(CmpLHS
, CmpRHS
);
2313 EVT CmpVT
= TLI
.getValueType(DL
, CmpLHS
->getType());
2314 if (!X86FastEmitCompare(CmpLHS
, CmpRHS
, CmpVT
, CI
->getDebugLoc()))
2317 unsigned CondReg
= getRegForValue(Cond
);
2320 bool CondIsKill
= hasTrivialKill(Cond
);
2322 // In case OpReg is a K register, COPY to a GPR
2323 if (MRI
.getRegClass(CondReg
) == &X86::VK1RegClass
) {
2324 unsigned KCondReg
= CondReg
;
2325 CondReg
= createResultReg(&X86::GR32RegClass
);
2326 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2327 TII
.get(TargetOpcode::COPY
), CondReg
)
2328 .addReg(KCondReg
, getKillRegState(CondIsKill
));
2329 CondReg
= fastEmitInst_extractsubreg(MVT::i8
, CondReg
, /*Kill=*/true,
2332 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::TEST8ri
))
2333 .addReg(CondReg
, getKillRegState(CondIsKill
))
2337 const Value
*LHS
= I
->getOperand(1);
2338 const Value
*RHS
= I
->getOperand(2);
2340 unsigned LHSReg
= getRegForValue(LHS
);
2341 bool LHSIsKill
= hasTrivialKill(LHS
);
2343 unsigned RHSReg
= getRegForValue(RHS
);
2344 bool RHSIsKill
= hasTrivialKill(RHS
);
2346 if (!LHSReg
|| !RHSReg
)
2349 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(RetVT
);
2351 unsigned ResultReg
=
2352 fastEmitInst_rri(Opc
, RC
, RHSReg
, RHSIsKill
, LHSReg
, LHSIsKill
, CC
);
2353 updateValueMap(I
, ResultReg
);
2357 bool X86FastISel::X86SelectSelect(const Instruction
*I
) {
2359 if (!isTypeLegal(I
->getType(), RetVT
))
2362 // Check if we can fold the select.
2363 if (const auto *CI
= dyn_cast
<CmpInst
>(I
->getOperand(0))) {
2364 CmpInst::Predicate Predicate
= optimizeCmpPredicate(CI
);
2365 const Value
*Opnd
= nullptr;
2366 switch (Predicate
) {
2368 case CmpInst::FCMP_FALSE
: Opnd
= I
->getOperand(2); break;
2369 case CmpInst::FCMP_TRUE
: Opnd
= I
->getOperand(1); break;
2371 // No need for a select anymore - this is an unconditional move.
2373 unsigned OpReg
= getRegForValue(Opnd
);
2376 bool OpIsKill
= hasTrivialKill(Opnd
);
2377 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(RetVT
);
2378 unsigned ResultReg
= createResultReg(RC
);
2379 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2380 TII
.get(TargetOpcode::COPY
), ResultReg
)
2381 .addReg(OpReg
, getKillRegState(OpIsKill
));
2382 updateValueMap(I
, ResultReg
);
2387 // First try to use real conditional move instructions.
2388 if (X86FastEmitCMoveSelect(RetVT
, I
))
2391 // Try to use a sequence of SSE instructions to simulate a conditional move.
2392 if (X86FastEmitSSESelect(RetVT
, I
))
2395 // Fall-back to pseudo conditional move instructions, which will be later
2396 // converted to control-flow.
2397 if (X86FastEmitPseudoSelect(RetVT
, I
))
2403 // Common code for X86SelectSIToFP and X86SelectUIToFP.
2404 bool X86FastISel::X86SelectIntToFP(const Instruction
*I
, bool IsSigned
) {
2405 // The target-independent selection algorithm in FastISel already knows how
2406 // to select a SINT_TO_FP if the target is SSE but not AVX.
2407 // Early exit if the subtarget doesn't have AVX.
2408 // Unsigned conversion requires avx512.
2409 bool HasAVX512
= Subtarget
->hasAVX512();
2410 if (!Subtarget
->hasAVX() || (!IsSigned
&& !HasAVX512
))
2413 // TODO: We could sign extend narrower types.
2414 MVT SrcVT
= TLI
.getSimpleValueType(DL
, I
->getOperand(0)->getType());
2415 if (SrcVT
!= MVT::i32
&& SrcVT
!= MVT::i64
)
2418 // Select integer to float/double conversion.
2419 unsigned OpReg
= getRegForValue(I
->getOperand(0));
2425 static const uint16_t SCvtOpc
[2][2][2] = {
2426 { { X86::VCVTSI2SSrr
, X86::VCVTSI642SSrr
},
2427 { X86::VCVTSI2SDrr
, X86::VCVTSI642SDrr
} },
2428 { { X86::VCVTSI2SSZrr
, X86::VCVTSI642SSZrr
},
2429 { X86::VCVTSI2SDZrr
, X86::VCVTSI642SDZrr
} },
2431 static const uint16_t UCvtOpc
[2][2] = {
2432 { X86::VCVTUSI2SSZrr
, X86::VCVTUSI642SSZrr
},
2433 { X86::VCVTUSI2SDZrr
, X86::VCVTUSI642SDZrr
},
2435 bool Is64Bit
= SrcVT
== MVT::i64
;
2437 if (I
->getType()->isDoubleTy()) {
2438 // s/uitofp int -> double
2439 Opcode
= IsSigned
? SCvtOpc
[HasAVX512
][1][Is64Bit
] : UCvtOpc
[1][Is64Bit
];
2440 } else if (I
->getType()->isFloatTy()) {
2441 // s/uitofp int -> float
2442 Opcode
= IsSigned
? SCvtOpc
[HasAVX512
][0][Is64Bit
] : UCvtOpc
[0][Is64Bit
];
2446 MVT DstVT
= TLI
.getValueType(DL
, I
->getType()).getSimpleVT();
2447 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(DstVT
);
2448 unsigned ImplicitDefReg
= createResultReg(RC
);
2449 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2450 TII
.get(TargetOpcode::IMPLICIT_DEF
), ImplicitDefReg
);
2451 unsigned ResultReg
=
2452 fastEmitInst_rr(Opcode
, RC
, ImplicitDefReg
, true, OpReg
, false);
2453 updateValueMap(I
, ResultReg
);
2457 bool X86FastISel::X86SelectSIToFP(const Instruction
*I
) {
2458 return X86SelectIntToFP(I
, /*IsSigned*/true);
2461 bool X86FastISel::X86SelectUIToFP(const Instruction
*I
) {
2462 return X86SelectIntToFP(I
, /*IsSigned*/false);
2465 // Helper method used by X86SelectFPExt and X86SelectFPTrunc.
2466 bool X86FastISel::X86SelectFPExtOrFPTrunc(const Instruction
*I
,
2468 const TargetRegisterClass
*RC
) {
2469 assert((I
->getOpcode() == Instruction::FPExt
||
2470 I
->getOpcode() == Instruction::FPTrunc
) &&
2471 "Instruction must be an FPExt or FPTrunc!");
2472 bool HasAVX
= Subtarget
->hasAVX();
2474 unsigned OpReg
= getRegForValue(I
->getOperand(0));
2478 unsigned ImplicitDefReg
;
2480 ImplicitDefReg
= createResultReg(RC
);
2481 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2482 TII
.get(TargetOpcode::IMPLICIT_DEF
), ImplicitDefReg
);
2486 unsigned ResultReg
= createResultReg(RC
);
2487 MachineInstrBuilder MIB
;
2488 MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(TargetOpc
),
2492 MIB
.addReg(ImplicitDefReg
);
2495 updateValueMap(I
, ResultReg
);
2499 bool X86FastISel::X86SelectFPExt(const Instruction
*I
) {
2500 if (X86ScalarSSEf64
&& I
->getType()->isDoubleTy() &&
2501 I
->getOperand(0)->getType()->isFloatTy()) {
2502 bool HasAVX512
= Subtarget
->hasAVX512();
2503 // fpext from float to double.
2505 HasAVX512
? X86::VCVTSS2SDZrr
2506 : Subtarget
->hasAVX() ? X86::VCVTSS2SDrr
: X86::CVTSS2SDrr
;
2507 return X86SelectFPExtOrFPTrunc(I
, Opc
, TLI
.getRegClassFor(MVT::f64
));
2513 bool X86FastISel::X86SelectFPTrunc(const Instruction
*I
) {
2514 if (X86ScalarSSEf64
&& I
->getType()->isFloatTy() &&
2515 I
->getOperand(0)->getType()->isDoubleTy()) {
2516 bool HasAVX512
= Subtarget
->hasAVX512();
2517 // fptrunc from double to float.
2519 HasAVX512
? X86::VCVTSD2SSZrr
2520 : Subtarget
->hasAVX() ? X86::VCVTSD2SSrr
: X86::CVTSD2SSrr
;
2521 return X86SelectFPExtOrFPTrunc(I
, Opc
, TLI
.getRegClassFor(MVT::f32
));
2527 bool X86FastISel::X86SelectTrunc(const Instruction
*I
) {
2528 EVT SrcVT
= TLI
.getValueType(DL
, I
->getOperand(0)->getType());
2529 EVT DstVT
= TLI
.getValueType(DL
, I
->getType());
2531 // This code only handles truncation to byte.
2532 if (DstVT
!= MVT::i8
&& DstVT
!= MVT::i1
)
2534 if (!TLI
.isTypeLegal(SrcVT
))
2537 unsigned InputReg
= getRegForValue(I
->getOperand(0));
2539 // Unhandled operand. Halt "fast" selection and bail.
2542 if (SrcVT
== MVT::i8
) {
2543 // Truncate from i8 to i1; no code needed.
2544 updateValueMap(I
, InputReg
);
2548 // Issue an extract_subreg.
2549 unsigned ResultReg
= fastEmitInst_extractsubreg(MVT::i8
,
2555 updateValueMap(I
, ResultReg
);
2559 bool X86FastISel::IsMemcpySmall(uint64_t Len
) {
2560 return Len
<= (Subtarget
->is64Bit() ? 32 : 16);
2563 bool X86FastISel::TryEmitSmallMemcpy(X86AddressMode DestAM
,
2564 X86AddressMode SrcAM
, uint64_t Len
) {
2566 // Make sure we don't bloat code by inlining very large memcpy's.
2567 if (!IsMemcpySmall(Len
))
2570 bool i64Legal
= Subtarget
->is64Bit();
2572 // We don't care about alignment here since we just emit integer accesses.
2575 if (Len
>= 8 && i64Legal
)
2585 bool RV
= X86FastEmitLoad(VT
, SrcAM
, nullptr, Reg
);
2586 RV
&= X86FastEmitStore(VT
, Reg
, /*Kill=*/true, DestAM
);
2587 assert(RV
&& "Failed to emit load or store??");
2589 unsigned Size
= VT
.getSizeInBits()/8;
2591 DestAM
.Disp
+= Size
;
2598 bool X86FastISel::fastLowerIntrinsicCall(const IntrinsicInst
*II
) {
2599 // FIXME: Handle more intrinsics.
2600 switch (II
->getIntrinsicID()) {
2601 default: return false;
2602 case Intrinsic::convert_from_fp16
:
2603 case Intrinsic::convert_to_fp16
: {
2604 if (Subtarget
->useSoftFloat() || !Subtarget
->hasF16C())
2607 const Value
*Op
= II
->getArgOperand(0);
2608 unsigned InputReg
= getRegForValue(Op
);
2612 // F16C only allows converting from float to half and from half to float.
2613 bool IsFloatToHalf
= II
->getIntrinsicID() == Intrinsic::convert_to_fp16
;
2614 if (IsFloatToHalf
) {
2615 if (!Op
->getType()->isFloatTy())
2618 if (!II
->getType()->isFloatTy())
2622 unsigned ResultReg
= 0;
2623 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(MVT::v8i16
);
2624 if (IsFloatToHalf
) {
2625 // 'InputReg' is implicitly promoted from register class FR32 to
2626 // register class VR128 by method 'constrainOperandRegClass' which is
2627 // directly called by 'fastEmitInst_ri'.
2628 // Instruction VCVTPS2PHrr takes an extra immediate operand which is
2629 // used to provide rounding control: use MXCSR.RC, encoded as 0b100.
2630 // It's consistent with the other FP instructions, which are usually
2631 // controlled by MXCSR.
2632 InputReg
= fastEmitInst_ri(X86::VCVTPS2PHrr
, RC
, InputReg
, false, 4);
2634 // Move the lower 32-bits of ResultReg to another register of class GR32.
2635 ResultReg
= createResultReg(&X86::GR32RegClass
);
2636 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2637 TII
.get(X86::VMOVPDI2DIrr
), ResultReg
)
2638 .addReg(InputReg
, RegState::Kill
);
2640 // The result value is in the lower 16-bits of ResultReg.
2641 unsigned RegIdx
= X86::sub_16bit
;
2642 ResultReg
= fastEmitInst_extractsubreg(MVT::i16
, ResultReg
, true, RegIdx
);
2644 assert(Op
->getType()->isIntegerTy(16) && "Expected a 16-bit integer!");
2645 // Explicitly sign-extend the input to 32-bit.
2646 InputReg
= fastEmit_r(MVT::i16
, MVT::i32
, ISD::SIGN_EXTEND
, InputReg
,
2649 // The following SCALAR_TO_VECTOR will be expanded into a VMOVDI2PDIrr.
2650 InputReg
= fastEmit_r(MVT::i32
, MVT::v4i32
, ISD::SCALAR_TO_VECTOR
,
2651 InputReg
, /*Kill=*/true);
2653 InputReg
= fastEmitInst_r(X86::VCVTPH2PSrr
, RC
, InputReg
, /*Kill=*/true);
2655 // The result value is in the lower 32-bits of ResultReg.
2656 // Emit an explicit copy from register class VR128 to register class FR32.
2657 ResultReg
= createResultReg(&X86::FR32RegClass
);
2658 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2659 TII
.get(TargetOpcode::COPY
), ResultReg
)
2660 .addReg(InputReg
, RegState::Kill
);
2663 updateValueMap(II
, ResultReg
);
2666 case Intrinsic::frameaddress
: {
2667 MachineFunction
*MF
= FuncInfo
.MF
;
2668 if (MF
->getTarget().getMCAsmInfo()->usesWindowsCFI())
2671 Type
*RetTy
= II
->getCalledFunction()->getReturnType();
2674 if (!isTypeLegal(RetTy
, VT
))
2678 const TargetRegisterClass
*RC
= nullptr;
2680 switch (VT
.SimpleTy
) {
2681 default: llvm_unreachable("Invalid result type for frameaddress.");
2682 case MVT::i32
: Opc
= X86::MOV32rm
; RC
= &X86::GR32RegClass
; break;
2683 case MVT::i64
: Opc
= X86::MOV64rm
; RC
= &X86::GR64RegClass
; break;
2686 // This needs to be set before we call getPtrSizedFrameRegister, otherwise
2687 // we get the wrong frame register.
2688 MachineFrameInfo
&MFI
= MF
->getFrameInfo();
2689 MFI
.setFrameAddressIsTaken(true);
2691 const X86RegisterInfo
*RegInfo
= Subtarget
->getRegisterInfo();
2692 unsigned FrameReg
= RegInfo
->getPtrSizedFrameRegister(*MF
);
2693 assert(((FrameReg
== X86::RBP
&& VT
== MVT::i64
) ||
2694 (FrameReg
== X86::EBP
&& VT
== MVT::i32
)) &&
2695 "Invalid Frame Register!");
2697 // Always make a copy of the frame register to a vreg first, so that we
2698 // never directly reference the frame register (the TwoAddressInstruction-
2699 // Pass doesn't like that).
2700 unsigned SrcReg
= createResultReg(RC
);
2701 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2702 TII
.get(TargetOpcode::COPY
), SrcReg
).addReg(FrameReg
);
2704 // Now recursively load from the frame address.
2705 // movq (%rbp), %rax
2706 // movq (%rax), %rax
2707 // movq (%rax), %rax
2710 unsigned Depth
= cast
<ConstantInt
>(II
->getOperand(0))->getZExtValue();
2712 DestReg
= createResultReg(RC
);
2713 addDirectMem(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2714 TII
.get(Opc
), DestReg
), SrcReg
);
2718 updateValueMap(II
, SrcReg
);
2721 case Intrinsic::memcpy
: {
2722 const MemCpyInst
*MCI
= cast
<MemCpyInst
>(II
);
2723 // Don't handle volatile or variable length memcpys.
2724 if (MCI
->isVolatile())
2727 if (isa
<ConstantInt
>(MCI
->getLength())) {
2728 // Small memcpy's are common enough that we want to do them
2729 // without a call if possible.
2730 uint64_t Len
= cast
<ConstantInt
>(MCI
->getLength())->getZExtValue();
2731 if (IsMemcpySmall(Len
)) {
2732 X86AddressMode DestAM
, SrcAM
;
2733 if (!X86SelectAddress(MCI
->getRawDest(), DestAM
) ||
2734 !X86SelectAddress(MCI
->getRawSource(), SrcAM
))
2736 TryEmitSmallMemcpy(DestAM
, SrcAM
, Len
);
2741 unsigned SizeWidth
= Subtarget
->is64Bit() ? 64 : 32;
2742 if (!MCI
->getLength()->getType()->isIntegerTy(SizeWidth
))
2745 if (MCI
->getSourceAddressSpace() > 255 || MCI
->getDestAddressSpace() > 255)
2748 return lowerCallTo(II
, "memcpy", II
->getNumArgOperands() - 1);
2750 case Intrinsic::memset
: {
2751 const MemSetInst
*MSI
= cast
<MemSetInst
>(II
);
2753 if (MSI
->isVolatile())
2756 unsigned SizeWidth
= Subtarget
->is64Bit() ? 64 : 32;
2757 if (!MSI
->getLength()->getType()->isIntegerTy(SizeWidth
))
2760 if (MSI
->getDestAddressSpace() > 255)
2763 return lowerCallTo(II
, "memset", II
->getNumArgOperands() - 1);
2765 case Intrinsic::stackprotector
: {
2766 // Emit code to store the stack guard onto the stack.
2767 EVT PtrTy
= TLI
.getPointerTy(DL
);
2769 const Value
*Op1
= II
->getArgOperand(0); // The guard's value.
2770 const AllocaInst
*Slot
= cast
<AllocaInst
>(II
->getArgOperand(1));
2772 MFI
.setStackProtectorIndex(FuncInfo
.StaticAllocaMap
[Slot
]);
2774 // Grab the frame index.
2776 if (!X86SelectAddress(Slot
, AM
)) return false;
2777 if (!X86FastEmitStore(PtrTy
, Op1
, AM
)) return false;
2780 case Intrinsic::dbg_declare
: {
2781 const DbgDeclareInst
*DI
= cast
<DbgDeclareInst
>(II
);
2783 assert(DI
->getAddress() && "Null address should be checked earlier!");
2784 if (!X86SelectAddress(DI
->getAddress(), AM
))
2786 const MCInstrDesc
&II
= TII
.get(TargetOpcode::DBG_VALUE
);
2787 // FIXME may need to add RegState::Debug to any registers produced,
2788 // although ESP/EBP should be the only ones at the moment.
2789 assert(DI
->getVariable()->isValidLocationForIntrinsic(DbgLoc
) &&
2790 "Expected inlined-at fields to agree");
2791 addFullAddress(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, II
), AM
)
2793 .addMetadata(DI
->getVariable())
2794 .addMetadata(DI
->getExpression());
2797 case Intrinsic::trap
: {
2798 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::TRAP
));
2801 case Intrinsic::sqrt
: {
2802 if (!Subtarget
->hasSSE1())
2805 Type
*RetTy
= II
->getCalledFunction()->getReturnType();
2808 if (!isTypeLegal(RetTy
, VT
))
2811 // Unfortunately we can't use fastEmit_r, because the AVX version of FSQRT
2812 // is not generated by FastISel yet.
2813 // FIXME: Update this code once tablegen can handle it.
2814 static const uint16_t SqrtOpc
[3][2] = {
2815 { X86::SQRTSSr
, X86::SQRTSDr
},
2816 { X86::VSQRTSSr
, X86::VSQRTSDr
},
2817 { X86::VSQRTSSZr
, X86::VSQRTSDZr
},
2819 unsigned AVXLevel
= Subtarget
->hasAVX512() ? 2 :
2820 Subtarget
->hasAVX() ? 1 :
2823 switch (VT
.SimpleTy
) {
2824 default: return false;
2825 case MVT::f32
: Opc
= SqrtOpc
[AVXLevel
][0]; break;
2826 case MVT::f64
: Opc
= SqrtOpc
[AVXLevel
][1]; break;
2829 const Value
*SrcVal
= II
->getArgOperand(0);
2830 unsigned SrcReg
= getRegForValue(SrcVal
);
2835 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(VT
);
2836 unsigned ImplicitDefReg
= 0;
2838 ImplicitDefReg
= createResultReg(RC
);
2839 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2840 TII
.get(TargetOpcode::IMPLICIT_DEF
), ImplicitDefReg
);
2843 unsigned ResultReg
= createResultReg(RC
);
2844 MachineInstrBuilder MIB
;
2845 MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Opc
),
2849 MIB
.addReg(ImplicitDefReg
);
2853 updateValueMap(II
, ResultReg
);
2856 case Intrinsic::sadd_with_overflow
:
2857 case Intrinsic::uadd_with_overflow
:
2858 case Intrinsic::ssub_with_overflow
:
2859 case Intrinsic::usub_with_overflow
:
2860 case Intrinsic::smul_with_overflow
:
2861 case Intrinsic::umul_with_overflow
: {
2862 // This implements the basic lowering of the xalu with overflow intrinsics
2863 // into add/sub/mul followed by either seto or setb.
2864 const Function
*Callee
= II
->getCalledFunction();
2865 auto *Ty
= cast
<StructType
>(Callee
->getReturnType());
2866 Type
*RetTy
= Ty
->getTypeAtIndex(0U);
2867 assert(Ty
->getTypeAtIndex(1)->isIntegerTy() &&
2868 Ty
->getTypeAtIndex(1)->getScalarSizeInBits() == 1 &&
2869 "Overflow value expected to be an i1");
2872 if (!isTypeLegal(RetTy
, VT
))
2875 if (VT
< MVT::i8
|| VT
> MVT::i64
)
2878 const Value
*LHS
= II
->getArgOperand(0);
2879 const Value
*RHS
= II
->getArgOperand(1);
2881 // Canonicalize immediate to the RHS.
2882 if (isa
<ConstantInt
>(LHS
) && !isa
<ConstantInt
>(RHS
) &&
2883 isCommutativeIntrinsic(II
))
2884 std::swap(LHS
, RHS
);
2886 unsigned BaseOpc
, CondCode
;
2887 switch (II
->getIntrinsicID()) {
2888 default: llvm_unreachable("Unexpected intrinsic!");
2889 case Intrinsic::sadd_with_overflow
:
2890 BaseOpc
= ISD::ADD
; CondCode
= X86::COND_O
; break;
2891 case Intrinsic::uadd_with_overflow
:
2892 BaseOpc
= ISD::ADD
; CondCode
= X86::COND_B
; break;
2893 case Intrinsic::ssub_with_overflow
:
2894 BaseOpc
= ISD::SUB
; CondCode
= X86::COND_O
; break;
2895 case Intrinsic::usub_with_overflow
:
2896 BaseOpc
= ISD::SUB
; CondCode
= X86::COND_B
; break;
2897 case Intrinsic::smul_with_overflow
:
2898 BaseOpc
= X86ISD::SMUL
; CondCode
= X86::COND_O
; break;
2899 case Intrinsic::umul_with_overflow
:
2900 BaseOpc
= X86ISD::UMUL
; CondCode
= X86::COND_O
; break;
2903 unsigned LHSReg
= getRegForValue(LHS
);
2906 bool LHSIsKill
= hasTrivialKill(LHS
);
2908 unsigned ResultReg
= 0;
2909 // Check if we have an immediate version.
2910 if (const auto *CI
= dyn_cast
<ConstantInt
>(RHS
)) {
2911 static const uint16_t Opc
[2][4] = {
2912 { X86::INC8r
, X86::INC16r
, X86::INC32r
, X86::INC64r
},
2913 { X86::DEC8r
, X86::DEC16r
, X86::DEC32r
, X86::DEC64r
}
2916 if (CI
->isOne() && (BaseOpc
== ISD::ADD
|| BaseOpc
== ISD::SUB
) &&
2917 CondCode
== X86::COND_O
) {
2918 // We can use INC/DEC.
2919 ResultReg
= createResultReg(TLI
.getRegClassFor(VT
));
2920 bool IsDec
= BaseOpc
== ISD::SUB
;
2921 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2922 TII
.get(Opc
[IsDec
][VT
.SimpleTy
-MVT::i8
]), ResultReg
)
2923 .addReg(LHSReg
, getKillRegState(LHSIsKill
));
2925 ResultReg
= fastEmit_ri(VT
, VT
, BaseOpc
, LHSReg
, LHSIsKill
,
2926 CI
->getZExtValue());
2932 RHSReg
= getRegForValue(RHS
);
2935 RHSIsKill
= hasTrivialKill(RHS
);
2936 ResultReg
= fastEmit_rr(VT
, VT
, BaseOpc
, LHSReg
, LHSIsKill
, RHSReg
,
2940 // FastISel doesn't have a pattern for all X86::MUL*r and X86::IMUL*r. Emit
2942 if (BaseOpc
== X86ISD::UMUL
&& !ResultReg
) {
2943 static const uint16_t MULOpc
[] =
2944 { X86::MUL8r
, X86::MUL16r
, X86::MUL32r
, X86::MUL64r
};
2945 static const MCPhysReg Reg
[] = { X86::AL
, X86::AX
, X86::EAX
, X86::RAX
};
2946 // First copy the first operand into RAX, which is an implicit input to
2947 // the X86::MUL*r instruction.
2948 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2949 TII
.get(TargetOpcode::COPY
), Reg
[VT
.SimpleTy
-MVT::i8
])
2950 .addReg(LHSReg
, getKillRegState(LHSIsKill
));
2951 ResultReg
= fastEmitInst_r(MULOpc
[VT
.SimpleTy
-MVT::i8
],
2952 TLI
.getRegClassFor(VT
), RHSReg
, RHSIsKill
);
2953 } else if (BaseOpc
== X86ISD::SMUL
&& !ResultReg
) {
2954 static const uint16_t MULOpc
[] =
2955 { X86::IMUL8r
, X86::IMUL16rr
, X86::IMUL32rr
, X86::IMUL64rr
};
2956 if (VT
== MVT::i8
) {
2957 // Copy the first operand into AL, which is an implicit input to the
2958 // X86::IMUL8r instruction.
2959 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
2960 TII
.get(TargetOpcode::COPY
), X86::AL
)
2961 .addReg(LHSReg
, getKillRegState(LHSIsKill
));
2962 ResultReg
= fastEmitInst_r(MULOpc
[0], TLI
.getRegClassFor(VT
), RHSReg
,
2965 ResultReg
= fastEmitInst_rr(MULOpc
[VT
.SimpleTy
-MVT::i8
],
2966 TLI
.getRegClassFor(VT
), LHSReg
, LHSIsKill
,
2973 // Assign to a GPR since the overflow return value is lowered to a SETcc.
2974 unsigned ResultReg2
= createResultReg(&X86::GR8RegClass
);
2975 assert((ResultReg
+1) == ResultReg2
&& "Nonconsecutive result registers.");
2976 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::SETCCr
),
2977 ResultReg2
).addImm(CondCode
);
2979 updateValueMap(II
, ResultReg
, 2);
2982 case Intrinsic::x86_sse_cvttss2si
:
2983 case Intrinsic::x86_sse_cvttss2si64
:
2984 case Intrinsic::x86_sse2_cvttsd2si
:
2985 case Intrinsic::x86_sse2_cvttsd2si64
: {
2987 switch (II
->getIntrinsicID()) {
2988 default: llvm_unreachable("Unexpected intrinsic.");
2989 case Intrinsic::x86_sse_cvttss2si
:
2990 case Intrinsic::x86_sse_cvttss2si64
:
2991 if (!Subtarget
->hasSSE1())
2993 IsInputDouble
= false;
2995 case Intrinsic::x86_sse2_cvttsd2si
:
2996 case Intrinsic::x86_sse2_cvttsd2si64
:
2997 if (!Subtarget
->hasSSE2())
2999 IsInputDouble
= true;
3003 Type
*RetTy
= II
->getCalledFunction()->getReturnType();
3005 if (!isTypeLegal(RetTy
, VT
))
3008 static const uint16_t CvtOpc
[3][2][2] = {
3009 { { X86::CVTTSS2SIrr
, X86::CVTTSS2SI64rr
},
3010 { X86::CVTTSD2SIrr
, X86::CVTTSD2SI64rr
} },
3011 { { X86::VCVTTSS2SIrr
, X86::VCVTTSS2SI64rr
},
3012 { X86::VCVTTSD2SIrr
, X86::VCVTTSD2SI64rr
} },
3013 { { X86::VCVTTSS2SIZrr
, X86::VCVTTSS2SI64Zrr
},
3014 { X86::VCVTTSD2SIZrr
, X86::VCVTTSD2SI64Zrr
} },
3016 unsigned AVXLevel
= Subtarget
->hasAVX512() ? 2 :
3017 Subtarget
->hasAVX() ? 1 :
3020 switch (VT
.SimpleTy
) {
3021 default: llvm_unreachable("Unexpected result type.");
3022 case MVT::i32
: Opc
= CvtOpc
[AVXLevel
][IsInputDouble
][0]; break;
3023 case MVT::i64
: Opc
= CvtOpc
[AVXLevel
][IsInputDouble
][1]; break;
3026 // Check if we can fold insertelement instructions into the convert.
3027 const Value
*Op
= II
->getArgOperand(0);
3028 while (auto *IE
= dyn_cast
<InsertElementInst
>(Op
)) {
3029 const Value
*Index
= IE
->getOperand(2);
3030 if (!isa
<ConstantInt
>(Index
))
3032 unsigned Idx
= cast
<ConstantInt
>(Index
)->getZExtValue();
3035 Op
= IE
->getOperand(1);
3038 Op
= IE
->getOperand(0);
3041 unsigned Reg
= getRegForValue(Op
);
3045 unsigned ResultReg
= createResultReg(TLI
.getRegClassFor(VT
));
3046 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Opc
), ResultReg
)
3049 updateValueMap(II
, ResultReg
);
3055 bool X86FastISel::fastLowerArguments() {
3056 if (!FuncInfo
.CanLowerReturn
)
3059 const Function
*F
= FuncInfo
.Fn
;
3063 CallingConv::ID CC
= F
->getCallingConv();
3064 if (CC
!= CallingConv::C
)
3067 if (Subtarget
->isCallingConvWin64(CC
))
3070 if (!Subtarget
->is64Bit())
3073 if (Subtarget
->useSoftFloat())
3076 // Only handle simple cases. i.e. Up to 6 i32/i64 scalar arguments.
3077 unsigned GPRCnt
= 0;
3078 unsigned FPRCnt
= 0;
3079 for (auto const &Arg
: F
->args()) {
3080 if (Arg
.hasAttribute(Attribute::ByVal
) ||
3081 Arg
.hasAttribute(Attribute::InReg
) ||
3082 Arg
.hasAttribute(Attribute::StructRet
) ||
3083 Arg
.hasAttribute(Attribute::SwiftSelf
) ||
3084 Arg
.hasAttribute(Attribute::SwiftError
) ||
3085 Arg
.hasAttribute(Attribute::Nest
))
3088 Type
*ArgTy
= Arg
.getType();
3089 if (ArgTy
->isStructTy() || ArgTy
->isArrayTy() || ArgTy
->isVectorTy())
3092 EVT ArgVT
= TLI
.getValueType(DL
, ArgTy
);
3093 if (!ArgVT
.isSimple()) return false;
3094 switch (ArgVT
.getSimpleVT().SimpleTy
) {
3095 default: return false;
3102 if (!Subtarget
->hasSSE1())
3115 static const MCPhysReg GPR32ArgRegs
[] = {
3116 X86::EDI
, X86::ESI
, X86::EDX
, X86::ECX
, X86::R8D
, X86::R9D
3118 static const MCPhysReg GPR64ArgRegs
[] = {
3119 X86::RDI
, X86::RSI
, X86::RDX
, X86::RCX
, X86::R8
, X86::R9
3121 static const MCPhysReg XMMArgRegs
[] = {
3122 X86::XMM0
, X86::XMM1
, X86::XMM2
, X86::XMM3
,
3123 X86::XMM4
, X86::XMM5
, X86::XMM6
, X86::XMM7
3126 unsigned GPRIdx
= 0;
3127 unsigned FPRIdx
= 0;
3128 for (auto const &Arg
: F
->args()) {
3129 MVT VT
= TLI
.getSimpleValueType(DL
, Arg
.getType());
3130 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(VT
);
3132 switch (VT
.SimpleTy
) {
3133 default: llvm_unreachable("Unexpected value type.");
3134 case MVT::i32
: SrcReg
= GPR32ArgRegs
[GPRIdx
++]; break;
3135 case MVT::i64
: SrcReg
= GPR64ArgRegs
[GPRIdx
++]; break;
3136 case MVT::f32
: LLVM_FALLTHROUGH
;
3137 case MVT::f64
: SrcReg
= XMMArgRegs
[FPRIdx
++]; break;
3139 unsigned DstReg
= FuncInfo
.MF
->addLiveIn(SrcReg
, RC
);
3140 // FIXME: Unfortunately it's necessary to emit a copy from the livein copy.
3141 // Without this, EmitLiveInCopies may eliminate the livein if its only
3142 // use is a bitcast (which isn't turned into an instruction).
3143 unsigned ResultReg
= createResultReg(RC
);
3144 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3145 TII
.get(TargetOpcode::COPY
), ResultReg
)
3146 .addReg(DstReg
, getKillRegState(true));
3147 updateValueMap(&Arg
, ResultReg
);
3152 static unsigned computeBytesPoppedByCalleeForSRet(const X86Subtarget
*Subtarget
,
3154 ImmutableCallSite
*CS
) {
3155 if (Subtarget
->is64Bit())
3157 if (Subtarget
->getTargetTriple().isOSMSVCRT())
3159 if (CC
== CallingConv::Fast
|| CC
== CallingConv::GHC
||
3160 CC
== CallingConv::HiPE
)
3164 if (CS
->arg_empty() || !CS
->paramHasAttr(0, Attribute::StructRet
) ||
3165 CS
->paramHasAttr(0, Attribute::InReg
) || Subtarget
->isTargetMCU())
3171 bool X86FastISel::fastLowerCall(CallLoweringInfo
&CLI
) {
3172 auto &OutVals
= CLI
.OutVals
;
3173 auto &OutFlags
= CLI
.OutFlags
;
3174 auto &OutRegs
= CLI
.OutRegs
;
3175 auto &Ins
= CLI
.Ins
;
3176 auto &InRegs
= CLI
.InRegs
;
3177 CallingConv::ID CC
= CLI
.CallConv
;
3178 bool &IsTailCall
= CLI
.IsTailCall
;
3179 bool IsVarArg
= CLI
.IsVarArg
;
3180 const Value
*Callee
= CLI
.Callee
;
3181 MCSymbol
*Symbol
= CLI
.Symbol
;
3183 bool Is64Bit
= Subtarget
->is64Bit();
3184 bool IsWin64
= Subtarget
->isCallingConvWin64(CC
);
3186 const CallInst
*CI
=
3187 CLI
.CS
? dyn_cast
<CallInst
>(CLI
.CS
->getInstruction()) : nullptr;
3188 const Function
*CalledFn
= CI
? CI
->getCalledFunction() : nullptr;
3190 // Call / invoke instructions with NoCfCheck attribute require special
3193 CLI
.CS
? dyn_cast
<InvokeInst
>(CLI
.CS
->getInstruction()) : nullptr;
3194 if ((CI
&& CI
->doesNoCfCheck()) || (II
&& II
->doesNoCfCheck()))
3197 // Functions with no_caller_saved_registers that need special handling.
3198 if ((CI
&& CI
->hasFnAttr("no_caller_saved_registers")) ||
3199 (CalledFn
&& CalledFn
->hasFnAttribute("no_caller_saved_registers")))
3202 // Functions using retpoline for indirect calls need to use SDISel.
3203 if (Subtarget
->useRetpolineIndirectCalls())
3206 // Handle only C, fastcc, and webkit_js calling conventions for now.
3208 default: return false;
3209 case CallingConv::C
:
3210 case CallingConv::Fast
:
3211 case CallingConv::WebKit_JS
:
3212 case CallingConv::Swift
:
3213 case CallingConv::X86_FastCall
:
3214 case CallingConv::X86_StdCall
:
3215 case CallingConv::X86_ThisCall
:
3216 case CallingConv::Win64
:
3217 case CallingConv::X86_64_SysV
:
3221 // Allow SelectionDAG isel to handle tail calls.
3225 // fastcc with -tailcallopt is intended to provide a guaranteed
3226 // tail call optimization. Fastisel doesn't know how to do that.
3227 if (CC
== CallingConv::Fast
&& TM
.Options
.GuaranteedTailCallOpt
)
3230 // Don't know how to handle Win64 varargs yet. Nothing special needed for
3231 // x86-32. Special handling for x86-64 is implemented.
3232 if (IsVarArg
&& IsWin64
)
3235 // Don't know about inalloca yet.
3236 if (CLI
.CS
&& CLI
.CS
->hasInAllocaArgument())
3239 for (auto Flag
: CLI
.OutFlags
)
3240 if (Flag
.isSwiftError())
3243 SmallVector
<MVT
, 16> OutVTs
;
3244 SmallVector
<unsigned, 16> ArgRegs
;
3246 // If this is a constant i1/i8/i16 argument, promote to i32 to avoid an extra
3247 // instruction. This is safe because it is common to all FastISel supported
3248 // calling conventions on x86.
3249 for (int i
= 0, e
= OutVals
.size(); i
!= e
; ++i
) {
3250 Value
*&Val
= OutVals
[i
];
3251 ISD::ArgFlagsTy Flags
= OutFlags
[i
];
3252 if (auto *CI
= dyn_cast
<ConstantInt
>(Val
)) {
3253 if (CI
->getBitWidth() < 32) {
3255 Val
= ConstantExpr::getSExt(CI
, Type::getInt32Ty(CI
->getContext()));
3257 Val
= ConstantExpr::getZExt(CI
, Type::getInt32Ty(CI
->getContext()));
3261 // Passing bools around ends up doing a trunc to i1 and passing it.
3262 // Codegen this as an argument + "and 1".
3264 auto *TI
= dyn_cast
<TruncInst
>(Val
);
3266 if (TI
&& TI
->getType()->isIntegerTy(1) && CLI
.CS
&&
3267 (TI
->getParent() == CLI
.CS
->getInstruction()->getParent()) &&
3269 Value
*PrevVal
= TI
->getOperand(0);
3270 ResultReg
= getRegForValue(PrevVal
);
3275 if (!isTypeLegal(PrevVal
->getType(), VT
))
3279 fastEmit_ri(VT
, VT
, ISD::AND
, ResultReg
, hasTrivialKill(PrevVal
), 1);
3281 if (!isTypeLegal(Val
->getType(), VT
))
3283 ResultReg
= getRegForValue(Val
);
3289 ArgRegs
.push_back(ResultReg
);
3290 OutVTs
.push_back(VT
);
3293 // Analyze operands of the call, assigning locations to each operand.
3294 SmallVector
<CCValAssign
, 16> ArgLocs
;
3295 CCState
CCInfo(CC
, IsVarArg
, *FuncInfo
.MF
, ArgLocs
, CLI
.RetTy
->getContext());
3297 // Allocate shadow area for Win64
3299 CCInfo
.AllocateStack(32, 8);
3301 CCInfo
.AnalyzeCallOperands(OutVTs
, OutFlags
, CC_X86
);
3303 // Get a count of how many bytes are to be pushed on the stack.
3304 unsigned NumBytes
= CCInfo
.getAlignedCallFrameSize();
3306 // Issue CALLSEQ_START
3307 unsigned AdjStackDown
= TII
.getCallFrameSetupOpcode();
3308 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(AdjStackDown
))
3309 .addImm(NumBytes
).addImm(0).addImm(0);
3311 // Walk the register/memloc assignments, inserting copies/loads.
3312 const X86RegisterInfo
*RegInfo
= Subtarget
->getRegisterInfo();
3313 for (unsigned i
= 0, e
= ArgLocs
.size(); i
!= e
; ++i
) {
3314 CCValAssign
const &VA
= ArgLocs
[i
];
3315 const Value
*ArgVal
= OutVals
[VA
.getValNo()];
3316 MVT ArgVT
= OutVTs
[VA
.getValNo()];
3318 if (ArgVT
== MVT::x86mmx
)
3321 unsigned ArgReg
= ArgRegs
[VA
.getValNo()];
3323 // Promote the value if needed.
3324 switch (VA
.getLocInfo()) {
3325 case CCValAssign::Full
: break;
3326 case CCValAssign::SExt
: {
3327 assert(VA
.getLocVT().isInteger() && !VA
.getLocVT().isVector() &&
3328 "Unexpected extend");
3330 if (ArgVT
== MVT::i1
)
3333 bool Emitted
= X86FastEmitExtend(ISD::SIGN_EXTEND
, VA
.getLocVT(), ArgReg
,
3335 assert(Emitted
&& "Failed to emit a sext!"); (void)Emitted
;
3336 ArgVT
= VA
.getLocVT();
3339 case CCValAssign::ZExt
: {
3340 assert(VA
.getLocVT().isInteger() && !VA
.getLocVT().isVector() &&
3341 "Unexpected extend");
3343 // Handle zero-extension from i1 to i8, which is common.
3344 if (ArgVT
== MVT::i1
) {
3345 // Set the high bits to zero.
3346 ArgReg
= fastEmitZExtFromI1(MVT::i8
, ArgReg
, /*TODO: Kill=*/false);
3353 bool Emitted
= X86FastEmitExtend(ISD::ZERO_EXTEND
, VA
.getLocVT(), ArgReg
,
3355 assert(Emitted
&& "Failed to emit a zext!"); (void)Emitted
;
3356 ArgVT
= VA
.getLocVT();
3359 case CCValAssign::AExt
: {
3360 assert(VA
.getLocVT().isInteger() && !VA
.getLocVT().isVector() &&
3361 "Unexpected extend");
3362 bool Emitted
= X86FastEmitExtend(ISD::ANY_EXTEND
, VA
.getLocVT(), ArgReg
,
3365 Emitted
= X86FastEmitExtend(ISD::ZERO_EXTEND
, VA
.getLocVT(), ArgReg
,
3368 Emitted
= X86FastEmitExtend(ISD::SIGN_EXTEND
, VA
.getLocVT(), ArgReg
,
3371 assert(Emitted
&& "Failed to emit a aext!"); (void)Emitted
;
3372 ArgVT
= VA
.getLocVT();
3375 case CCValAssign::BCvt
: {
3376 ArgReg
= fastEmit_r(ArgVT
, VA
.getLocVT(), ISD::BITCAST
, ArgReg
,
3377 /*TODO: Kill=*/false);
3378 assert(ArgReg
&& "Failed to emit a bitcast!");
3379 ArgVT
= VA
.getLocVT();
3382 case CCValAssign::VExt
:
3383 // VExt has not been implemented, so this should be impossible to reach
3384 // for now. However, fallback to Selection DAG isel once implemented.
3386 case CCValAssign::AExtUpper
:
3387 case CCValAssign::SExtUpper
:
3388 case CCValAssign::ZExtUpper
:
3389 case CCValAssign::FPExt
:
3390 llvm_unreachable("Unexpected loc info!");
3391 case CCValAssign::Indirect
:
3392 // FIXME: Indirect doesn't need extending, but fast-isel doesn't fully
3397 if (VA
.isRegLoc()) {
3398 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3399 TII
.get(TargetOpcode::COPY
), VA
.getLocReg()).addReg(ArgReg
);
3400 OutRegs
.push_back(VA
.getLocReg());
3402 assert(VA
.isMemLoc());
3404 // Don't emit stores for undef values.
3405 if (isa
<UndefValue
>(ArgVal
))
3408 unsigned LocMemOffset
= VA
.getLocMemOffset();
3410 AM
.Base
.Reg
= RegInfo
->getStackRegister();
3411 AM
.Disp
= LocMemOffset
;
3412 ISD::ArgFlagsTy Flags
= OutFlags
[VA
.getValNo()];
3413 unsigned Alignment
= DL
.getABITypeAlignment(ArgVal
->getType());
3414 MachineMemOperand
*MMO
= FuncInfo
.MF
->getMachineMemOperand(
3415 MachinePointerInfo::getStack(*FuncInfo
.MF
, LocMemOffset
),
3416 MachineMemOperand::MOStore
, ArgVT
.getStoreSize(), Alignment
);
3417 if (Flags
.isByVal()) {
3418 X86AddressMode SrcAM
;
3419 SrcAM
.Base
.Reg
= ArgReg
;
3420 if (!TryEmitSmallMemcpy(AM
, SrcAM
, Flags
.getByValSize()))
3422 } else if (isa
<ConstantInt
>(ArgVal
) || isa
<ConstantPointerNull
>(ArgVal
)) {
3423 // If this is a really simple value, emit this with the Value* version
3424 // of X86FastEmitStore. If it isn't simple, we don't want to do this,
3425 // as it can cause us to reevaluate the argument.
3426 if (!X86FastEmitStore(ArgVT
, ArgVal
, AM
, MMO
))
3429 bool ValIsKill
= hasTrivialKill(ArgVal
);
3430 if (!X86FastEmitStore(ArgVT
, ArgReg
, ValIsKill
, AM
, MMO
))
3436 // ELF / PIC requires GOT in the EBX register before function calls via PLT
3438 if (Subtarget
->isPICStyleGOT()) {
3439 unsigned Base
= getInstrInfo()->getGlobalBaseReg(FuncInfo
.MF
);
3440 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3441 TII
.get(TargetOpcode::COPY
), X86::EBX
).addReg(Base
);
3444 if (Is64Bit
&& IsVarArg
&& !IsWin64
) {
3445 // From AMD64 ABI document:
3446 // For calls that may call functions that use varargs or stdargs
3447 // (prototype-less calls or calls to functions containing ellipsis (...) in
3448 // the declaration) %al is used as hidden argument to specify the number
3449 // of SSE registers used. The contents of %al do not need to match exactly
3450 // the number of registers, but must be an ubound on the number of SSE
3451 // registers used and is in the range 0 - 8 inclusive.
3453 // Count the number of XMM registers allocated.
3454 static const MCPhysReg XMMArgRegs
[] = {
3455 X86::XMM0
, X86::XMM1
, X86::XMM2
, X86::XMM3
,
3456 X86::XMM4
, X86::XMM5
, X86::XMM6
, X86::XMM7
3458 unsigned NumXMMRegs
= CCInfo
.getFirstUnallocated(XMMArgRegs
);
3459 assert((Subtarget
->hasSSE1() || !NumXMMRegs
)
3460 && "SSE registers cannot be used when SSE is disabled");
3461 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOV8ri
),
3462 X86::AL
).addImm(NumXMMRegs
);
3465 // Materialize callee address in a register. FIXME: GV address can be
3466 // handled with a CALLpcrel32 instead.
3467 X86AddressMode CalleeAM
;
3468 if (!X86SelectCallAddress(Callee
, CalleeAM
))
3471 unsigned CalleeOp
= 0;
3472 const GlobalValue
*GV
= nullptr;
3473 if (CalleeAM
.GV
!= nullptr) {
3475 } else if (CalleeAM
.Base
.Reg
!= 0) {
3476 CalleeOp
= CalleeAM
.Base
.Reg
;
3481 MachineInstrBuilder MIB
;
3483 // Register-indirect call.
3484 unsigned CallOpc
= Is64Bit
? X86::CALL64r
: X86::CALL32r
;
3485 MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(CallOpc
))
3489 assert(GV
&& "Not a direct call");
3490 // See if we need any target-specific flags on the GV operand.
3491 unsigned char OpFlags
= Subtarget
->classifyGlobalFunctionReference(GV
);
3493 // This will be a direct call, or an indirect call through memory for
3494 // NonLazyBind calls or dllimport calls.
3495 bool NeedLoad
= OpFlags
== X86II::MO_DLLIMPORT
||
3496 OpFlags
== X86II::MO_GOTPCREL
||
3497 OpFlags
== X86II::MO_COFFSTUB
;
3498 unsigned CallOpc
= NeedLoad
3499 ? (Is64Bit
? X86::CALL64m
: X86::CALL32m
)
3500 : (Is64Bit
? X86::CALL64pcrel32
: X86::CALLpcrel32
);
3502 MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(CallOpc
));
3504 MIB
.addReg(Is64Bit
? X86::RIP
: 0).addImm(1).addReg(0);
3506 MIB
.addSym(Symbol
, OpFlags
);
3508 MIB
.addGlobalAddress(GV
, 0, OpFlags
);
3513 // Add a register mask operand representing the call-preserved registers.
3514 // Proper defs for return values will be added by setPhysRegsDeadExcept().
3515 MIB
.addRegMask(TRI
.getCallPreservedMask(*FuncInfo
.MF
, CC
));
3517 // Add an implicit use GOT pointer in EBX.
3518 if (Subtarget
->isPICStyleGOT())
3519 MIB
.addReg(X86::EBX
, RegState::Implicit
);
3521 if (Is64Bit
&& IsVarArg
&& !IsWin64
)
3522 MIB
.addReg(X86::AL
, RegState::Implicit
);
3524 // Add implicit physical register uses to the call.
3525 for (auto Reg
: OutRegs
)
3526 MIB
.addReg(Reg
, RegState::Implicit
);
3528 // Issue CALLSEQ_END
3529 unsigned NumBytesForCalleeToPop
=
3530 X86::isCalleePop(CC
, Subtarget
->is64Bit(), IsVarArg
,
3531 TM
.Options
.GuaranteedTailCallOpt
)
3532 ? NumBytes
// Callee pops everything.
3533 : computeBytesPoppedByCalleeForSRet(Subtarget
, CC
, CLI
.CS
);
3534 unsigned AdjStackUp
= TII
.getCallFrameDestroyOpcode();
3535 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(AdjStackUp
))
3536 .addImm(NumBytes
).addImm(NumBytesForCalleeToPop
);
3538 // Now handle call return values.
3539 SmallVector
<CCValAssign
, 16> RVLocs
;
3540 CCState
CCRetInfo(CC
, IsVarArg
, *FuncInfo
.MF
, RVLocs
,
3541 CLI
.RetTy
->getContext());
3542 CCRetInfo
.AnalyzeCallResult(Ins
, RetCC_X86
);
3544 // Copy all of the result registers out of their specified physreg.
3545 unsigned ResultReg
= FuncInfo
.CreateRegs(CLI
.RetTy
);
3546 for (unsigned i
= 0; i
!= RVLocs
.size(); ++i
) {
3547 CCValAssign
&VA
= RVLocs
[i
];
3548 EVT CopyVT
= VA
.getValVT();
3549 unsigned CopyReg
= ResultReg
+ i
;
3550 unsigned SrcReg
= VA
.getLocReg();
3552 // If this is x86-64, and we disabled SSE, we can't return FP values
3553 if ((CopyVT
== MVT::f32
|| CopyVT
== MVT::f64
) &&
3554 ((Is64Bit
|| Ins
[i
].Flags
.isInReg()) && !Subtarget
->hasSSE1())) {
3555 report_fatal_error("SSE register return with SSE disabled");
3558 // If we prefer to use the value in xmm registers, copy it out as f80 and
3559 // use a truncate to move it from fp stack reg to xmm reg.
3560 if ((SrcReg
== X86::FP0
|| SrcReg
== X86::FP1
) &&
3561 isScalarFPTypeInSSEReg(VA
.getValVT())) {
3563 CopyReg
= createResultReg(&X86::RFP80RegClass
);
3566 // Copy out the result.
3567 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3568 TII
.get(TargetOpcode::COPY
), CopyReg
).addReg(SrcReg
);
3569 InRegs
.push_back(VA
.getLocReg());
3571 // Round the f80 to the right size, which also moves it to the appropriate
3572 // xmm register. This is accomplished by storing the f80 value in memory
3573 // and then loading it back.
3574 if (CopyVT
!= VA
.getValVT()) {
3575 EVT ResVT
= VA
.getValVT();
3576 unsigned Opc
= ResVT
== MVT::f32
? X86::ST_Fp80m32
: X86::ST_Fp80m64
;
3577 unsigned MemSize
= ResVT
.getSizeInBits()/8;
3578 int FI
= MFI
.CreateStackObject(MemSize
, MemSize
, false);
3579 addFrameReference(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3582 Opc
= ResVT
== MVT::f32
? X86::MOVSSrm_alt
: X86::MOVSDrm_alt
;
3583 addFrameReference(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3584 TII
.get(Opc
), ResultReg
+ i
), FI
);
3588 CLI
.ResultReg
= ResultReg
;
3589 CLI
.NumResultRegs
= RVLocs
.size();
3596 X86FastISel::fastSelectInstruction(const Instruction
*I
) {
3597 switch (I
->getOpcode()) {
3599 case Instruction::Load
:
3600 return X86SelectLoad(I
);
3601 case Instruction::Store
:
3602 return X86SelectStore(I
);
3603 case Instruction::Ret
:
3604 return X86SelectRet(I
);
3605 case Instruction::ICmp
:
3606 case Instruction::FCmp
:
3607 return X86SelectCmp(I
);
3608 case Instruction::ZExt
:
3609 return X86SelectZExt(I
);
3610 case Instruction::SExt
:
3611 return X86SelectSExt(I
);
3612 case Instruction::Br
:
3613 return X86SelectBranch(I
);
3614 case Instruction::LShr
:
3615 case Instruction::AShr
:
3616 case Instruction::Shl
:
3617 return X86SelectShift(I
);
3618 case Instruction::SDiv
:
3619 case Instruction::UDiv
:
3620 case Instruction::SRem
:
3621 case Instruction::URem
:
3622 return X86SelectDivRem(I
);
3623 case Instruction::Select
:
3624 return X86SelectSelect(I
);
3625 case Instruction::Trunc
:
3626 return X86SelectTrunc(I
);
3627 case Instruction::FPExt
:
3628 return X86SelectFPExt(I
);
3629 case Instruction::FPTrunc
:
3630 return X86SelectFPTrunc(I
);
3631 case Instruction::SIToFP
:
3632 return X86SelectSIToFP(I
);
3633 case Instruction::UIToFP
:
3634 return X86SelectUIToFP(I
);
3635 case Instruction::IntToPtr
: // Deliberate fall-through.
3636 case Instruction::PtrToInt
: {
3637 EVT SrcVT
= TLI
.getValueType(DL
, I
->getOperand(0)->getType());
3638 EVT DstVT
= TLI
.getValueType(DL
, I
->getType());
3639 if (DstVT
.bitsGT(SrcVT
))
3640 return X86SelectZExt(I
);
3641 if (DstVT
.bitsLT(SrcVT
))
3642 return X86SelectTrunc(I
);
3643 unsigned Reg
= getRegForValue(I
->getOperand(0));
3644 if (Reg
== 0) return false;
3645 updateValueMap(I
, Reg
);
3648 case Instruction::BitCast
: {
3649 // Select SSE2/AVX bitcasts between 128/256/512 bit vector types.
3650 if (!Subtarget
->hasSSE2())
3654 if (!isTypeLegal(I
->getOperand(0)->getType(), SrcVT
) ||
3655 !isTypeLegal(I
->getType(), DstVT
))
3658 // Only allow vectors that use xmm/ymm/zmm.
3659 if (!SrcVT
.isVector() || !DstVT
.isVector() ||
3660 SrcVT
.getVectorElementType() == MVT::i1
||
3661 DstVT
.getVectorElementType() == MVT::i1
)
3664 unsigned Reg
= getRegForValue(I
->getOperand(0));
3668 // No instruction is needed for conversion. Reuse the register used by
3669 // the fist operand.
3670 updateValueMap(I
, Reg
);
3678 unsigned X86FastISel::X86MaterializeInt(const ConstantInt
*CI
, MVT VT
) {
3682 uint64_t Imm
= CI
->getZExtValue();
3684 unsigned SrcReg
= fastEmitInst_(X86::MOV32r0
, &X86::GR32RegClass
);
3685 switch (VT
.SimpleTy
) {
3686 default: llvm_unreachable("Unexpected value type");
3689 return fastEmitInst_extractsubreg(MVT::i8
, SrcReg
, /*Kill=*/true,
3692 return fastEmitInst_extractsubreg(MVT::i16
, SrcReg
, /*Kill=*/true,
3697 unsigned ResultReg
= createResultReg(&X86::GR64RegClass
);
3698 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3699 TII
.get(TargetOpcode::SUBREG_TO_REG
), ResultReg
)
3700 .addImm(0).addReg(SrcReg
).addImm(X86::sub_32bit
);
3707 switch (VT
.SimpleTy
) {
3708 default: llvm_unreachable("Unexpected value type");
3712 case MVT::i8
: Opc
= X86::MOV8ri
; break;
3713 case MVT::i16
: Opc
= X86::MOV16ri
; break;
3714 case MVT::i32
: Opc
= X86::MOV32ri
; break;
3716 if (isUInt
<32>(Imm
))
3717 Opc
= X86::MOV32ri64
;
3718 else if (isInt
<32>(Imm
))
3719 Opc
= X86::MOV64ri32
;
3725 return fastEmitInst_i(Opc
, TLI
.getRegClassFor(VT
), Imm
);
3728 unsigned X86FastISel::X86MaterializeFP(const ConstantFP
*CFP
, MVT VT
) {
3729 if (CFP
->isNullValue())
3730 return fastMaterializeFloatZero(CFP
);
3732 // Can't handle alternate code models yet.
3733 CodeModel::Model CM
= TM
.getCodeModel();
3734 if (CM
!= CodeModel::Small
&& CM
!= CodeModel::Large
)
3737 // Get opcode and regclass of the output for the given load instruction.
3739 bool HasAVX
= Subtarget
->hasAVX();
3740 bool HasAVX512
= Subtarget
->hasAVX512();
3741 switch (VT
.SimpleTy
) {
3744 if (X86ScalarSSEf32
)
3745 Opc
= HasAVX512
? X86::VMOVSSZrm_alt
:
3746 HasAVX
? X86::VMOVSSrm_alt
:
3749 Opc
= X86::LD_Fp32m
;
3752 if (X86ScalarSSEf64
)
3753 Opc
= HasAVX512
? X86::VMOVSDZrm_alt
:
3754 HasAVX
? X86::VMOVSDrm_alt
:
3757 Opc
= X86::LD_Fp64m
;
3760 // No f80 support yet.
3764 // MachineConstantPool wants an explicit alignment.
3765 unsigned Align
= DL
.getPrefTypeAlignment(CFP
->getType());
3767 // Alignment of vector types. FIXME!
3768 Align
= DL
.getTypeAllocSize(CFP
->getType());
3771 // x86-32 PIC requires a PIC base register for constant pools.
3772 unsigned PICBase
= 0;
3773 unsigned char OpFlag
= Subtarget
->classifyLocalReference(nullptr);
3774 if (OpFlag
== X86II::MO_PIC_BASE_OFFSET
)
3775 PICBase
= getInstrInfo()->getGlobalBaseReg(FuncInfo
.MF
);
3776 else if (OpFlag
== X86II::MO_GOTOFF
)
3777 PICBase
= getInstrInfo()->getGlobalBaseReg(FuncInfo
.MF
);
3778 else if (Subtarget
->is64Bit() && TM
.getCodeModel() == CodeModel::Small
)
3781 // Create the load from the constant pool.
3782 unsigned CPI
= MCP
.getConstantPoolIndex(CFP
, Align
);
3783 unsigned ResultReg
= createResultReg(TLI
.getRegClassFor(VT
.SimpleTy
));
3785 if (CM
== CodeModel::Large
) {
3786 unsigned AddrReg
= createResultReg(&X86::GR64RegClass
);
3787 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOV64ri
),
3789 .addConstantPoolIndex(CPI
, 0, OpFlag
);
3790 MachineInstrBuilder MIB
= BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3791 TII
.get(Opc
), ResultReg
);
3792 addDirectMem(MIB
, AddrReg
);
3793 MachineMemOperand
*MMO
= FuncInfo
.MF
->getMachineMemOperand(
3794 MachinePointerInfo::getConstantPool(*FuncInfo
.MF
),
3795 MachineMemOperand::MOLoad
, DL
.getPointerSize(), Align
);
3796 MIB
->addMemOperand(*FuncInfo
.MF
, MMO
);
3800 addConstantPoolReference(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3801 TII
.get(Opc
), ResultReg
),
3802 CPI
, PICBase
, OpFlag
);
3806 unsigned X86FastISel::X86MaterializeGV(const GlobalValue
*GV
, MVT VT
) {
3807 // Can't handle alternate code models yet.
3808 if (TM
.getCodeModel() != CodeModel::Small
)
3811 // Materialize addresses with LEA/MOV instructions.
3813 if (X86SelectAddress(GV
, AM
)) {
3814 // If the expression is just a basereg, then we're done, otherwise we need
3816 if (AM
.BaseType
== X86AddressMode::RegBase
&&
3817 AM
.IndexReg
== 0 && AM
.Disp
== 0 && AM
.GV
== nullptr)
3820 unsigned ResultReg
= createResultReg(TLI
.getRegClassFor(VT
));
3821 if (TM
.getRelocationModel() == Reloc::Static
&&
3822 TLI
.getPointerTy(DL
) == MVT::i64
) {
3823 // The displacement code could be more than 32 bits away so we need to use
3824 // an instruction with a 64 bit immediate
3825 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(X86::MOV64ri
),
3827 .addGlobalAddress(GV
);
3830 TLI
.getPointerTy(DL
) == MVT::i32
3831 ? (Subtarget
->isTarget64BitILP32() ? X86::LEA64_32r
: X86::LEA32r
)
3833 addFullAddress(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3834 TII
.get(Opc
), ResultReg
), AM
);
3841 unsigned X86FastISel::fastMaterializeConstant(const Constant
*C
) {
3842 EVT CEVT
= TLI
.getValueType(DL
, C
->getType(), true);
3844 // Only handle simple types.
3845 if (!CEVT
.isSimple())
3847 MVT VT
= CEVT
.getSimpleVT();
3849 if (const auto *CI
= dyn_cast
<ConstantInt
>(C
))
3850 return X86MaterializeInt(CI
, VT
);
3851 else if (const ConstantFP
*CFP
= dyn_cast
<ConstantFP
>(C
))
3852 return X86MaterializeFP(CFP
, VT
);
3853 else if (const GlobalValue
*GV
= dyn_cast
<GlobalValue
>(C
))
3854 return X86MaterializeGV(GV
, VT
);
3859 unsigned X86FastISel::fastMaterializeAlloca(const AllocaInst
*C
) {
3860 // Fail on dynamic allocas. At this point, getRegForValue has already
3861 // checked its CSE maps, so if we're here trying to handle a dynamic
3862 // alloca, we're not going to succeed. X86SelectAddress has a
3863 // check for dynamic allocas, because it's called directly from
3864 // various places, but targetMaterializeAlloca also needs a check
3865 // in order to avoid recursion between getRegForValue,
3866 // X86SelectAddrss, and targetMaterializeAlloca.
3867 if (!FuncInfo
.StaticAllocaMap
.count(C
))
3869 assert(C
->isStaticAlloca() && "dynamic alloca in the static alloca map?");
3872 if (!X86SelectAddress(C
, AM
))
3875 TLI
.getPointerTy(DL
) == MVT::i32
3876 ? (Subtarget
->isTarget64BitILP32() ? X86::LEA64_32r
: X86::LEA32r
)
3878 const TargetRegisterClass
*RC
= TLI
.getRegClassFor(TLI
.getPointerTy(DL
));
3879 unsigned ResultReg
= createResultReg(RC
);
3880 addFullAddress(BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3881 TII
.get(Opc
), ResultReg
), AM
);
3885 unsigned X86FastISel::fastMaterializeFloatZero(const ConstantFP
*CF
) {
3887 if (!isTypeLegal(CF
->getType(), VT
))
3890 // Get opcode and regclass for the given zero.
3891 bool HasAVX512
= Subtarget
->hasAVX512();
3893 switch (VT
.SimpleTy
) {
3896 if (X86ScalarSSEf32
)
3897 Opc
= HasAVX512
? X86::AVX512_FsFLD0SS
: X86::FsFLD0SS
;
3899 Opc
= X86::LD_Fp032
;
3902 if (X86ScalarSSEf64
)
3903 Opc
= HasAVX512
? X86::AVX512_FsFLD0SD
: X86::FsFLD0SD
;
3905 Opc
= X86::LD_Fp064
;
3908 // No f80 support yet.
3912 unsigned ResultReg
= createResultReg(TLI
.getRegClassFor(VT
));
3913 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, TII
.get(Opc
), ResultReg
);
3918 bool X86FastISel::tryToFoldLoadIntoMI(MachineInstr
*MI
, unsigned OpNo
,
3919 const LoadInst
*LI
) {
3920 const Value
*Ptr
= LI
->getPointerOperand();
3922 if (!X86SelectAddress(Ptr
, AM
))
3925 const X86InstrInfo
&XII
= (const X86InstrInfo
&)TII
;
3927 unsigned Size
= DL
.getTypeAllocSize(LI
->getType());
3928 unsigned Alignment
= LI
->getAlignment();
3930 if (Alignment
== 0) // Ensure that codegen never sees alignment 0
3931 Alignment
= DL
.getABITypeAlignment(LI
->getType());
3933 SmallVector
<MachineOperand
, 8> AddrOps
;
3934 AM
.getFullAddress(AddrOps
);
3936 MachineInstr
*Result
= XII
.foldMemoryOperandImpl(
3937 *FuncInfo
.MF
, *MI
, OpNo
, AddrOps
, FuncInfo
.InsertPt
, Size
, Alignment
,
3938 /*AllowCommute=*/true);
3942 // The index register could be in the wrong register class. Unfortunately,
3943 // foldMemoryOperandImpl could have commuted the instruction so its not enough
3944 // to just look at OpNo + the offset to the index reg. We actually need to
3945 // scan the instruction to find the index reg and see if its the correct reg
3947 unsigned OperandNo
= 0;
3948 for (MachineInstr::mop_iterator I
= Result
->operands_begin(),
3949 E
= Result
->operands_end(); I
!= E
; ++I
, ++OperandNo
) {
3950 MachineOperand
&MO
= *I
;
3951 if (!MO
.isReg() || MO
.isDef() || MO
.getReg() != AM
.IndexReg
)
3953 // Found the index reg, now try to rewrite it.
3954 unsigned IndexReg
= constrainOperandRegClass(Result
->getDesc(),
3955 MO
.getReg(), OperandNo
);
3956 if (IndexReg
== MO
.getReg())
3958 MO
.setReg(IndexReg
);
3961 Result
->addMemOperand(*FuncInfo
.MF
, createMachineMemOperandFor(LI
));
3962 Result
->cloneInstrSymbols(*FuncInfo
.MF
, *MI
);
3963 MachineBasicBlock::iterator
I(MI
);
3964 removeDeadCode(I
, std::next(I
));
3968 unsigned X86FastISel::fastEmitInst_rrrr(unsigned MachineInstOpcode
,
3969 const TargetRegisterClass
*RC
,
3970 unsigned Op0
, bool Op0IsKill
,
3971 unsigned Op1
, bool Op1IsKill
,
3972 unsigned Op2
, bool Op2IsKill
,
3973 unsigned Op3
, bool Op3IsKill
) {
3974 const MCInstrDesc
&II
= TII
.get(MachineInstOpcode
);
3976 unsigned ResultReg
= createResultReg(RC
);
3977 Op0
= constrainOperandRegClass(II
, Op0
, II
.getNumDefs());
3978 Op1
= constrainOperandRegClass(II
, Op1
, II
.getNumDefs() + 1);
3979 Op2
= constrainOperandRegClass(II
, Op2
, II
.getNumDefs() + 2);
3980 Op3
= constrainOperandRegClass(II
, Op3
, II
.getNumDefs() + 3);
3982 if (II
.getNumDefs() >= 1)
3983 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, II
, ResultReg
)
3984 .addReg(Op0
, getKillRegState(Op0IsKill
))
3985 .addReg(Op1
, getKillRegState(Op1IsKill
))
3986 .addReg(Op2
, getKillRegState(Op2IsKill
))
3987 .addReg(Op3
, getKillRegState(Op3IsKill
));
3989 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
, II
)
3990 .addReg(Op0
, getKillRegState(Op0IsKill
))
3991 .addReg(Op1
, getKillRegState(Op1IsKill
))
3992 .addReg(Op2
, getKillRegState(Op2IsKill
))
3993 .addReg(Op3
, getKillRegState(Op3IsKill
));
3994 BuildMI(*FuncInfo
.MBB
, FuncInfo
.InsertPt
, DbgLoc
,
3995 TII
.get(TargetOpcode::COPY
), ResultReg
).addReg(II
.ImplicitDefs
[0]);
4002 FastISel
*X86::createFastISel(FunctionLoweringInfo
&funcInfo
,
4003 const TargetLibraryInfo
*libInfo
) {
4004 return new X86FastISel(funcInfo
, libInfo
);