?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[SampleProfileLoader] Fix integer overflow in generateMDProfMetadata (#90217)
[llvm-project.git] / llvm / lib / Target / SPIRV / SPIRVInstructionSelector.cpp
blob2051cdc7e01ff81ef4ad12b94ac6c7349381fd15
1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the targeting of the InstructionSelector class for
10 // SPIRV.
11 // TODO: This should be generated by TableGen.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "MCTargetDesc/SPIRVBaseInfo.h"
16 #include "MCTargetDesc/SPIRVMCTargetDesc.h"
17 #include "SPIRV.h"
18 #include "SPIRVGlobalRegistry.h"
19 #include "SPIRVInstrInfo.h"
20 #include "SPIRVRegisterBankInfo.h"
21 #include "SPIRVRegisterInfo.h"
22 #include "SPIRVTargetMachine.h"
23 #include "SPIRVUtils.h"
24 #include "llvm/ADT/APFloat.h"
25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28 #include "llvm/CodeGen/MachineInstrBuilder.h"
29 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/CodeGen/TargetOpcodes.h"
32 #include "llvm/IR/IntrinsicsSPIRV.h"
33 #include "llvm/Support/Debug.h"
34
35 namespace llvm {
36
37 class SPIRVMachineModuleInfo : public MachineModuleInfoImpl {
38 public:
39 SyncScope::ID Work_ItemSSID;
40 SyncScope::ID WorkGroupSSID;
41 SyncScope::ID DeviceSSID;
42 SyncScope::ID AllSVMDevicesSSID;
43 SyncScope::ID SubGroupSSID;
44
45 SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) {
46 LLVMContext &CTX = MMI.getModule()->getContext();
47 Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item");
48 WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup");
49 DeviceSSID = CTX.getOrInsertSyncScopeID("device");
50 AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices");
51 SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group");
52 }
53 };
54
55 } // end namespace llvm
56
57 #define DEBUG_TYPE "spirv-isel"
58
59 using namespace llvm;
60 namespace CL = SPIRV::OpenCLExtInst;
61 namespace GL = SPIRV::GLSLExtInst;
62
63 using ExtInstList =
64 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
65
66 namespace {
67
68 #define GET_GLOBALISEL_PREDICATE_BITSET
69 #include "SPIRVGenGlobalISel.inc"
70 #undef GET_GLOBALISEL_PREDICATE_BITSET
71
72 class SPIRVInstructionSelector : public InstructionSelector {
73 const SPIRVSubtarget &STI;
74 const SPIRVInstrInfo &TII;
75 const SPIRVRegisterInfo &TRI;
76 const RegisterBankInfo &RBI;
77 SPIRVGlobalRegistry &GR;
78 MachineRegisterInfo *MRI;
79 SPIRVMachineModuleInfo *MMI = nullptr;
80
81 /// We need to keep track of the number we give to anonymous global values to
82 /// generate the same name every time when this is needed.
83 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
84
85 public:
86 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
87 const SPIRVSubtarget &ST,
88 const RegisterBankInfo &RBI);
89 void setupMF(MachineFunction &MF, GISelKnownBits *KB,
90 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
91 BlockFrequencyInfo *BFI) override;
92 // Common selection code. Instruction-specific selection occurs in spvSelect.
93 bool select(MachineInstr &I) override;
94 static const char *getName() { return DEBUG_TYPE; }
95
96 #define GET_GLOBALISEL_PREDICATES_DECL
97 #include "SPIRVGenGlobalISel.inc"
98 #undef GET_GLOBALISEL_PREDICATES_DECL
99
100 #define GET_GLOBALISEL_TEMPORARIES_DECL
101 #include "SPIRVGenGlobalISel.inc"
102 #undef GET_GLOBALISEL_TEMPORARIES_DECL
103
104 private:
105 // tblgen-erated 'select' implementation, used as the initial selector for
106 // the patterns that don't require complex C++.
107 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
108
109 // All instruction-specific selection that didn't happen in "select()".
110 // Is basically a large Switch/Case delegating to all other select method.
111 bool spvSelect(Register ResVReg, const SPIRVType *ResType,
112 MachineInstr &I) const;
113
114 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
115 const MachineInstr *Init = nullptr) const;
116
117 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType,
118 MachineInstr &I, Register SrcReg,
119 unsigned Opcode) const;
120 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
121 unsigned Opcode) const;
122
123 bool selectBitcast(Register ResVReg, const SPIRVType *ResType,
124 MachineInstr &I) const;
125
126 bool selectLoad(Register ResVReg, const SPIRVType *ResType,
127 MachineInstr &I) const;
128 bool selectStore(MachineInstr &I) const;
129
130 bool selectStackSave(Register ResVReg, const SPIRVType *ResType,
131 MachineInstr &I) const;
132 bool selectStackRestore(MachineInstr &I) const;
133
134 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
135
136 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
137 MachineInstr &I, unsigned NewOpcode,
138 unsigned NegateOpcode = 0) const;
139
140 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
141 MachineInstr &I) const;
142
143 bool selectFence(MachineInstr &I) const;
144
145 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
146 MachineInstr &I) const;
147
148 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType,
149 MachineInstr &I, unsigned OpType) const;
150
151 bool selectAll(Register ResVReg, const SPIRVType *ResType,
152 MachineInstr &I) const;
153
154 bool selectAny(Register ResVReg, const SPIRVType *ResType,
155 MachineInstr &I) const;
156
157 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
158 MachineInstr &I) const;
159
160 bool selectConstVector(Register ResVReg, const SPIRVType *ResType,
161 MachineInstr &I) const;
162 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType,
163 MachineInstr &I) const;
164
165 bool selectCmp(Register ResVReg, const SPIRVType *ResType,
166 unsigned comparisonOpcode, MachineInstr &I) const;
167
168 bool selectICmp(Register ResVReg, const SPIRVType *ResType,
169 MachineInstr &I) const;
170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
171 MachineInstr &I) const;
172
173 bool selectFmix(Register ResVReg, const SPIRVType *ResType,
174 MachineInstr &I) const;
175
176 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
177 int OpIdx) const;
178 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
179 int OpIdx) const;
180
181 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm,
182 MachineInstr &I) const;
183
184 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
185 bool IsSigned) const;
186 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
187 bool IsSigned, unsigned Opcode) const;
188 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
189 bool IsSigned) const;
190
191 bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
192 MachineInstr &I) const;
193
194 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
195 const SPIRVType *intTy, const SPIRVType *boolTy) const;
196
197 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
198 MachineInstr &I) const;
199 bool selectFreeze(Register ResVReg, const SPIRVType *ResType,
200 MachineInstr &I) const;
201 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
202 MachineInstr &I) const;
203 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
204 MachineInstr &I) const;
205 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
206 MachineInstr &I) const;
207 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
208 MachineInstr &I) const;
209 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
210 MachineInstr &I) const;
211 bool selectGEP(Register ResVReg, const SPIRVType *ResType,
212 MachineInstr &I) const;
213
214 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
215 MachineInstr &I) const;
216 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType,
217 MachineInstr &I) const;
218
219 bool selectBranch(MachineInstr &I) const;
220 bool selectBranchCond(MachineInstr &I) const;
221
222 bool selectPhi(Register ResVReg, const SPIRVType *ResType,
223 MachineInstr &I) const;
224
225 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
226 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
227 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
228 MachineInstr &I, CL::OpenCLExtInst CLInst,
229 GL::GLSLExtInst GLInst) const;
230 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
231 MachineInstr &I, const ExtInstList &ExtInsts) const;
232
233 bool selectLog10(Register ResVReg, const SPIRVType *ResType,
234 MachineInstr &I) const;
235
236 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType,
237 MachineInstr &I) const;
238
239 bool selectUnmergeValues(MachineInstr &I) const;
240
241 Register buildI32Constant(uint32_t Val, MachineInstr &I,
242 const SPIRVType *ResType = nullptr) const;
243
244 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
245 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const;
246 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
247 MachineInstr &I) const;
248
249 bool wrapIntoSpecConstantOp(MachineInstr &I,
250 SmallVector<Register> &CompositeArgs) const;
251 };
252
253 } // end anonymous namespace
254
255 #define GET_GLOBALISEL_IMPL
256 #include "SPIRVGenGlobalISel.inc"
257 #undef GET_GLOBALISEL_IMPL
258
259 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
260 const SPIRVSubtarget &ST,
261 const RegisterBankInfo &RBI)
262 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
263 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
264 #define GET_GLOBALISEL_PREDICATES_INIT
265 #include "SPIRVGenGlobalISel.inc"
266 #undef GET_GLOBALISEL_PREDICATES_INIT
267 #define GET_GLOBALISEL_TEMPORARIES_INIT
268 #include "SPIRVGenGlobalISel.inc"
269 #undef GET_GLOBALISEL_TEMPORARIES_INIT
270 {
271 }
272
273 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
274 CodeGenCoverage *CoverageInfo,
275 ProfileSummaryInfo *PSI,
276 BlockFrequencyInfo *BFI) {
277 MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>();
278 MRI = &MF.getRegInfo();
279 GR.setCurrentFunc(MF);
280 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
281 }
282
283 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI);
284
285 // Defined in SPIRVLegalizerInfo.cpp.
286 extern bool isTypeFoldingSupported(unsigned Opcode);
287
288 bool SPIRVInstructionSelector::select(MachineInstr &I) {
289 assert(I.getParent() && "Instruction should be in a basic block!");
290 assert(I.getParent()->getParent() && "Instruction should be in a function!");
291
292 Register Opcode = I.getOpcode();
293 // If it's not a GMIR instruction, we've selected it already.
294 if (!isPreISelGenericOpcode(Opcode)) {
295 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
296 Register DstReg = I.getOperand(0).getReg();
297 Register SrcReg = I.getOperand(1).getReg();
298 auto *Def = MRI->getVRegDef(SrcReg);
299 if (isTypeFoldingSupported(Def->getOpcode())) {
300 if (MRI->getType(DstReg).isPointer())
301 MRI->setType(DstReg, LLT::scalar(32));
302 bool Res = selectImpl(I, *CoverageInfo);
303 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
304 if (Res)
305 return Res;
306 }
307 MRI->replaceRegWith(SrcReg, DstReg);
308 I.removeFromParent();
309 return true;
310 } else if (I.getNumDefs() == 1) {
311 // Make all vregs 32 bits (for SPIR-V IDs).
312 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32));
313 }
314 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
315 }
316
317 if (I.getNumOperands() != I.getNumExplicitOperands()) {
318 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
319 return false;
320 }
321
322 // Common code for getting return reg+type, and removing selected instr
323 // from parent occurs here. Instr-specific selection happens in spvSelect().
324 bool HasDefs = I.getNumDefs() > 0;
325 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
326 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
327 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
328 if (spvSelect(ResVReg, ResType, I)) {
329 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs).
330 for (unsigned i = 0; i < I.getNumDefs(); ++i)
331 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32));
332 I.removeFromParent();
333 return true;
334 }
335 return false;
336 }
337
338 bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
339 const SPIRVType *ResType,
340 MachineInstr &I) const {
341 const unsigned Opcode = I.getOpcode();
342 if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT)
343 return selectImpl(I, *CoverageInfo);
344 switch (Opcode) {
345 case TargetOpcode::G_CONSTANT:
346 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(),
347 I);
348 case TargetOpcode::G_GLOBAL_VALUE:
349 return selectGlobalValue(ResVReg, I);
350 case TargetOpcode::G_IMPLICIT_DEF:
351 return selectOpUndef(ResVReg, ResType, I);
352 case TargetOpcode::G_FREEZE:
353 return selectFreeze(ResVReg, ResType, I);
354
355 case TargetOpcode::G_INTRINSIC:
356 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
357 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
358 return selectIntrinsic(ResVReg, ResType, I);
359 case TargetOpcode::G_BITREVERSE:
360 return selectBitreverse(ResVReg, ResType, I);
361
362 case TargetOpcode::G_BUILD_VECTOR:
363 return selectConstVector(ResVReg, ResType, I);
364 case TargetOpcode::G_SPLAT_VECTOR:
365 return selectSplatVector(ResVReg, ResType, I);
366
367 case TargetOpcode::G_SHUFFLE_VECTOR: {
368 MachineBasicBlock &BB = *I.getParent();
369 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
370 .addDef(ResVReg)
371 .addUse(GR.getSPIRVTypeID(ResType))
372 .addUse(I.getOperand(1).getReg())
373 .addUse(I.getOperand(2).getReg());
374 for (auto V : I.getOperand(3).getShuffleMask())
375 MIB.addImm(V);
376 return MIB.constrainAllUses(TII, TRI, RBI);
377 }
378 case TargetOpcode::G_MEMMOVE:
379 case TargetOpcode::G_MEMCPY:
380 case TargetOpcode::G_MEMSET:
381 return selectMemOperation(ResVReg, I);
382
383 case TargetOpcode::G_ICMP:
384 return selectICmp(ResVReg, ResType, I);
385 case TargetOpcode::G_FCMP:
386 return selectFCmp(ResVReg, ResType, I);
387
388 case TargetOpcode::G_FRAME_INDEX:
389 return selectFrameIndex(ResVReg, ResType, I);
390
391 case TargetOpcode::G_LOAD:
392 return selectLoad(ResVReg, ResType, I);
393 case TargetOpcode::G_STORE:
394 return selectStore(I);
395
396 case TargetOpcode::G_BR:
397 return selectBranch(I);
398 case TargetOpcode::G_BRCOND:
399 return selectBranchCond(I);
400
401 case TargetOpcode::G_PHI:
402 return selectPhi(ResVReg, ResType, I);
403
404 case TargetOpcode::G_FPTOSI:
405 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
406 case TargetOpcode::G_FPTOUI:
407 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
408
409 case TargetOpcode::G_SITOFP:
410 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
411 case TargetOpcode::G_UITOFP:
412 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
413
414 case TargetOpcode::G_CTPOP:
415 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
416 case TargetOpcode::G_SMIN:
417 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
418 case TargetOpcode::G_UMIN:
419 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
420
421 case TargetOpcode::G_SMAX:
422 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
423 case TargetOpcode::G_UMAX:
424 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
425
426 case TargetOpcode::G_FMA:
427 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
428
429 case TargetOpcode::G_FPOW:
430 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
431 case TargetOpcode::G_FPOWI:
432 return selectExtInst(ResVReg, ResType, I, CL::pown);
433
434 case TargetOpcode::G_FEXP:
435 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
436 case TargetOpcode::G_FEXP2:
437 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
438
439 case TargetOpcode::G_FLOG:
440 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
441 case TargetOpcode::G_FLOG2:
442 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
443 case TargetOpcode::G_FLOG10:
444 return selectLog10(ResVReg, ResType, I);
445
446 case TargetOpcode::G_FABS:
447 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
448 case TargetOpcode::G_ABS:
449 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
450
451 case TargetOpcode::G_FMINNUM:
452 case TargetOpcode::G_FMINIMUM:
453 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
454 case TargetOpcode::G_FMAXNUM:
455 case TargetOpcode::G_FMAXIMUM:
456 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
457
458 case TargetOpcode::G_FCOPYSIGN:
459 return selectExtInst(ResVReg, ResType, I, CL::copysign);
460
461 case TargetOpcode::G_FCEIL:
462 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
463 case TargetOpcode::G_FFLOOR:
464 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
465
466 case TargetOpcode::G_FCOS:
467 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
468 case TargetOpcode::G_FSIN:
469 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
470 case TargetOpcode::G_FTAN:
471 return selectExtInst(ResVReg, ResType, I, CL::tan, GL::Tan);
472
473 case TargetOpcode::G_FSQRT:
474 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
475
476 case TargetOpcode::G_CTTZ:
477 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
478 return selectExtInst(ResVReg, ResType, I, CL::ctz);
479 case TargetOpcode::G_CTLZ:
480 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
481 return selectExtInst(ResVReg, ResType, I, CL::clz);
482
483 case TargetOpcode::G_INTRINSIC_ROUND:
484 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
485 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
486 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
487 case TargetOpcode::G_INTRINSIC_TRUNC:
488 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
489 case TargetOpcode::G_FRINT:
490 case TargetOpcode::G_FNEARBYINT:
491 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
492
493 case TargetOpcode::G_SMULH:
494 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
495 case TargetOpcode::G_UMULH:
496 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
497
498 case TargetOpcode::G_SEXT:
499 return selectExt(ResVReg, ResType, I, true);
500 case TargetOpcode::G_ANYEXT:
501 case TargetOpcode::G_ZEXT:
502 return selectExt(ResVReg, ResType, I, false);
503 case TargetOpcode::G_TRUNC:
504 return selectTrunc(ResVReg, ResType, I);
505 case TargetOpcode::G_FPTRUNC:
506 case TargetOpcode::G_FPEXT:
507 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
508
509 case TargetOpcode::G_PTRTOINT:
510 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
511 case TargetOpcode::G_INTTOPTR:
512 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
513 case TargetOpcode::G_BITCAST:
514 return selectBitcast(ResVReg, ResType, I);
515 case TargetOpcode::G_ADDRSPACE_CAST:
516 return selectAddrSpaceCast(ResVReg, ResType, I);
517 case TargetOpcode::G_PTR_ADD: {
518 // Currently, we get G_PTR_ADD only as a result of translating
519 // global variables, initialized with constant expressions like GV + Const
520 // (see test opencl/basic/progvar_prog_scope_init.ll).
521 // TODO: extend the handler once we have other cases.
522 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
523 Register GV = I.getOperand(1).getReg();
524 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
525 (void)II;
526 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
527 (*II).getOpcode() == TargetOpcode::COPY ||
528 (*II).getOpcode() == SPIRV::OpVariable) &&
529 isImm(I.getOperand(2), MRI));
530 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
531 MachineBasicBlock &BB = *I.getParent();
532 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
533 .addDef(ResVReg)
534 .addUse(GR.getSPIRVTypeID(ResType))
535 .addImm(static_cast<uint32_t>(
536 SPIRV::Opcode::InBoundsPtrAccessChain))
537 .addUse(GV)
538 .addUse(Idx)
539 .addUse(I.getOperand(2).getReg());
540 return MIB.constrainAllUses(TII, TRI, RBI);
541 }
542
543 case TargetOpcode::G_ATOMICRMW_OR:
544 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
545 case TargetOpcode::G_ATOMICRMW_ADD:
546 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
547 case TargetOpcode::G_ATOMICRMW_AND:
548 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
549 case TargetOpcode::G_ATOMICRMW_MAX:
550 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
551 case TargetOpcode::G_ATOMICRMW_MIN:
552 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
553 case TargetOpcode::G_ATOMICRMW_SUB:
554 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
555 case TargetOpcode::G_ATOMICRMW_XOR:
556 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
557 case TargetOpcode::G_ATOMICRMW_UMAX:
558 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
559 case TargetOpcode::G_ATOMICRMW_UMIN:
560 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
561 case TargetOpcode::G_ATOMICRMW_XCHG:
562 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
563 case TargetOpcode::G_ATOMIC_CMPXCHG:
564 return selectAtomicCmpXchg(ResVReg, ResType, I);
565
566 case TargetOpcode::G_ATOMICRMW_FADD:
567 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
568 case TargetOpcode::G_ATOMICRMW_FSUB:
569 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
570 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
571 SPIRV::OpFNegate);
572 case TargetOpcode::G_ATOMICRMW_FMIN:
573 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
574 case TargetOpcode::G_ATOMICRMW_FMAX:
575 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
576
577 case TargetOpcode::G_FENCE:
578 return selectFence(I);
579
580 case TargetOpcode::G_STACKSAVE:
581 return selectStackSave(ResVReg, ResType, I);
582 case TargetOpcode::G_STACKRESTORE:
583 return selectStackRestore(I);
584
585 case TargetOpcode::G_UNMERGE_VALUES:
586 return selectUnmergeValues(I);
587
588 default:
589 return false;
590 }
591 }
592
593 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
594 const SPIRVType *ResType,
595 MachineInstr &I,
596 CL::OpenCLExtInst CLInst) const {
597 return selectExtInst(ResVReg, ResType, I,
598 {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
599 }
600
601 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
602 const SPIRVType *ResType,
603 MachineInstr &I,
604 CL::OpenCLExtInst CLInst,
605 GL::GLSLExtInst GLInst) const {
606 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
607 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
608 return selectExtInst(ResVReg, ResType, I, ExtInsts);
609 }
610
611 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
612 const SPIRVType *ResType,
613 MachineInstr &I,
614 const ExtInstList &Insts) const {
615
616 for (const auto &Ex : Insts) {
617 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
618 uint32_t Opcode = Ex.second;
619 if (STI.canUseExtInstSet(Set)) {
620 MachineBasicBlock &BB = *I.getParent();
621 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
622 .addDef(ResVReg)
623 .addUse(GR.getSPIRVTypeID(ResType))
624 .addImm(static_cast<uint32_t>(Set))
625 .addImm(Opcode);
626 const unsigned NumOps = I.getNumOperands();
627 for (unsigned i = 1; i < NumOps; ++i)
628 MIB.add(I.getOperand(i));
629 return MIB.constrainAllUses(TII, TRI, RBI);
630 }
631 }
632 return false;
633 }
634
635 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg,
636 const SPIRVType *ResType,
637 MachineInstr &I,
638 Register SrcReg,
639 unsigned Opcode) const {
640 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
641 .addDef(ResVReg)
642 .addUse(GR.getSPIRVTypeID(ResType))
643 .addUse(SrcReg)
644 .constrainAllUses(TII, TRI, RBI);
645 }
646
647 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
648 const SPIRVType *ResType,
649 MachineInstr &I,
650 unsigned Opcode) const {
651 if (STI.isOpenCLEnv() && I.getOperand(1).isReg()) {
652 Register SrcReg = I.getOperand(1).getReg();
653 bool IsGV = false;
654 for (MachineRegisterInfo::def_instr_iterator DefIt =
655 MRI->def_instr_begin(SrcReg);
656 DefIt != MRI->def_instr_end(); DefIt = std::next(DefIt)) {
657 if ((*DefIt).getOpcode() == TargetOpcode::G_GLOBAL_VALUE) {
658 IsGV = true;
659 break;
660 }
661 }
662 if (IsGV) {
663 uint32_t SpecOpcode = 0;
664 switch (Opcode) {
665 case SPIRV::OpConvertPtrToU:
666 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertPtrToU);
667 break;
668 case SPIRV::OpConvertUToPtr:
669 SpecOpcode = static_cast<uint32_t>(SPIRV::Opcode::ConvertUToPtr);
670 break;
671 }
672 if (SpecOpcode)
673 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
674 TII.get(SPIRV::OpSpecConstantOp))
675 .addDef(ResVReg)
676 .addUse(GR.getSPIRVTypeID(ResType))
677 .addImm(SpecOpcode)
678 .addUse(SrcReg)
679 .constrainAllUses(TII, TRI, RBI);
680 }
681 }
682 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(),
683 Opcode);
684 }
685
686 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
687 const SPIRVType *ResType,
688 MachineInstr &I) const {
689 Register OpReg = I.getOperand(1).getReg();
690 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
691 if (!GR.isBitcastCompatible(ResType, OpType))
692 report_fatal_error("incompatible result and operand types in a bitcast");
693 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
694 }
695
696 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord,
697 SPIRVMachineModuleInfo *MMI) {
698 if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID)
699 return SPIRV::Scope::Invocation;
700 else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID)
701 return SPIRV::Scope::Device;
702 else if (Ord == MMI->WorkGroupSSID)
703 return SPIRV::Scope::Workgroup;
704 else if (Ord == MMI->AllSVMDevicesSSID)
705 return SPIRV::Scope::CrossDevice;
706 else if (Ord == MMI->SubGroupSSID)
707 return SPIRV::Scope::Subgroup;
708 else
709 // OpenCL approach is: "The functions that do not have memory_scope argument
710 // have the same semantics as the corresponding functions with the
711 // memory_scope argument set to memory_scope_device." See ref.: //
712 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
713 // In our case if the scope is unknown, assuming that SPIR-V code is to be
714 // consumed in an OpenCL environment, we use the same approach and set the
715 // scope to memory_scope_device.
716 return SPIRV::Scope::Device;
717 }
718
719 static void addMemoryOperands(MachineMemOperand *MemOp,
720 MachineInstrBuilder &MIB) {
721 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
722 if (MemOp->isVolatile())
723 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
724 if (MemOp->isNonTemporal())
725 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
726 if (MemOp->getAlign().value())
727 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
728
729 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
730 MIB.addImm(SpvMemOp);
731 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
732 MIB.addImm(MemOp->getAlign().value());
733 }
734 }
735
736 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
737 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
738 if (Flags & MachineMemOperand::Flags::MOVolatile)
739 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
740 if (Flags & MachineMemOperand::Flags::MONonTemporal)
741 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
742
743 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
744 MIB.addImm(SpvMemOp);
745 }
746
747 bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
748 const SPIRVType *ResType,
749 MachineInstr &I) const {
750 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
751 Register Ptr = I.getOperand(1 + OpOffset).getReg();
752 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
753 .addDef(ResVReg)
754 .addUse(GR.getSPIRVTypeID(ResType))
755 .addUse(Ptr);
756 if (!I.getNumMemOperands()) {
757 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
758 I.getOpcode() ==
759 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
760 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
761 } else {
762 addMemoryOperands(*I.memoperands_begin(), MIB);
763 }
764 return MIB.constrainAllUses(TII, TRI, RBI);
765 }
766
767 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
768 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
769 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
770 Register Ptr = I.getOperand(1 + OpOffset).getReg();
771 MachineBasicBlock &BB = *I.getParent();
772 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
773 .addUse(Ptr)
774 .addUse(StoreVal);
775 if (!I.getNumMemOperands()) {
776 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
777 I.getOpcode() ==
778 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
779 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
780 } else {
781 addMemoryOperands(*I.memoperands_begin(), MIB);
782 }
783 return MIB.constrainAllUses(TII, TRI, RBI);
784 }
785
786 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
787 const SPIRVType *ResType,
788 MachineInstr &I) const {
789 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
790 report_fatal_error(
791 "llvm.stacksave intrinsic: this instruction requires the following "
792 "SPIR-V extension: SPV_INTEL_variable_length_array",
793 false);
794 MachineBasicBlock &BB = *I.getParent();
795 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
796 .addDef(ResVReg)
797 .addUse(GR.getSPIRVTypeID(ResType))
798 .constrainAllUses(TII, TRI, RBI);
799 }
800
801 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
802 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
803 report_fatal_error(
804 "llvm.stackrestore intrinsic: this instruction requires the following "
805 "SPIR-V extension: SPV_INTEL_variable_length_array",
806 false);
807 if (!I.getOperand(0).isReg())
808 return false;
809 MachineBasicBlock &BB = *I.getParent();
810 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
811 .addUse(I.getOperand(0).getReg())
812 .constrainAllUses(TII, TRI, RBI);
813 }
814
815 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
816 MachineInstr &I) const {
817 MachineBasicBlock &BB = *I.getParent();
818 Register SrcReg = I.getOperand(1).getReg();
819 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
820 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
821 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
822 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
823 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
824 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
825 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII);
826 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
827 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
828 // TODO: check if we have such GV, add init, use buildGlobalVariable.
829 Function &CurFunction = GR.CurMF->getFunction();
830 Type *LLVMArrTy =
831 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
832 // Module takes ownership of the global var.
833 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
834 true, GlobalValue::InternalLinkage,
835 Constant::getNullValue(LLVMArrTy));
836 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
837 GR.add(GV, GR.CurMF, VarReg);
838
839 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
840 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
841 .addDef(VarReg)
842 .addUse(GR.getSPIRVTypeID(VarTy))
843 .addImm(SPIRV::StorageClass::UniformConstant)
844 .addUse(Const)
845 .constrainAllUses(TII, TRI, RBI);
846 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
847 ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
848 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
849 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast);
850 }
851 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
852 .addUse(I.getOperand(0).getReg())
853 .addUse(SrcReg)
854 .addUse(I.getOperand(2).getReg());
855 if (I.getNumMemOperands())
856 addMemoryOperands(*I.memoperands_begin(), MIB);
857 bool Result = MIB.constrainAllUses(TII, TRI, RBI);
858 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg())
859 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg)
860 .addUse(MIB->getOperand(0).getReg());
861 return Result;
862 }
863
864 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
865 const SPIRVType *ResType,
866 MachineInstr &I,
867 unsigned NewOpcode,
868 unsigned NegateOpcode) const {
869 assert(I.hasOneMemOperand());
870 const MachineMemOperand *MemOp = *I.memoperands_begin();
871 uint32_t Scope =
872 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI));
873 Register ScopeReg = buildI32Constant(Scope, I);
874
875 Register Ptr = I.getOperand(1).getReg();
876 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
877 // auto ScSem =
878 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
879 AtomicOrdering AO = MemOp->getSuccessOrdering();
880 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
881 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I);
882
883 bool Result = false;
884 Register ValueReg = I.getOperand(2).getReg();
885 if (NegateOpcode != 0) {
886 // Translation with negative value operand is requested
887 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
888 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode);
889 ValueReg = TmpReg;
890 }
891
892 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
893 .addDef(ResVReg)
894 .addUse(GR.getSPIRVTypeID(ResType))
895 .addUse(Ptr)
896 .addUse(ScopeReg)
897 .addUse(MemSemReg)
898 .addUse(ValueReg)
899 .constrainAllUses(TII, TRI, RBI);
900 return Result;
901 }
902
903 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
904 unsigned ArgI = I.getNumOperands() - 1;
905 Register SrcReg =
906 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
907 SPIRVType *DefType =
908 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr;
909 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector)
910 report_fatal_error(
911 "cannot select G_UNMERGE_VALUES with a non-vector argument");
912
913 SPIRVType *ScalarType =
914 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg());
915 MachineBasicBlock &BB = *I.getParent();
916 bool Res = false;
917 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
918 Register ResVReg = I.getOperand(i).getReg();
919 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg);
920 if (!ResType) {
921 // There was no "assign type" actions, let's fix this now
922 ResType = ScalarType;
923 MRI->setRegClass(ResVReg, &SPIRV::IDRegClass);
924 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType)));
925 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
926 }
927 auto MIB =
928 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
929 .addDef(ResVReg)
930 .addUse(GR.getSPIRVTypeID(ResType))
931 .addUse(SrcReg)
932 .addImm(static_cast<int64_t>(i));
933 Res |= MIB.constrainAllUses(TII, TRI, RBI);
934 }
935 return Res;
936 }
937
938 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
939 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
940 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
941 Register MemSemReg = buildI32Constant(MemSem, I);
942 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
943 uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI));
944 Register ScopeReg = buildI32Constant(Scope, I);
945 MachineBasicBlock &BB = *I.getParent();
946 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
947 .addUse(ScopeReg)
948 .addUse(MemSemReg)
949 .constrainAllUses(TII, TRI, RBI);
950 }
951
952 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
953 const SPIRVType *ResType,
954 MachineInstr &I) const {
955 Register ScopeReg;
956 Register MemSemEqReg;
957 Register MemSemNeqReg;
958 Register Ptr = I.getOperand(2).getReg();
959 if (!isa<GIntrinsic>(I)) {
960 assert(I.hasOneMemOperand());
961 const MachineMemOperand *MemOp = *I.memoperands_begin();
962 unsigned Scope =
963 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI));
964 ScopeReg = buildI32Constant(Scope, I);
965
966 unsigned ScSem = static_cast<uint32_t>(
967 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
968 AtomicOrdering AO = MemOp->getSuccessOrdering();
969 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem;
970 MemSemEqReg = buildI32Constant(MemSemEq, I);
971 AtomicOrdering FO = MemOp->getFailureOrdering();
972 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem;
973 MemSemNeqReg =
974 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I);
975 } else {
976 ScopeReg = I.getOperand(5).getReg();
977 MemSemEqReg = I.getOperand(6).getReg();
978 MemSemNeqReg = I.getOperand(7).getReg();
979 }
980
981 Register Cmp = I.getOperand(3).getReg();
982 Register Val = I.getOperand(4).getReg();
983 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val);
984 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass);
985 const DebugLoc &DL = I.getDebugLoc();
986 bool Result =
987 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
988 .addDef(ACmpRes)
989 .addUse(GR.getSPIRVTypeID(SpvValTy))
990 .addUse(Ptr)
991 .addUse(ScopeReg)
992 .addUse(MemSemEqReg)
993 .addUse(MemSemNeqReg)
994 .addUse(Val)
995 .addUse(Cmp)
996 .constrainAllUses(TII, TRI, RBI);
997 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
998 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
999 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
1000 .addDef(CmpSuccReg)
1001 .addUse(GR.getSPIRVTypeID(BoolTy))
1002 .addUse(ACmpRes)
1003 .addUse(Cmp)
1004 .constrainAllUses(TII, TRI, RBI);
1005 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1006 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
1007 .addDef(TmpReg)
1008 .addUse(GR.getSPIRVTypeID(ResType))
1009 .addUse(ACmpRes)
1010 .addUse(GR.getOrCreateUndef(I, ResType, TII))
1011 .addImm(0)
1012 .constrainAllUses(TII, TRI, RBI);
1013 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
1014 .addDef(ResVReg)
1015 .addUse(GR.getSPIRVTypeID(ResType))
1016 .addUse(CmpSuccReg)
1017 .addUse(TmpReg)
1018 .addImm(1)
1019 .constrainAllUses(TII, TRI, RBI);
1020 return Result;
1021 }
1022
1023 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) {
1024 switch (SC) {
1025 case SPIRV::StorageClass::Workgroup:
1026 case SPIRV::StorageClass::CrossWorkgroup:
1027 case SPIRV::StorageClass::Function:
1028 return true;
1029 default:
1030 return false;
1031 }
1032 }
1033
1034 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
1035 switch (SC) {
1036 case SPIRV::StorageClass::DeviceOnlyINTEL:
1037 case SPIRV::StorageClass::HostOnlyINTEL:
1038 return true;
1039 default:
1040 return false;
1041 }
1042 }
1043
1044 // In SPIR-V address space casting can only happen to and from the Generic
1045 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
1046 // pointers to and from Generic pointers. As such, we can convert e.g. from
1047 // Workgroup to Function by going via a Generic pointer as an intermediary. All
1048 // other combinations can only be done by a bitcast, and are probably not safe.
1049 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
1050 const SPIRVType *ResType,
1051 MachineInstr &I) const {
1052 // If the AddrSpaceCast user is single and in OpConstantComposite or
1053 // OpVariable, we should select OpSpecConstantOp.
1054 auto UIs = MRI->use_instructions(ResVReg);
1055 if (!UIs.empty() && ++UIs.begin() == UIs.end() &&
1056 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite ||
1057 UIs.begin()->getOpcode() == SPIRV::OpVariable ||
1058 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) {
1059 Register NewReg = I.getOperand(1).getReg();
1060 MachineBasicBlock &BB = *I.getParent();
1061 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
1062 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII,
1063 SPIRV::StorageClass::Generic);
1064 bool Result =
1065 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1066 .addDef(ResVReg)
1067 .addUse(GR.getSPIRVTypeID(ResType))
1068 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric))
1069 .addUse(NewReg)
1070 .constrainAllUses(TII, TRI, RBI);
1071 return Result;
1072 }
1073 Register SrcPtr = I.getOperand(1).getReg();
1074 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
1075 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr);
1076 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg);
1077
1078 // don't generate a cast between identical storage classes
1079 if (SrcSC == DstSC)
1080 return true;
1081
1082 // Casting from an eligible pointer to Generic.
1083 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
1084 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
1085 // Casting from Generic to an eligible pointer.
1086 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
1087 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
1088 // Casting between 2 eligible pointers using Generic as an intermediary.
1089 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
1090 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1091 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType(
1092 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic);
1093 MachineBasicBlock &BB = *I.getParent();
1094 const DebugLoc &DL = I.getDebugLoc();
1095 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
1096 .addDef(Tmp)
1097 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
1098 .addUse(SrcPtr)
1099 .constrainAllUses(TII, TRI, RBI);
1100 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
1101 .addDef(ResVReg)
1102 .addUse(GR.getSPIRVTypeID(ResType))
1103 .addUse(Tmp)
1104 .constrainAllUses(TII, TRI, RBI);
1105 }
1106
1107 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
1108 // be applied
1109 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
1110 return selectUnOp(ResVReg, ResType, I,
1111 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
1112 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
1113 return selectUnOp(ResVReg, ResType, I,
1114 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
1115
1116 // TODO Should this case just be disallowed completely?
1117 // We're casting 2 other arbitrary address spaces, so have to bitcast.
1118 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
1119 }
1120
1121 static unsigned getFCmpOpcode(unsigned PredNum) {
1122 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1123 switch (Pred) {
1124 case CmpInst::FCMP_OEQ:
1125 return SPIRV::OpFOrdEqual;
1126 case CmpInst::FCMP_OGE:
1127 return SPIRV::OpFOrdGreaterThanEqual;
1128 case CmpInst::FCMP_OGT:
1129 return SPIRV::OpFOrdGreaterThan;
1130 case CmpInst::FCMP_OLE:
1131 return SPIRV::OpFOrdLessThanEqual;
1132 case CmpInst::FCMP_OLT:
1133 return SPIRV::OpFOrdLessThan;
1134 case CmpInst::FCMP_ONE:
1135 return SPIRV::OpFOrdNotEqual;
1136 case CmpInst::FCMP_ORD:
1137 return SPIRV::OpOrdered;
1138 case CmpInst::FCMP_UEQ:
1139 return SPIRV::OpFUnordEqual;
1140 case CmpInst::FCMP_UGE:
1141 return SPIRV::OpFUnordGreaterThanEqual;
1142 case CmpInst::FCMP_UGT:
1143 return SPIRV::OpFUnordGreaterThan;
1144 case CmpInst::FCMP_ULE:
1145 return SPIRV::OpFUnordLessThanEqual;
1146 case CmpInst::FCMP_ULT:
1147 return SPIRV::OpFUnordLessThan;
1148 case CmpInst::FCMP_UNE:
1149 return SPIRV::OpFUnordNotEqual;
1150 case CmpInst::FCMP_UNO:
1151 return SPIRV::OpUnordered;
1152 default:
1153 llvm_unreachable("Unknown predicate type for FCmp");
1154 }
1155 }
1156
1157 static unsigned getICmpOpcode(unsigned PredNum) {
1158 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1159 switch (Pred) {
1160 case CmpInst::ICMP_EQ:
1161 return SPIRV::OpIEqual;
1162 case CmpInst::ICMP_NE:
1163 return SPIRV::OpINotEqual;
1164 case CmpInst::ICMP_SGE:
1165 return SPIRV::OpSGreaterThanEqual;
1166 case CmpInst::ICMP_SGT:
1167 return SPIRV::OpSGreaterThan;
1168 case CmpInst::ICMP_SLE:
1169 return SPIRV::OpSLessThanEqual;
1170 case CmpInst::ICMP_SLT:
1171 return SPIRV::OpSLessThan;
1172 case CmpInst::ICMP_UGE:
1173 return SPIRV::OpUGreaterThanEqual;
1174 case CmpInst::ICMP_UGT:
1175 return SPIRV::OpUGreaterThan;
1176 case CmpInst::ICMP_ULE:
1177 return SPIRV::OpULessThanEqual;
1178 case CmpInst::ICMP_ULT:
1179 return SPIRV::OpULessThan;
1180 default:
1181 llvm_unreachable("Unknown predicate type for ICmp");
1182 }
1183 }
1184
1185 static unsigned getPtrCmpOpcode(unsigned Pred) {
1186 switch (static_cast<CmpInst::Predicate>(Pred)) {
1187 case CmpInst::ICMP_EQ:
1188 return SPIRV::OpPtrEqual;
1189 case CmpInst::ICMP_NE:
1190 return SPIRV::OpPtrNotEqual;
1191 default:
1192 llvm_unreachable("Unknown predicate type for pointer comparison");
1193 }
1194 }
1195
1196 // Return the logical operation, or abort if none exists.
1197 static unsigned getBoolCmpOpcode(unsigned PredNum) {
1198 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1199 switch (Pred) {
1200 case CmpInst::ICMP_EQ:
1201 return SPIRV::OpLogicalEqual;
1202 case CmpInst::ICMP_NE:
1203 return SPIRV::OpLogicalNotEqual;
1204 default:
1205 llvm_unreachable("Unknown predicate type for Bool comparison");
1206 }
1207 }
1208
1209 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
1210 const SPIRVType *ResType,
1211 MachineInstr &I,
1212 unsigned OpAnyOrAll) const {
1213 assert(I.getNumOperands() == 3);
1214 assert(I.getOperand(2).isReg());
1215 MachineBasicBlock &BB = *I.getParent();
1216 Register InputRegister = I.getOperand(2).getReg();
1217 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
1218
1219 if (!InputType)
1220 report_fatal_error("Input Type could not be determined.");
1221
1222 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool);
1223 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
1224 if (IsBoolTy && !IsVectorTy) {
1225 assert(ResVReg == I.getOperand(0).getReg());
1226 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1227 TII.get(TargetOpcode::COPY))
1228 .addDef(ResVReg)
1229 .addUse(InputRegister)
1230 .constrainAllUses(TII, TRI, RBI);
1231 }
1232
1233 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
1234 unsigned SpirvNotEqualId =
1235 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
1236 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
1237 SPIRVType *SpvBoolTy = SpvBoolScalarTy;
1238 Register NotEqualReg = ResVReg;
1239
1240 if (IsVectorTy) {
1241 NotEqualReg = IsBoolTy ? InputRegister
1242 : MRI->createVirtualRegister(&SPIRV::IDRegClass);
1243 const unsigned NumElts = InputType->getOperand(2).getImm();
1244 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
1245 }
1246
1247 if (!IsBoolTy) {
1248 Register ConstZeroReg =
1249 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
1250
1251 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
1252 .addDef(NotEqualReg)
1253 .addUse(GR.getSPIRVTypeID(SpvBoolTy))
1254 .addUse(InputRegister)
1255 .addUse(ConstZeroReg)
1256 .constrainAllUses(TII, TRI, RBI);
1257 }
1258
1259 if (!IsVectorTy)
1260 return true;
1261
1262 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
1263 .addDef(ResVReg)
1264 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
1265 .addUse(NotEqualReg)
1266 .constrainAllUses(TII, TRI, RBI);
1267 }
1268
1269 bool SPIRVInstructionSelector::selectAll(Register ResVReg,
1270 const SPIRVType *ResType,
1271 MachineInstr &I) const {
1272 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
1273 }
1274
1275 bool SPIRVInstructionSelector::selectAny(Register ResVReg,
1276 const SPIRVType *ResType,
1277 MachineInstr &I) const {
1278 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
1279 }
1280
1281 bool SPIRVInstructionSelector::selectFmix(Register ResVReg,
1282 const SPIRVType *ResType,
1283 MachineInstr &I) const {
1284
1285 assert(I.getNumOperands() == 5);
1286 assert(I.getOperand(2).isReg());
1287 assert(I.getOperand(3).isReg());
1288 assert(I.getOperand(4).isReg());
1289 MachineBasicBlock &BB = *I.getParent();
1290
1291 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
1292 .addDef(ResVReg)
1293 .addUse(GR.getSPIRVTypeID(ResType))
1294 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
1295 .addImm(GL::FMix)
1296 .addUse(I.getOperand(2).getReg())
1297 .addUse(I.getOperand(3).getReg())
1298 .addUse(I.getOperand(4).getReg())
1299 .constrainAllUses(TII, TRI, RBI);
1300 }
1301
1302 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
1303 const SPIRVType *ResType,
1304 MachineInstr &I) const {
1305 MachineBasicBlock &BB = *I.getParent();
1306 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
1307 .addDef(ResVReg)
1308 .addUse(GR.getSPIRVTypeID(ResType))
1309 .addUse(I.getOperand(1).getReg())
1310 .constrainAllUses(TII, TRI, RBI);
1311 }
1312
1313 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
1314 const SPIRVType *ResType,
1315 MachineInstr &I) const {
1316 // There is no way to implement `freeze` correctly without support on SPIR-V
1317 // standard side, but we may at least address a simple (static) case when
1318 // undef/poison value presence is obvious. The main benefit of even
1319 // incomplete `freeze` support is preventing of translation from crashing due
1320 // to lack of support on legalization and instruction selection steps.
1321 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg())
1322 return false;
1323 Register OpReg = I.getOperand(1).getReg();
1324 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) {
1325 Register Reg;
1326 switch (Def->getOpcode()) {
1327 case SPIRV::ASSIGN_TYPE:
1328 if (MachineInstr *AssignToDef =
1329 MRI->getVRegDef(Def->getOperand(1).getReg())) {
1330 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1331 Reg = Def->getOperand(2).getReg();
1332 }
1333 break;
1334 case SPIRV::OpUndef:
1335 Reg = Def->getOperand(1).getReg();
1336 break;
1337 }
1338 unsigned DestOpCode;
1339 if (Reg.isValid()) {
1340 DestOpCode = SPIRV::OpConstantNull;
1341 } else {
1342 DestOpCode = TargetOpcode::COPY;
1343 Reg = OpReg;
1344 }
1345 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
1346 .addDef(I.getOperand(0).getReg())
1347 .addUse(Reg)
1348 .constrainAllUses(TII, TRI, RBI);
1349 }
1350 return false;
1351 }
1352
1353 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg,
1354 const SPIRVType *ResType,
1355 MachineInstr &I) const {
1356 // TODO: only const case is supported for now.
1357 assert(std::all_of(
1358 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) {
1359 if (MO.isDef())
1360 return true;
1361 if (!MO.isReg())
1362 return false;
1363 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg());
1364 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE &&
1365 ConstTy->getOperand(1).isReg());
1366 Register ConstReg = ConstTy->getOperand(1).getReg();
1367 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg);
1368 assert(Const);
1369 return (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
1370 Const->getOpcode() == TargetOpcode::G_FCONSTANT);
1371 }));
1372
1373 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1374 TII.get(SPIRV::OpConstantComposite))
1375 .addDef(ResVReg)
1376 .addUse(GR.getSPIRVTypeID(ResType));
1377 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
1378 MIB.addUse(I.getOperand(i).getReg());
1379 return MIB.constrainAllUses(TII, TRI, RBI);
1380 }
1381
1382 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI,
1383 const SPIRVType *ResType) {
1384 Register OpReg = ResType->getOperand(2).getReg();
1385 SPIRVType *OpDef = MRI->getVRegDef(OpReg);
1386 if (!OpDef)
1387 return 0;
1388 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
1389 OpDef->getOperand(1).isReg()) {
1390 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
1391 OpDef = RefDef;
1392 }
1393 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT
1394 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue()
1395 : 0;
1396 return N;
1397 }
1398
1399 // Return true if the type represents a constant register
1400 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) {
1401 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
1402 OpDef->getOperand(1).isReg()) {
1403 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
1404 OpDef = RefDef;
1405 }
1406 return OpDef->getOpcode() == TargetOpcode::G_CONSTANT ||
1407 OpDef->getOpcode() == TargetOpcode::G_FCONSTANT;
1408 }
1409
1410 // Return true if the virtual register represents a constant
1411 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
1412 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg))
1413 return isConstReg(MRI, OpDef);
1414 return false;
1415 }
1416
1417 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
1418 const SPIRVType *ResType,
1419 MachineInstr &I) const {
1420 unsigned N = 0;
1421 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1422 N = GR.getScalarOrVectorComponentCount(ResType);
1423 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
1424 N = getArrayComponentCount(MRI, ResType);
1425 else
1426 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
1427
1428 unsigned OpIdx = I.getNumExplicitDefs();
1429 if (!I.getOperand(OpIdx).isReg())
1430 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
1431
1432 // check if we may construct a constant vector
1433 Register OpReg = I.getOperand(OpIdx).getReg();
1434 bool IsConst = isConstReg(MRI, OpReg);
1435
1436 if (!IsConst && N < 2)
1437 report_fatal_error(
1438 "There must be at least two constituent operands in a vector");
1439
1440 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1441 TII.get(IsConst ? SPIRV::OpConstantComposite
1442 : SPIRV::OpCompositeConstruct))
1443 .addDef(ResVReg)
1444 .addUse(GR.getSPIRVTypeID(ResType));
1445 for (unsigned i = 0; i < N; ++i)
1446 MIB.addUse(OpReg);
1447 return MIB.constrainAllUses(TII, TRI, RBI);
1448 }
1449
1450 bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
1451 const SPIRVType *ResType,
1452 unsigned CmpOpc,
1453 MachineInstr &I) const {
1454 Register Cmp0 = I.getOperand(2).getReg();
1455 Register Cmp1 = I.getOperand(3).getReg();
1456 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
1457 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
1458 "CMP operands should have the same type");
1459 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
1460 .addDef(ResVReg)
1461 .addUse(GR.getSPIRVTypeID(ResType))
1462 .addUse(Cmp0)
1463 .addUse(Cmp1)
1464 .constrainAllUses(TII, TRI, RBI);
1465 }
1466
1467 bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
1468 const SPIRVType *ResType,
1469 MachineInstr &I) const {
1470 auto Pred = I.getOperand(1).getPredicate();
1471 unsigned CmpOpc;
1472
1473 Register CmpOperand = I.getOperand(2).getReg();
1474 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer))
1475 CmpOpc = getPtrCmpOpcode(Pred);
1476 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
1477 CmpOpc = getBoolCmpOpcode(Pred);
1478 else
1479 CmpOpc = getICmpOpcode(Pred);
1480 return selectCmp(ResVReg, ResType, CmpOpc, I);
1481 }
1482
1483 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB,
1484 const MachineInstr &I,
1485 int OpIdx) const {
1486 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
1487 "Expected G_FCONSTANT");
1488 const ConstantFP *FPImm = I.getOperand(1).getFPImm();
1489 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB);
1490 }
1491
1492 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB,
1493 const MachineInstr &I,
1494 int OpIdx) const {
1495 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1496 "Expected G_CONSTANT");
1497 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB);
1498 }
1499
1500 Register
1501 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
1502 const SPIRVType *ResType) const {
1503 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
1504 const SPIRVType *SpvI32Ty =
1505 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
1506 // Find a constant in DT or build a new one.
1507 auto ConstInt = ConstantInt::get(LLVMTy, Val);
1508 Register NewReg = GR.find(ConstInt, GR.CurMF);
1509 if (!NewReg.isValid()) {
1510 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
1511 GR.add(ConstInt, GR.CurMF, NewReg);
1512 MachineInstr *MI;
1513 MachineBasicBlock &BB = *I.getParent();
1514 if (Val == 0) {
1515 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1516 .addDef(NewReg)
1517 .addUse(GR.getSPIRVTypeID(SpvI32Ty));
1518 } else {
1519 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1520 .addDef(NewReg)
1521 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
1522 .addImm(APInt(32, Val).getZExtValue());
1523 }
1524 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
1525 }
1526 return NewReg;
1527 }
1528
1529 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
1530 const SPIRVType *ResType,
1531 MachineInstr &I) const {
1532 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
1533 return selectCmp(ResVReg, ResType, CmpOp, I);
1534 }
1535
1536 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
1537 MachineInstr &I) const {
1538 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
1539 bool ZeroAsNull = STI.isOpenCLEnv();
1540 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1541 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
1542 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
1543 }
1544
1545 static APFloat getZeroFP(const Type *LLVMFloatTy) {
1546 if (!LLVMFloatTy)
1547 return APFloat::getZero(APFloat::IEEEsingle());
1548 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
1549 case Type::HalfTyID:
1550 return APFloat::getZero(APFloat::IEEEhalf());
1551 default:
1552 case Type::FloatTyID:
1553 return APFloat::getZero(APFloat::IEEEsingle());
1554 case Type::DoubleTyID:
1555 return APFloat::getZero(APFloat::IEEEdouble());
1556 }
1557 }
1558
1559 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
1560 MachineInstr &I) const {
1561 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
1562 bool ZeroAsNull = STI.isOpenCLEnv();
1563 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
1564 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1565 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
1566 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
1567 }
1568
1569 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
1570 const SPIRVType *ResType,
1571 MachineInstr &I) const {
1572 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1573 APInt One =
1574 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
1575 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1576 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII);
1577 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII);
1578 }
1579
1580 bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
1581 const SPIRVType *ResType,
1582 MachineInstr &I,
1583 bool IsSigned) const {
1584 // To extend a bool, we need to use OpSelect between constants.
1585 Register ZeroReg = buildZerosVal(ResType, I);
1586 Register OneReg = buildOnesVal(IsSigned, ResType, I);
1587 bool IsScalarBool =
1588 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
1589 unsigned Opcode =
1590 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond;
1591 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1592 .addDef(ResVReg)
1593 .addUse(GR.getSPIRVTypeID(ResType))
1594 .addUse(I.getOperand(1).getReg())
1595 .addUse(OneReg)
1596 .addUse(ZeroReg)
1597 .constrainAllUses(TII, TRI, RBI);
1598 }
1599
1600 bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
1601 const SPIRVType *ResType,
1602 MachineInstr &I, bool IsSigned,
1603 unsigned Opcode) const {
1604 Register SrcReg = I.getOperand(1).getReg();
1605 // We can convert bool value directly to float type without OpConvert*ToF,
1606 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
1607 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
1608 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1609 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1610 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
1611 const unsigned NumElts = ResType->getOperand(2).getImm();
1612 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
1613 }
1614 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1615 selectSelect(SrcReg, TmpType, I, false);
1616 }
1617 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode);
1618 }
1619
1620 bool SPIRVInstructionSelector::selectExt(Register ResVReg,
1621 const SPIRVType *ResType,
1622 MachineInstr &I, bool IsSigned) const {
1623 Register SrcReg = I.getOperand(1).getReg();
1624 if (GR.isScalarOrVectorOfType(SrcReg, SPIRV::OpTypeBool))
1625 return selectSelect(ResVReg, ResType, I, IsSigned);
1626
1627 SPIRVType *SrcType = GR.getSPIRVTypeForVReg(SrcReg);
1628 if (SrcType == ResType)
1629 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1630 TII.get(TargetOpcode::COPY))
1631 .addDef(ResVReg)
1632 .addUse(SrcReg)
1633 .constrainAllUses(TII, TRI, RBI);
1634
1635 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1636 return selectUnOp(ResVReg, ResType, I, Opcode);
1637 }
1638
1639 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
1640 Register ResVReg,
1641 MachineInstr &I,
1642 const SPIRVType *IntTy,
1643 const SPIRVType *BoolTy) const {
1644 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
1645 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1646 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
1647 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
1648 Register Zero = buildZerosVal(IntTy, I);
1649 Register One = buildOnesVal(false, IntTy, I);
1650 MachineBasicBlock &BB = *I.getParent();
1651 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1652 .addDef(BitIntReg)
1653 .addUse(GR.getSPIRVTypeID(IntTy))
1654 .addUse(IntReg)
1655 .addUse(One)
1656 .constrainAllUses(TII, TRI, RBI);
1657 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
1658 .addDef(ResVReg)
1659 .addUse(GR.getSPIRVTypeID(BoolTy))
1660 .addUse(BitIntReg)
1661 .addUse(Zero)
1662 .constrainAllUses(TII, TRI, RBI);
1663 }
1664
1665 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
1666 const SPIRVType *ResType,
1667 MachineInstr &I) const {
1668 Register IntReg = I.getOperand(1).getReg();
1669 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
1670 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool))
1671 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
1672 if (ArgType == ResType)
1673 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1674 TII.get(TargetOpcode::COPY))
1675 .addDef(ResVReg)
1676 .addUse(IntReg)
1677 .constrainAllUses(TII, TRI, RBI);
1678 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1679 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1680 return selectUnOp(ResVReg, ResType, I, Opcode);
1681 }
1682
1683 bool SPIRVInstructionSelector::selectConst(Register ResVReg,
1684 const SPIRVType *ResType,
1685 const APInt &Imm,
1686 MachineInstr &I) const {
1687 unsigned TyOpcode = ResType->getOpcode();
1688 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero());
1689 MachineBasicBlock &BB = *I.getParent();
1690 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) &&
1691 Imm.isZero())
1692 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1693 .addDef(ResVReg)
1694 .addUse(GR.getSPIRVTypeID(ResType))
1695 .constrainAllUses(TII, TRI, RBI);
1696 if (TyOpcode == SPIRV::OpTypeInt) {
1697 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!");
1698 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
1699 if (Reg == ResVReg)
1700 return true;
1701 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1702 .addDef(ResVReg)
1703 .addUse(Reg)
1704 .constrainAllUses(TII, TRI, RBI);
1705 }
1706 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1707 .addDef(ResVReg)
1708 .addUse(GR.getSPIRVTypeID(ResType));
1709 // <=32-bit integers should be caught by the sdag pattern.
1710 assert(Imm.getBitWidth() > 32);
1711 addNumImm(Imm, MIB);
1712 return MIB.constrainAllUses(TII, TRI, RBI);
1713 }
1714
1715 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
1716 const SPIRVType *ResType,
1717 MachineInstr &I) const {
1718 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1719 .addDef(ResVReg)
1720 .addUse(GR.getSPIRVTypeID(ResType))
1721 .constrainAllUses(TII, TRI, RBI);
1722 }
1723
1724 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1725 assert(MO.isReg());
1726 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1727 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE)
1728 return false;
1729 assert(TypeInst->getOperand(1).isReg());
1730 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1731 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT;
1732 }
1733
1734 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1735 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1736 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1737 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT);
1738 return ImmInst->getOperand(1).getCImm()->getZExtValue();
1739 }
1740
1741 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
1742 const SPIRVType *ResType,
1743 MachineInstr &I) const {
1744 MachineBasicBlock &BB = *I.getParent();
1745 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
1746 .addDef(ResVReg)
1747 .addUse(GR.getSPIRVTypeID(ResType))
1748 // object to insert
1749 .addUse(I.getOperand(3).getReg())
1750 // composite to insert into
1751 .addUse(I.getOperand(2).getReg());
1752 for (unsigned i = 4; i < I.getNumOperands(); i++)
1753 MIB.addImm(foldImm(I.getOperand(i), MRI));
1754 return MIB.constrainAllUses(TII, TRI, RBI);
1755 }
1756
1757 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
1758 const SPIRVType *ResType,
1759 MachineInstr &I) const {
1760 MachineBasicBlock &BB = *I.getParent();
1761 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1762 .addDef(ResVReg)
1763 .addUse(GR.getSPIRVTypeID(ResType))
1764 .addUse(I.getOperand(2).getReg());
1765 for (unsigned i = 3; i < I.getNumOperands(); i++)
1766 MIB.addImm(foldImm(I.getOperand(i), MRI));
1767 return MIB.constrainAllUses(TII, TRI, RBI);
1768 }
1769
1770 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
1771 const SPIRVType *ResType,
1772 MachineInstr &I) const {
1773 if (isImm(I.getOperand(4), MRI))
1774 return selectInsertVal(ResVReg, ResType, I);
1775 MachineBasicBlock &BB = *I.getParent();
1776 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
1777 .addDef(ResVReg)
1778 .addUse(GR.getSPIRVTypeID(ResType))
1779 .addUse(I.getOperand(2).getReg())
1780 .addUse(I.getOperand(3).getReg())
1781 .addUse(I.getOperand(4).getReg())
1782 .constrainAllUses(TII, TRI, RBI);
1783 }
1784
1785 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
1786 const SPIRVType *ResType,
1787 MachineInstr &I) const {
1788 if (isImm(I.getOperand(3), MRI))
1789 return selectExtractVal(ResVReg, ResType, I);
1790 MachineBasicBlock &BB = *I.getParent();
1791 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
1792 .addDef(ResVReg)
1793 .addUse(GR.getSPIRVTypeID(ResType))
1794 .addUse(I.getOperand(2).getReg())
1795 .addUse(I.getOperand(3).getReg())
1796 .constrainAllUses(TII, TRI, RBI);
1797 }
1798
1799 bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
1800 const SPIRVType *ResType,
1801 MachineInstr &I) const {
1802 const bool IsGEPInBounds = I.getOperand(2).getImm();
1803
1804 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
1805 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
1806 // we have to use Op[InBounds]AccessChain.
1807 const unsigned Opcode = STI.isVulkanEnv()
1808 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
1809 : SPIRV::OpAccessChain)
1810 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
1811 : SPIRV::OpPtrAccessChain);
1812
1813 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1814 .addDef(ResVReg)
1815 .addUse(GR.getSPIRVTypeID(ResType))
1816 // Object to get a pointer to.
1817 .addUse(I.getOperand(3).getReg());
1818 // Adding indices.
1819 const unsigned StartingIndex =
1820 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
1821 ? 5
1822 : 4;
1823 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
1824 Res.addUse(I.getOperand(i).getReg());
1825 return Res.constrainAllUses(TII, TRI, RBI);
1826 }
1827
1828 // Maybe wrap a value into OpSpecConstantOp
1829 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
1830 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
1831 bool Result = true;
1832 unsigned Lim = I.getNumExplicitOperands();
1833 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
1834 Register OpReg = I.getOperand(i).getReg();
1835 SPIRVType *OpDefine = MRI->getVRegDef(OpReg);
1836 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg);
1837 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) ||
1838 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) {
1839 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
1840 // by selectAddrSpaceCast()
1841 CompositeArgs.push_back(OpReg);
1842 continue;
1843 }
1844 MachineFunction *MF = I.getMF();
1845 Register WrapReg = GR.find(OpDefine, MF);
1846 if (WrapReg.isValid()) {
1847 CompositeArgs.push_back(WrapReg);
1848 continue;
1849 }
1850 // Create a new register for the wrapper
1851 WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1852 GR.add(OpDefine, MF, WrapReg);
1853 CompositeArgs.push_back(WrapReg);
1854 // Decorate the wrapper register and generate a new instruction
1855 MRI->setType(WrapReg, LLT::pointer(0, 32));
1856 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF);
1857 MachineBasicBlock &BB = *I.getParent();
1858 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1859 .addDef(WrapReg)
1860 .addUse(GR.getSPIRVTypeID(OpType))
1861 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
1862 .addUse(OpReg)
1863 .constrainAllUses(TII, TRI, RBI);
1864 if (!Result)
1865 break;
1866 }
1867 return Result;
1868 }
1869
1870 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
1871 const SPIRVType *ResType,
1872 MachineInstr &I) const {
1873 MachineBasicBlock &BB = *I.getParent();
1874 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
1875 switch (IID) {
1876 case Intrinsic::spv_load:
1877 return selectLoad(ResVReg, ResType, I);
1878 case Intrinsic::spv_store:
1879 return selectStore(I);
1880 case Intrinsic::spv_extractv:
1881 return selectExtractVal(ResVReg, ResType, I);
1882 case Intrinsic::spv_insertv:
1883 return selectInsertVal(ResVReg, ResType, I);
1884 case Intrinsic::spv_extractelt:
1885 return selectExtractElt(ResVReg, ResType, I);
1886 case Intrinsic::spv_insertelt:
1887 return selectInsertElt(ResVReg, ResType, I);
1888 case Intrinsic::spv_gep:
1889 return selectGEP(ResVReg, ResType, I);
1890 case Intrinsic::spv_unref_global:
1891 case Intrinsic::spv_init_global: {
1892 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
1893 MachineInstr *Init = I.getNumExplicitOperands() > 2
1894 ? MRI->getVRegDef(I.getOperand(2).getReg())
1895 : nullptr;
1896 assert(MI);
1897 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init);
1898 }
1899 case Intrinsic::spv_undef: {
1900 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1901 .addDef(ResVReg)
1902 .addUse(GR.getSPIRVTypeID(ResType));
1903 return MIB.constrainAllUses(TII, TRI, RBI);
1904 }
1905 case Intrinsic::spv_const_composite: {
1906 // If no values are attached, the composite is null constant.
1907 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
1908 // Select a proper instruction.
1909 unsigned Opcode = SPIRV::OpConstantNull;
1910 SmallVector<Register> CompositeArgs;
1911 if (!IsNull) {
1912 Opcode = SPIRV::OpConstantComposite;
1913 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
1914 return false;
1915 }
1916 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1917 .addDef(ResVReg)
1918 .addUse(GR.getSPIRVTypeID(ResType));
1919 // skip type MD node we already used when generated assign.type for this
1920 if (!IsNull) {
1921 for (Register OpReg : CompositeArgs)
1922 MIB.addUse(OpReg);
1923 }
1924 return MIB.constrainAllUses(TII, TRI, RBI);
1925 }
1926 case Intrinsic::spv_assign_name: {
1927 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
1928 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
1929 for (unsigned i = I.getNumExplicitDefs() + 2;
1930 i < I.getNumExplicitOperands(); ++i) {
1931 MIB.addImm(I.getOperand(i).getImm());
1932 }
1933 return MIB.constrainAllUses(TII, TRI, RBI);
1934 }
1935 case Intrinsic::spv_switch: {
1936 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
1937 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
1938 if (I.getOperand(i).isReg())
1939 MIB.addReg(I.getOperand(i).getReg());
1940 else if (I.getOperand(i).isCImm())
1941 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
1942 else if (I.getOperand(i).isMBB())
1943 MIB.addMBB(I.getOperand(i).getMBB());
1944 else
1945 llvm_unreachable("Unexpected OpSwitch operand");
1946 }
1947 return MIB.constrainAllUses(TII, TRI, RBI);
1948 }
1949 case Intrinsic::spv_cmpxchg:
1950 return selectAtomicCmpXchg(ResVReg, ResType, I);
1951 case Intrinsic::spv_unreachable:
1952 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable));
1953 break;
1954 case Intrinsic::spv_alloca:
1955 return selectFrameIndex(ResVReg, ResType, I);
1956 case Intrinsic::spv_alloca_array:
1957 return selectAllocaArray(ResVReg, ResType, I);
1958 case Intrinsic::spv_assume:
1959 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1960 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
1961 .addUse(I.getOperand(1).getReg());
1962 break;
1963 case Intrinsic::spv_expect:
1964 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1965 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
1966 .addDef(ResVReg)
1967 .addUse(GR.getSPIRVTypeID(ResType))
1968 .addUse(I.getOperand(2).getReg())
1969 .addUse(I.getOperand(3).getReg());
1970 break;
1971 case Intrinsic::spv_thread_id:
1972 return selectSpvThreadId(ResVReg, ResType, I);
1973 case Intrinsic::spv_all:
1974 return selectAll(ResVReg, ResType, I);
1975 case Intrinsic::spv_any:
1976 return selectAny(ResVReg, ResType, I);
1977 case Intrinsic::spv_lerp:
1978 return selectFmix(ResVReg, ResType, I);
1979 case Intrinsic::spv_lifetime_start:
1980 case Intrinsic::spv_lifetime_end: {
1981 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
1982 : SPIRV::OpLifetimeStop;
1983 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
1984 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
1985 unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg);
1986 bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid;
1987 if (Size == -1 || IsNonvoidPtr)
1988 Size = 0;
1989 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size);
1990 } break;
1991 default: {
1992 std::string DiagMsg;
1993 raw_string_ostream OS(DiagMsg);
1994 I.print(OS);
1995 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
1996 report_fatal_error(DiagMsg.c_str(), false);
1997 }
1998 }
1999 return true;
2000 }
2001
2002 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
2003 const SPIRVType *ResType,
2004 MachineInstr &I) const {
2005 // there was an allocation size parameter to the allocation instruction
2006 // that is not 1
2007 MachineBasicBlock &BB = *I.getParent();
2008 return BuildMI(BB, I, I.getDebugLoc(),
2009 TII.get(SPIRV::OpVariableLengthArrayINTEL))
2010 .addDef(ResVReg)
2011 .addUse(GR.getSPIRVTypeID(ResType))
2012 .addUse(I.getOperand(2).getReg())
2013 .constrainAllUses(TII, TRI, RBI);
2014 }
2015
2016 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
2017 const SPIRVType *ResType,
2018 MachineInstr &I) const {
2019 // Change order of instructions if needed: all OpVariable instructions in a
2020 // function must be the first instructions in the first block
2021 MachineFunction *MF = I.getParent()->getParent();
2022 MachineBasicBlock *MBB = &MF->front();
2023 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end();
2024 bool IsHeader = false;
2025 unsigned Opcode;
2026 for (; It != E && It != I; ++It) {
2027 Opcode = It->getOpcode();
2028 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
2029 IsHeader = true;
2030 } else if (IsHeader &&
2031 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
2032 ++It;
2033 break;
2034 }
2035 }
2036 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
2037 .addDef(ResVReg)
2038 .addUse(GR.getSPIRVTypeID(ResType))
2039 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
2040 .constrainAllUses(TII, TRI, RBI);
2041 }
2042
2043 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
2044 // InstructionSelector walks backwards through the instructions. We can use
2045 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
2046 // first, so can generate an OpBranchConditional here. If there is no
2047 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
2048 const MachineInstr *PrevI = I.getPrevNode();
2049 MachineBasicBlock &MBB = *I.getParent();
2050 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
2051 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
2052 .addUse(PrevI->getOperand(0).getReg())
2053 .addMBB(PrevI->getOperand(1).getMBB())
2054 .addMBB(I.getOperand(0).getMBB())
2055 .constrainAllUses(TII, TRI, RBI);
2056 }
2057 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
2058 .addMBB(I.getOperand(0).getMBB())
2059 .constrainAllUses(TII, TRI, RBI);
2060 }
2061
2062 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
2063 // InstructionSelector walks backwards through the instructions. For an
2064 // explicit conditional branch with no fallthrough, we use both a G_BR and a
2065 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
2066 // generate the OpBranchConditional in selectBranch above.
2067 //
2068 // If an OpBranchConditional has been generated, we simply return, as the work
2069 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
2070 // implicit fallthrough to the next basic block, so we need to create an
2071 // OpBranchConditional with an explicit "false" argument pointing to the next
2072 // basic block that LLVM would fall through to.
2073 const MachineInstr *NextI = I.getNextNode();
2074 // Check if this has already been successfully selected.
2075 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
2076 return true;
2077 // Must be relying on implicit block fallthrough, so generate an
2078 // OpBranchConditional with the "next" basic block as the "false" target.
2079 MachineBasicBlock &MBB = *I.getParent();
2080 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
2081 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
2082 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
2083 .addUse(I.getOperand(0).getReg())
2084 .addMBB(I.getOperand(1).getMBB())
2085 .addMBB(NextMBB)
2086 .constrainAllUses(TII, TRI, RBI);
2087 }
2088
2089 bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
2090 const SPIRVType *ResType,
2091 MachineInstr &I) const {
2092 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi))
2093 .addDef(ResVReg)
2094 .addUse(GR.getSPIRVTypeID(ResType));
2095 const unsigned NumOps = I.getNumOperands();
2096 for (unsigned i = 1; i < NumOps; i += 2) {
2097 MIB.addUse(I.getOperand(i + 0).getReg());
2098 MIB.addMBB(I.getOperand(i + 1).getMBB());
2099 }
2100 return MIB.constrainAllUses(TII, TRI, RBI);
2101 }
2102
2103 bool SPIRVInstructionSelector::selectGlobalValue(
2104 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
2105 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
2106 MachineIRBuilder MIRBuilder(I);
2107 const GlobalValue *GV = I.getOperand(1).getGlobal();
2108 Type *GVType = GR.getDeducedGlobalValueType(GV);
2109 SPIRVType *PointerBaseType;
2110 if (GVType->isArrayTy()) {
2111 SPIRVType *ArrayElementType =
2112 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder,
2113 SPIRV::AccessQualifier::ReadWrite, false);
2114 PointerBaseType = GR.getOrCreateSPIRVArrayType(
2115 ArrayElementType, GVType->getArrayNumElements(), I, TII);
2116 } else {
2117 PointerBaseType = GR.getOrCreateSPIRVType(
2118 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
2119 }
2120 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
2121 PointerBaseType, I, TII,
2122 addressSpaceToStorageClass(GV->getAddressSpace(), STI));
2123
2124 std::string GlobalIdent;
2125 if (!GV->hasName()) {
2126 unsigned &ID = UnnamedGlobalIDs[GV];
2127 if (ID == 0)
2128 ID = UnnamedGlobalIDs.size();
2129 GlobalIdent = "__unnamed_" + Twine(ID).str();
2130 } else {
2131 GlobalIdent = GV->getGlobalIdentifier();
2132 }
2133
2134 // Behaviour of functions as operands depends on availability of the
2135 // corresponding extension (SPV_INTEL_function_pointers):
2136 // - If there is an extension to operate with functions as operands:
2137 // We create a proper constant operand and evaluate a correct type for a
2138 // function pointer.
2139 // - Without the required extension:
2140 // We have functions as operands in tests with blocks of instruction e.g. in
2141 // transcoding/global_block.ll. These operands are not used and should be
2142 // substituted by zero constants. Their type is expected to be always
2143 // OpTypePointer Function %uchar.
2144 if (isa<Function>(GV)) {
2145 const Constant *ConstVal = GV;
2146 MachineBasicBlock &BB = *I.getParent();
2147 Register NewReg = GR.find(ConstVal, GR.CurMF);
2148 if (!NewReg.isValid()) {
2149 Register NewReg = ResVReg;
2150 GR.add(ConstVal, GR.CurMF, NewReg);
2151 const Function *GVFun =
2152 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)
2153 ? dyn_cast<Function>(GV)
2154 : nullptr;
2155 if (GVFun) {
2156 // References to a function via function pointers generate virtual
2157 // registers without a definition. We will resolve it later, during
2158 // module analysis stage.
2159 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2160 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
2161 MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass);
2162 MachineInstrBuilder MB =
2163 BuildMI(BB, I, I.getDebugLoc(),
2164 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
2165 .addDef(NewReg)
2166 .addUse(GR.getSPIRVTypeID(ResType))
2167 .addUse(FuncVReg);
2168 // mapping the function pointer to the used Function
2169 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun);
2170 return MB.constrainAllUses(TII, TRI, RBI);
2171 }
2172 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
2173 .addDef(NewReg)
2174 .addUse(GR.getSPIRVTypeID(ResType))
2175 .constrainAllUses(TII, TRI, RBI);
2176 }
2177 assert(NewReg != ResVReg);
2178 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
2179 .addDef(ResVReg)
2180 .addUse(NewReg)
2181 .constrainAllUses(TII, TRI, RBI);
2182 }
2183 auto GlobalVar = cast<GlobalVariable>(GV);
2184 assert(GlobalVar->getName() != "llvm.global.annotations");
2185
2186 bool HasInit = GlobalVar->hasInitializer() &&
2187 !isa<UndefValue>(GlobalVar->getInitializer());
2188 // Skip empty declaration for GVs with initilaizers till we get the decl with
2189 // passed initializer.
2190 if (HasInit && !Init)
2191 return true;
2192
2193 unsigned AddrSpace = GV->getAddressSpace();
2194 SPIRV::StorageClass::StorageClass Storage =
2195 addressSpaceToStorageClass(AddrSpace, STI);
2196 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage &&
2197 Storage != SPIRV::StorageClass::Function;
2198 SPIRV::LinkageType::LinkageType LnkType =
2199 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
2200 ? SPIRV::LinkageType::Import
2201 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage &&
2202 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr)
2203 ? SPIRV::LinkageType::LinkOnceODR
2204 : SPIRV::LinkageType::Export);
2205
2206 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV,
2207 Storage, Init, GlobalVar->isConstant(),
2208 HasLnkTy, LnkType, MIRBuilder, true);
2209 return Reg.isValid();
2210 }
2211
2212 bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
2213 const SPIRVType *ResType,
2214 MachineInstr &I) const {
2215 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
2216 return selectExtInst(ResVReg, ResType, I, CL::log10);
2217 }
2218
2219 // There is no log10 instruction in the GLSL Extended Instruction set, so it
2220 // is implemented as:
2221 // log10(x) = log2(x) * (1 / log2(10))
2222 // = log2(x) * 0.30103
2223
2224 MachineIRBuilder MIRBuilder(I);
2225 MachineBasicBlock &BB = *I.getParent();
2226
2227 // Build log2(x).
2228 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2229 bool Result =
2230 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
2231 .addDef(VarReg)
2232 .addUse(GR.getSPIRVTypeID(ResType))
2233 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2234 .addImm(GL::Log2)
2235 .add(I.getOperand(1))
2236 .constrainAllUses(TII, TRI, RBI);
2237
2238 // Build 0.30103.
2239 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
2240 ResType->getOpcode() == SPIRV::OpTypeFloat);
2241 // TODO: Add matrix implementation once supported by the HLSL frontend.
2242 const SPIRVType *SpirvScalarType =
2243 ResType->getOpcode() == SPIRV::OpTypeVector
2244 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg())
2245 : ResType;
2246 Register ScaleReg =
2247 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType);
2248
2249 // Multiply log2(x) by 0.30103 to get log10(x) result.
2250 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
2251 ? SPIRV::OpVectorTimesScalar
2252 : SPIRV::OpFMulS;
2253 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2254 .addDef(ResVReg)
2255 .addUse(GR.getSPIRVTypeID(ResType))
2256 .addUse(VarReg)
2257 .addUse(ScaleReg)
2258 .constrainAllUses(TII, TRI, RBI);
2259
2260 return Result;
2261 }
2262
2263 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg,
2264 const SPIRVType *ResType,
2265 MachineInstr &I) const {
2266 // DX intrinsic: @llvm.dx.thread.id(i32)
2267 // ID Name Description
2268 // 93 ThreadId reads the thread ID
2269
2270 MachineIRBuilder MIRBuilder(I);
2271 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
2272 const SPIRVType *Vec3Ty =
2273 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder);
2274 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
2275 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
2276
2277 // Create new register for GlobalInvocationID builtin variable.
2278 Register NewRegister =
2279 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass);
2280 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32));
2281 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
2282
2283 // Build GlobalInvocationID global variable with the necessary decorations.
2284 Register Variable = GR.buildGlobalVariable(
2285 NewRegister, PtrType,
2286 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr,
2287 SPIRV::StorageClass::Input, nullptr, true, true,
2288 SPIRV::LinkageType::Import, MIRBuilder, false);
2289
2290 // Create new register for loading value.
2291 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2292 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2293 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32));
2294 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF());
2295
2296 // Load v3uint value from the global variable.
2297 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
2298 .addDef(LoadedRegister)
2299 .addUse(GR.getSPIRVTypeID(Vec3Ty))
2300 .addUse(Variable);
2301
2302 // Get Thread ID index. Expecting operand is a constant immediate value,
2303 // wrapped in a type assignment.
2304 assert(I.getOperand(2).isReg());
2305 Register ThreadIdReg = I.getOperand(2).getReg();
2306 SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg);
2307 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE &&
2308 ConstTy->getOperand(1).isReg());
2309 Register ConstReg = ConstTy->getOperand(1).getReg();
2310 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg);
2311 assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT);
2312 const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue();
2313 const uint32_t ThreadId = Val.getZExtValue();
2314
2315 // Extract the thread ID from the loaded vector value.
2316 MachineBasicBlock &BB = *I.getParent();
2317 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2318 .addDef(ResVReg)
2319 .addUse(GR.getSPIRVTypeID(ResType))
2320 .addUse(LoadedRegister)
2321 .addImm(ThreadId);
2322 return MIB.constrainAllUses(TII, TRI, RBI);
2323 }
2324
2325 namespace llvm {
2326 InstructionSelector *
2327 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
2328 const SPIRVSubtarget &Subtarget,
2329 const RegisterBankInfo &RBI) {
2330 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
2331 }
2332 } // namespace llvm
LLVM monorepo