?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
[HLSL][SPIRV] Add any intrinsic lowering (#88325)
[llvm-project.git] / llvm / lib / Target / SPIRV / SPIRVInstructionSelector.cpp
blob200fe38298c0211ab1ac75a5c1f99be327e57890
1 //===- SPIRVInstructionSelector.cpp ------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the targeting of the InstructionSelector class for
10 // SPIRV.
11 // TODO: This should be generated by TableGen.
12 //
13 //===----------------------------------------------------------------------===//
14
15 #include "MCTargetDesc/SPIRVBaseInfo.h"
16 #include "MCTargetDesc/SPIRVMCTargetDesc.h"
17 #include "SPIRV.h"
18 #include "SPIRVGlobalRegistry.h"
19 #include "SPIRVInstrInfo.h"
20 #include "SPIRVRegisterBankInfo.h"
21 #include "SPIRVRegisterInfo.h"
22 #include "SPIRVTargetMachine.h"
23 #include "SPIRVUtils.h"
24 #include "llvm/ADT/APFloat.h"
25 #include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h"
26 #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
27 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
28 #include "llvm/CodeGen/MachineInstrBuilder.h"
29 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
30 #include "llvm/CodeGen/MachineRegisterInfo.h"
31 #include "llvm/CodeGen/TargetOpcodes.h"
32 #include "llvm/IR/IntrinsicsSPIRV.h"
33 #include "llvm/Support/Debug.h"
34
35 namespace llvm {
36
37 class SPIRVMachineModuleInfo : public MachineModuleInfoImpl {
38 public:
39 SyncScope::ID Work_ItemSSID;
40 SyncScope::ID WorkGroupSSID;
41 SyncScope::ID DeviceSSID;
42 SyncScope::ID AllSVMDevicesSSID;
43 SyncScope::ID SubGroupSSID;
44
45 SPIRVMachineModuleInfo(const MachineModuleInfo &MMI) {
46 LLVMContext &CTX = MMI.getModule()->getContext();
47 Work_ItemSSID = CTX.getOrInsertSyncScopeID("work_item");
48 WorkGroupSSID = CTX.getOrInsertSyncScopeID("workgroup");
49 DeviceSSID = CTX.getOrInsertSyncScopeID("device");
50 AllSVMDevicesSSID = CTX.getOrInsertSyncScopeID("all_svm_devices");
51 SubGroupSSID = CTX.getOrInsertSyncScopeID("sub_group");
52 }
53 };
54
55 } // end namespace llvm
56
57 #define DEBUG_TYPE "spirv-isel"
58
59 using namespace llvm;
60 namespace CL = SPIRV::OpenCLExtInst;
61 namespace GL = SPIRV::GLSLExtInst;
62
63 using ExtInstList =
64 std::vector<std::pair<SPIRV::InstructionSet::InstructionSet, uint32_t>>;
65
66 namespace {
67
68 #define GET_GLOBALISEL_PREDICATE_BITSET
69 #include "SPIRVGenGlobalISel.inc"
70 #undef GET_GLOBALISEL_PREDICATE_BITSET
71
72 class SPIRVInstructionSelector : public InstructionSelector {
73 const SPIRVSubtarget &STI;
74 const SPIRVInstrInfo &TII;
75 const SPIRVRegisterInfo &TRI;
76 const RegisterBankInfo &RBI;
77 SPIRVGlobalRegistry &GR;
78 MachineRegisterInfo *MRI;
79 SPIRVMachineModuleInfo *MMI = nullptr;
80
81 /// We need to keep track of the number we give to anonymous global values to
82 /// generate the same name every time when this is needed.
83 mutable DenseMap<const GlobalValue *, unsigned> UnnamedGlobalIDs;
84
85 public:
86 SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
87 const SPIRVSubtarget &ST,
88 const RegisterBankInfo &RBI);
89 void setupMF(MachineFunction &MF, GISelKnownBits *KB,
90 CodeGenCoverage *CoverageInfo, ProfileSummaryInfo *PSI,
91 BlockFrequencyInfo *BFI) override;
92 // Common selection code. Instruction-specific selection occurs in spvSelect.
93 bool select(MachineInstr &I) override;
94 static const char *getName() { return DEBUG_TYPE; }
95
96 #define GET_GLOBALISEL_PREDICATES_DECL
97 #include "SPIRVGenGlobalISel.inc"
98 #undef GET_GLOBALISEL_PREDICATES_DECL
99
100 #define GET_GLOBALISEL_TEMPORARIES_DECL
101 #include "SPIRVGenGlobalISel.inc"
102 #undef GET_GLOBALISEL_TEMPORARIES_DECL
103
104 private:
105 // tblgen-erated 'select' implementation, used as the initial selector for
106 // the patterns that don't require complex C++.
107 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
108
109 // All instruction-specific selection that didn't happen in "select()".
110 // Is basically a large Switch/Case delegating to all other select method.
111 bool spvSelect(Register ResVReg, const SPIRVType *ResType,
112 MachineInstr &I) const;
113
114 bool selectGlobalValue(Register ResVReg, MachineInstr &I,
115 const MachineInstr *Init = nullptr) const;
116
117 bool selectUnOpWithSrc(Register ResVReg, const SPIRVType *ResType,
118 MachineInstr &I, Register SrcReg,
119 unsigned Opcode) const;
120 bool selectUnOp(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
121 unsigned Opcode) const;
122
123 bool selectBitcast(Register ResVReg, const SPIRVType *ResType,
124 MachineInstr &I) const;
125
126 bool selectLoad(Register ResVReg, const SPIRVType *ResType,
127 MachineInstr &I) const;
128 bool selectStore(MachineInstr &I) const;
129
130 bool selectStackSave(Register ResVReg, const SPIRVType *ResType,
131 MachineInstr &I) const;
132 bool selectStackRestore(MachineInstr &I) const;
133
134 bool selectMemOperation(Register ResVReg, MachineInstr &I) const;
135
136 bool selectAtomicRMW(Register ResVReg, const SPIRVType *ResType,
137 MachineInstr &I, unsigned NewOpcode,
138 unsigned NegateOpcode = 0) const;
139
140 bool selectAtomicCmpXchg(Register ResVReg, const SPIRVType *ResType,
141 MachineInstr &I) const;
142
143 bool selectFence(MachineInstr &I) const;
144
145 bool selectAddrSpaceCast(Register ResVReg, const SPIRVType *ResType,
146 MachineInstr &I) const;
147
148 bool selectAnyOrAll(Register ResVReg, const SPIRVType *ResType,
149 MachineInstr &I, unsigned OpType) const;
150
151 bool selectAll(Register ResVReg, const SPIRVType *ResType,
152 MachineInstr &I) const;
153
154 bool selectAny(Register ResVReg, const SPIRVType *ResType,
155 MachineInstr &I) const;
156
157 bool selectBitreverse(Register ResVReg, const SPIRVType *ResType,
158 MachineInstr &I) const;
159
160 bool selectConstVector(Register ResVReg, const SPIRVType *ResType,
161 MachineInstr &I) const;
162 bool selectSplatVector(Register ResVReg, const SPIRVType *ResType,
163 MachineInstr &I) const;
164
165 bool selectCmp(Register ResVReg, const SPIRVType *ResType,
166 unsigned comparisonOpcode, MachineInstr &I) const;
167
168 bool selectICmp(Register ResVReg, const SPIRVType *ResType,
169 MachineInstr &I) const;
170 bool selectFCmp(Register ResVReg, const SPIRVType *ResType,
171 MachineInstr &I) const;
172
173 void renderImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
174 int OpIdx) const;
175 void renderFImm32(MachineInstrBuilder &MIB, const MachineInstr &I,
176 int OpIdx) const;
177
178 bool selectConst(Register ResVReg, const SPIRVType *ResType, const APInt &Imm,
179 MachineInstr &I) const;
180
181 bool selectSelect(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
182 bool IsSigned) const;
183 bool selectIToF(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
184 bool IsSigned, unsigned Opcode) const;
185 bool selectExt(Register ResVReg, const SPIRVType *ResType, MachineInstr &I,
186 bool IsSigned) const;
187
188 bool selectTrunc(Register ResVReg, const SPIRVType *ResType,
189 MachineInstr &I) const;
190
191 bool selectIntToBool(Register IntReg, Register ResVReg, MachineInstr &I,
192 const SPIRVType *intTy, const SPIRVType *boolTy) const;
193
194 bool selectOpUndef(Register ResVReg, const SPIRVType *ResType,
195 MachineInstr &I) const;
196 bool selectFreeze(Register ResVReg, const SPIRVType *ResType,
197 MachineInstr &I) const;
198 bool selectIntrinsic(Register ResVReg, const SPIRVType *ResType,
199 MachineInstr &I) const;
200 bool selectExtractVal(Register ResVReg, const SPIRVType *ResType,
201 MachineInstr &I) const;
202 bool selectInsertVal(Register ResVReg, const SPIRVType *ResType,
203 MachineInstr &I) const;
204 bool selectExtractElt(Register ResVReg, const SPIRVType *ResType,
205 MachineInstr &I) const;
206 bool selectInsertElt(Register ResVReg, const SPIRVType *ResType,
207 MachineInstr &I) const;
208 bool selectGEP(Register ResVReg, const SPIRVType *ResType,
209 MachineInstr &I) const;
210
211 bool selectFrameIndex(Register ResVReg, const SPIRVType *ResType,
212 MachineInstr &I) const;
213 bool selectAllocaArray(Register ResVReg, const SPIRVType *ResType,
214 MachineInstr &I) const;
215
216 bool selectBranch(MachineInstr &I) const;
217 bool selectBranchCond(MachineInstr &I) const;
218
219 bool selectPhi(Register ResVReg, const SPIRVType *ResType,
220 MachineInstr &I) const;
221
222 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
223 MachineInstr &I, CL::OpenCLExtInst CLInst) const;
224 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
225 MachineInstr &I, CL::OpenCLExtInst CLInst,
226 GL::GLSLExtInst GLInst) const;
227 bool selectExtInst(Register ResVReg, const SPIRVType *ResType,
228 MachineInstr &I, const ExtInstList &ExtInsts) const;
229
230 bool selectLog10(Register ResVReg, const SPIRVType *ResType,
231 MachineInstr &I) const;
232
233 bool selectSpvThreadId(Register ResVReg, const SPIRVType *ResType,
234 MachineInstr &I) const;
235
236 bool selectUnmergeValues(MachineInstr &I) const;
237
238 Register buildI32Constant(uint32_t Val, MachineInstr &I,
239 const SPIRVType *ResType = nullptr) const;
240
241 Register buildZerosVal(const SPIRVType *ResType, MachineInstr &I) const;
242 Register buildZerosValF(const SPIRVType *ResType, MachineInstr &I) const;
243 Register buildOnesVal(bool AllOnes, const SPIRVType *ResType,
244 MachineInstr &I) const;
245
246 bool wrapIntoSpecConstantOp(MachineInstr &I,
247 SmallVector<Register> &CompositeArgs) const;
248 };
249
250 } // end anonymous namespace
251
252 #define GET_GLOBALISEL_IMPL
253 #include "SPIRVGenGlobalISel.inc"
254 #undef GET_GLOBALISEL_IMPL
255
256 SPIRVInstructionSelector::SPIRVInstructionSelector(const SPIRVTargetMachine &TM,
257 const SPIRVSubtarget &ST,
258 const RegisterBankInfo &RBI)
259 : InstructionSelector(), STI(ST), TII(*ST.getInstrInfo()),
260 TRI(*ST.getRegisterInfo()), RBI(RBI), GR(*ST.getSPIRVGlobalRegistry()),
261 #define GET_GLOBALISEL_PREDICATES_INIT
262 #include "SPIRVGenGlobalISel.inc"
263 #undef GET_GLOBALISEL_PREDICATES_INIT
264 #define GET_GLOBALISEL_TEMPORARIES_INIT
265 #include "SPIRVGenGlobalISel.inc"
266 #undef GET_GLOBALISEL_TEMPORARIES_INIT
267 {
268 }
269
270 void SPIRVInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits *KB,
271 CodeGenCoverage *CoverageInfo,
272 ProfileSummaryInfo *PSI,
273 BlockFrequencyInfo *BFI) {
274 MMI = &MF.getMMI().getObjFileInfo<SPIRVMachineModuleInfo>();
275 MRI = &MF.getRegInfo();
276 GR.setCurrentFunc(MF);
277 InstructionSelector::setupMF(MF, KB, CoverageInfo, PSI, BFI);
278 }
279
280 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI);
281
282 // Defined in SPIRVLegalizerInfo.cpp.
283 extern bool isTypeFoldingSupported(unsigned Opcode);
284
285 bool SPIRVInstructionSelector::select(MachineInstr &I) {
286 assert(I.getParent() && "Instruction should be in a basic block!");
287 assert(I.getParent()->getParent() && "Instruction should be in a function!");
288
289 Register Opcode = I.getOpcode();
290 // If it's not a GMIR instruction, we've selected it already.
291 if (!isPreISelGenericOpcode(Opcode)) {
292 if (Opcode == SPIRV::ASSIGN_TYPE) { // These pseudos aren't needed any more.
293 auto *Def = MRI->getVRegDef(I.getOperand(1).getReg());
294 if (isTypeFoldingSupported(Def->getOpcode())) {
295 bool Res = selectImpl(I, *CoverageInfo);
296 assert(Res || Def->getOpcode() == TargetOpcode::G_CONSTANT);
297 if (Res)
298 return Res;
299 }
300 MRI->replaceRegWith(I.getOperand(1).getReg(), I.getOperand(0).getReg());
301 I.removeFromParent();
302 return true;
303 } else if (I.getNumDefs() == 1) {
304 // Make all vregs 32 bits (for SPIR-V IDs).
305 MRI->setType(I.getOperand(0).getReg(), LLT::scalar(32));
306 }
307 return constrainSelectedInstRegOperands(I, TII, TRI, RBI);
308 }
309
310 if (I.getNumOperands() != I.getNumExplicitOperands()) {
311 LLVM_DEBUG(errs() << "Generic instr has unexpected implicit operands\n");
312 return false;
313 }
314
315 // Common code for getting return reg+type, and removing selected instr
316 // from parent occurs here. Instr-specific selection happens in spvSelect().
317 bool HasDefs = I.getNumDefs() > 0;
318 Register ResVReg = HasDefs ? I.getOperand(0).getReg() : Register(0);
319 SPIRVType *ResType = HasDefs ? GR.getSPIRVTypeForVReg(ResVReg) : nullptr;
320 assert(!HasDefs || ResType || I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
321 if (spvSelect(ResVReg, ResType, I)) {
322 if (HasDefs) // Make all vregs 32 bits (for SPIR-V IDs).
323 for (unsigned i = 0; i < I.getNumDefs(); ++i)
324 MRI->setType(I.getOperand(i).getReg(), LLT::scalar(32));
325 I.removeFromParent();
326 return true;
327 }
328 return false;
329 }
330
331 bool SPIRVInstructionSelector::spvSelect(Register ResVReg,
332 const SPIRVType *ResType,
333 MachineInstr &I) const {
334 const unsigned Opcode = I.getOpcode();
335 if (isTypeFoldingSupported(Opcode) && Opcode != TargetOpcode::G_CONSTANT)
336 return selectImpl(I, *CoverageInfo);
337 switch (Opcode) {
338 case TargetOpcode::G_CONSTANT:
339 return selectConst(ResVReg, ResType, I.getOperand(1).getCImm()->getValue(),
340 I);
341 case TargetOpcode::G_GLOBAL_VALUE:
342 return selectGlobalValue(ResVReg, I);
343 case TargetOpcode::G_IMPLICIT_DEF:
344 return selectOpUndef(ResVReg, ResType, I);
345 case TargetOpcode::G_FREEZE:
346 return selectFreeze(ResVReg, ResType, I);
347
348 case TargetOpcode::G_INTRINSIC:
349 case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS:
350 case TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS:
351 return selectIntrinsic(ResVReg, ResType, I);
352 case TargetOpcode::G_BITREVERSE:
353 return selectBitreverse(ResVReg, ResType, I);
354
355 case TargetOpcode::G_BUILD_VECTOR:
356 return selectConstVector(ResVReg, ResType, I);
357 case TargetOpcode::G_SPLAT_VECTOR:
358 return selectSplatVector(ResVReg, ResType, I);
359
360 case TargetOpcode::G_SHUFFLE_VECTOR: {
361 MachineBasicBlock &BB = *I.getParent();
362 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorShuffle))
363 .addDef(ResVReg)
364 .addUse(GR.getSPIRVTypeID(ResType))
365 .addUse(I.getOperand(1).getReg())
366 .addUse(I.getOperand(2).getReg());
367 for (auto V : I.getOperand(3).getShuffleMask())
368 MIB.addImm(V);
369 return MIB.constrainAllUses(TII, TRI, RBI);
370 }
371 case TargetOpcode::G_MEMMOVE:
372 case TargetOpcode::G_MEMCPY:
373 case TargetOpcode::G_MEMSET:
374 return selectMemOperation(ResVReg, I);
375
376 case TargetOpcode::G_ICMP:
377 return selectICmp(ResVReg, ResType, I);
378 case TargetOpcode::G_FCMP:
379 return selectFCmp(ResVReg, ResType, I);
380
381 case TargetOpcode::G_FRAME_INDEX:
382 return selectFrameIndex(ResVReg, ResType, I);
383
384 case TargetOpcode::G_LOAD:
385 return selectLoad(ResVReg, ResType, I);
386 case TargetOpcode::G_STORE:
387 return selectStore(I);
388
389 case TargetOpcode::G_BR:
390 return selectBranch(I);
391 case TargetOpcode::G_BRCOND:
392 return selectBranchCond(I);
393
394 case TargetOpcode::G_PHI:
395 return selectPhi(ResVReg, ResType, I);
396
397 case TargetOpcode::G_FPTOSI:
398 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToS);
399 case TargetOpcode::G_FPTOUI:
400 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertFToU);
401
402 case TargetOpcode::G_SITOFP:
403 return selectIToF(ResVReg, ResType, I, true, SPIRV::OpConvertSToF);
404 case TargetOpcode::G_UITOFP:
405 return selectIToF(ResVReg, ResType, I, false, SPIRV::OpConvertUToF);
406
407 case TargetOpcode::G_CTPOP:
408 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitCount);
409 case TargetOpcode::G_SMIN:
410 return selectExtInst(ResVReg, ResType, I, CL::s_min, GL::SMin);
411 case TargetOpcode::G_UMIN:
412 return selectExtInst(ResVReg, ResType, I, CL::u_min, GL::UMin);
413
414 case TargetOpcode::G_SMAX:
415 return selectExtInst(ResVReg, ResType, I, CL::s_max, GL::SMax);
416 case TargetOpcode::G_UMAX:
417 return selectExtInst(ResVReg, ResType, I, CL::u_max, GL::UMax);
418
419 case TargetOpcode::G_FMA:
420 return selectExtInst(ResVReg, ResType, I, CL::fma, GL::Fma);
421
422 case TargetOpcode::G_FPOW:
423 return selectExtInst(ResVReg, ResType, I, CL::pow, GL::Pow);
424 case TargetOpcode::G_FPOWI:
425 return selectExtInst(ResVReg, ResType, I, CL::pown);
426
427 case TargetOpcode::G_FEXP:
428 return selectExtInst(ResVReg, ResType, I, CL::exp, GL::Exp);
429 case TargetOpcode::G_FEXP2:
430 return selectExtInst(ResVReg, ResType, I, CL::exp2, GL::Exp2);
431
432 case TargetOpcode::G_FLOG:
433 return selectExtInst(ResVReg, ResType, I, CL::log, GL::Log);
434 case TargetOpcode::G_FLOG2:
435 return selectExtInst(ResVReg, ResType, I, CL::log2, GL::Log2);
436 case TargetOpcode::G_FLOG10:
437 return selectLog10(ResVReg, ResType, I);
438
439 case TargetOpcode::G_FABS:
440 return selectExtInst(ResVReg, ResType, I, CL::fabs, GL::FAbs);
441 case TargetOpcode::G_ABS:
442 return selectExtInst(ResVReg, ResType, I, CL::s_abs, GL::SAbs);
443
444 case TargetOpcode::G_FMINNUM:
445 case TargetOpcode::G_FMINIMUM:
446 return selectExtInst(ResVReg, ResType, I, CL::fmin, GL::NMin);
447 case TargetOpcode::G_FMAXNUM:
448 case TargetOpcode::G_FMAXIMUM:
449 return selectExtInst(ResVReg, ResType, I, CL::fmax, GL::NMax);
450
451 case TargetOpcode::G_FCOPYSIGN:
452 return selectExtInst(ResVReg, ResType, I, CL::copysign);
453
454 case TargetOpcode::G_FCEIL:
455 return selectExtInst(ResVReg, ResType, I, CL::ceil, GL::Ceil);
456 case TargetOpcode::G_FFLOOR:
457 return selectExtInst(ResVReg, ResType, I, CL::floor, GL::Floor);
458
459 case TargetOpcode::G_FCOS:
460 return selectExtInst(ResVReg, ResType, I, CL::cos, GL::Cos);
461 case TargetOpcode::G_FSIN:
462 return selectExtInst(ResVReg, ResType, I, CL::sin, GL::Sin);
463
464 case TargetOpcode::G_FSQRT:
465 return selectExtInst(ResVReg, ResType, I, CL::sqrt, GL::Sqrt);
466
467 case TargetOpcode::G_CTTZ:
468 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
469 return selectExtInst(ResVReg, ResType, I, CL::ctz);
470 case TargetOpcode::G_CTLZ:
471 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
472 return selectExtInst(ResVReg, ResType, I, CL::clz);
473
474 case TargetOpcode::G_INTRINSIC_ROUND:
475 return selectExtInst(ResVReg, ResType, I, CL::round, GL::Round);
476 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
477 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
478 case TargetOpcode::G_INTRINSIC_TRUNC:
479 return selectExtInst(ResVReg, ResType, I, CL::trunc, GL::Trunc);
480 case TargetOpcode::G_FRINT:
481 case TargetOpcode::G_FNEARBYINT:
482 return selectExtInst(ResVReg, ResType, I, CL::rint, GL::RoundEven);
483
484 case TargetOpcode::G_SMULH:
485 return selectExtInst(ResVReg, ResType, I, CL::s_mul_hi);
486 case TargetOpcode::G_UMULH:
487 return selectExtInst(ResVReg, ResType, I, CL::u_mul_hi);
488
489 case TargetOpcode::G_SEXT:
490 return selectExt(ResVReg, ResType, I, true);
491 case TargetOpcode::G_ANYEXT:
492 case TargetOpcode::G_ZEXT:
493 return selectExt(ResVReg, ResType, I, false);
494 case TargetOpcode::G_TRUNC:
495 return selectTrunc(ResVReg, ResType, I);
496 case TargetOpcode::G_FPTRUNC:
497 case TargetOpcode::G_FPEXT:
498 return selectUnOp(ResVReg, ResType, I, SPIRV::OpFConvert);
499
500 case TargetOpcode::G_PTRTOINT:
501 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertPtrToU);
502 case TargetOpcode::G_INTTOPTR:
503 return selectUnOp(ResVReg, ResType, I, SPIRV::OpConvertUToPtr);
504 case TargetOpcode::G_BITCAST:
505 return selectBitcast(ResVReg, ResType, I);
506 case TargetOpcode::G_ADDRSPACE_CAST:
507 return selectAddrSpaceCast(ResVReg, ResType, I);
508 case TargetOpcode::G_PTR_ADD: {
509 // Currently, we get G_PTR_ADD only as a result of translating
510 // global variables, initialized with constant expressions like GV + Const
511 // (see test opencl/basic/progvar_prog_scope_init.ll).
512 // TODO: extend the handler once we have other cases.
513 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
514 Register GV = I.getOperand(1).getReg();
515 MachineRegisterInfo::def_instr_iterator II = MRI->def_instr_begin(GV);
516 (void)II;
517 assert(((*II).getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
518 (*II).getOpcode() == TargetOpcode::COPY ||
519 (*II).getOpcode() == SPIRV::OpVariable) &&
520 isImm(I.getOperand(2), MRI));
521 Register Idx = buildZerosVal(GR.getOrCreateSPIRVIntegerType(32, I, TII), I);
522 MachineBasicBlock &BB = *I.getParent();
523 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
524 .addDef(ResVReg)
525 .addUse(GR.getSPIRVTypeID(ResType))
526 .addImm(static_cast<uint32_t>(
527 SPIRV::Opcode::InBoundsPtrAccessChain))
528 .addUse(GV)
529 .addUse(Idx)
530 .addUse(I.getOperand(2).getReg());
531 return MIB.constrainAllUses(TII, TRI, RBI);
532 }
533
534 case TargetOpcode::G_ATOMICRMW_OR:
535 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicOr);
536 case TargetOpcode::G_ATOMICRMW_ADD:
537 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicIAdd);
538 case TargetOpcode::G_ATOMICRMW_AND:
539 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicAnd);
540 case TargetOpcode::G_ATOMICRMW_MAX:
541 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMax);
542 case TargetOpcode::G_ATOMICRMW_MIN:
543 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicSMin);
544 case TargetOpcode::G_ATOMICRMW_SUB:
545 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicISub);
546 case TargetOpcode::G_ATOMICRMW_XOR:
547 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicXor);
548 case TargetOpcode::G_ATOMICRMW_UMAX:
549 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMax);
550 case TargetOpcode::G_ATOMICRMW_UMIN:
551 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicUMin);
552 case TargetOpcode::G_ATOMICRMW_XCHG:
553 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicExchange);
554 case TargetOpcode::G_ATOMIC_CMPXCHG:
555 return selectAtomicCmpXchg(ResVReg, ResType, I);
556
557 case TargetOpcode::G_ATOMICRMW_FADD:
558 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT);
559 case TargetOpcode::G_ATOMICRMW_FSUB:
560 // Translate G_ATOMICRMW_FSUB to OpAtomicFAddEXT with negative value operand
561 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFAddEXT,
562 SPIRV::OpFNegate);
563 case TargetOpcode::G_ATOMICRMW_FMIN:
564 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMinEXT);
565 case TargetOpcode::G_ATOMICRMW_FMAX:
566 return selectAtomicRMW(ResVReg, ResType, I, SPIRV::OpAtomicFMaxEXT);
567
568 case TargetOpcode::G_FENCE:
569 return selectFence(I);
570
571 case TargetOpcode::G_STACKSAVE:
572 return selectStackSave(ResVReg, ResType, I);
573 case TargetOpcode::G_STACKRESTORE:
574 return selectStackRestore(I);
575
576 case TargetOpcode::G_UNMERGE_VALUES:
577 return selectUnmergeValues(I);
578
579 default:
580 return false;
581 }
582 }
583
584 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
585 const SPIRVType *ResType,
586 MachineInstr &I,
587 CL::OpenCLExtInst CLInst) const {
588 return selectExtInst(ResVReg, ResType, I,
589 {{SPIRV::InstructionSet::OpenCL_std, CLInst}});
590 }
591
592 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
593 const SPIRVType *ResType,
594 MachineInstr &I,
595 CL::OpenCLExtInst CLInst,
596 GL::GLSLExtInst GLInst) const {
597 ExtInstList ExtInsts = {{SPIRV::InstructionSet::OpenCL_std, CLInst},
598 {SPIRV::InstructionSet::GLSL_std_450, GLInst}};
599 return selectExtInst(ResVReg, ResType, I, ExtInsts);
600 }
601
602 bool SPIRVInstructionSelector::selectExtInst(Register ResVReg,
603 const SPIRVType *ResType,
604 MachineInstr &I,
605 const ExtInstList &Insts) const {
606
607 for (const auto &Ex : Insts) {
608 SPIRV::InstructionSet::InstructionSet Set = Ex.first;
609 uint32_t Opcode = Ex.second;
610 if (STI.canUseExtInstSet(Set)) {
611 MachineBasicBlock &BB = *I.getParent();
612 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
613 .addDef(ResVReg)
614 .addUse(GR.getSPIRVTypeID(ResType))
615 .addImm(static_cast<uint32_t>(Set))
616 .addImm(Opcode);
617 const unsigned NumOps = I.getNumOperands();
618 for (unsigned i = 1; i < NumOps; ++i)
619 MIB.add(I.getOperand(i));
620 return MIB.constrainAllUses(TII, TRI, RBI);
621 }
622 }
623 return false;
624 }
625
626 bool SPIRVInstructionSelector::selectUnOpWithSrc(Register ResVReg,
627 const SPIRVType *ResType,
628 MachineInstr &I,
629 Register SrcReg,
630 unsigned Opcode) const {
631 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
632 .addDef(ResVReg)
633 .addUse(GR.getSPIRVTypeID(ResType))
634 .addUse(SrcReg)
635 .constrainAllUses(TII, TRI, RBI);
636 }
637
638 bool SPIRVInstructionSelector::selectUnOp(Register ResVReg,
639 const SPIRVType *ResType,
640 MachineInstr &I,
641 unsigned Opcode) const {
642 return selectUnOpWithSrc(ResVReg, ResType, I, I.getOperand(1).getReg(),
643 Opcode);
644 }
645
646 bool SPIRVInstructionSelector::selectBitcast(Register ResVReg,
647 const SPIRVType *ResType,
648 MachineInstr &I) const {
649 Register OpReg = I.getOperand(1).getReg();
650 SPIRVType *OpType = OpReg.isValid() ? GR.getSPIRVTypeForVReg(OpReg) : nullptr;
651 if (!GR.isBitcastCompatible(ResType, OpType))
652 report_fatal_error("incompatible result and operand types in a bitcast");
653 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
654 }
655
656 static SPIRV::Scope::Scope getScope(SyncScope::ID Ord,
657 SPIRVMachineModuleInfo *MMI) {
658 if (Ord == SyncScope::SingleThread || Ord == MMI->Work_ItemSSID)
659 return SPIRV::Scope::Invocation;
660 else if (Ord == SyncScope::System || Ord == MMI->DeviceSSID)
661 return SPIRV::Scope::Device;
662 else if (Ord == MMI->WorkGroupSSID)
663 return SPIRV::Scope::Workgroup;
664 else if (Ord == MMI->AllSVMDevicesSSID)
665 return SPIRV::Scope::CrossDevice;
666 else if (Ord == MMI->SubGroupSSID)
667 return SPIRV::Scope::Subgroup;
668 else
669 // OpenCL approach is: "The functions that do not have memory_scope argument
670 // have the same semantics as the corresponding functions with the
671 // memory_scope argument set to memory_scope_device." See ref.: //
672 // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_C.html#atomic-functions
673 // In our case if the scope is unknown, assuming that SPIR-V code is to be
674 // consumed in an OpenCL environment, we use the same approach and set the
675 // scope to memory_scope_device.
676 return SPIRV::Scope::Device;
677 }
678
679 static void addMemoryOperands(MachineMemOperand *MemOp,
680 MachineInstrBuilder &MIB) {
681 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
682 if (MemOp->isVolatile())
683 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
684 if (MemOp->isNonTemporal())
685 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
686 if (MemOp->getAlign().value())
687 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned);
688
689 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None)) {
690 MIB.addImm(SpvMemOp);
691 if (SpvMemOp & static_cast<uint32_t>(SPIRV::MemoryOperand::Aligned))
692 MIB.addImm(MemOp->getAlign().value());
693 }
694 }
695
696 static void addMemoryOperands(uint64_t Flags, MachineInstrBuilder &MIB) {
697 uint32_t SpvMemOp = static_cast<uint32_t>(SPIRV::MemoryOperand::None);
698 if (Flags & MachineMemOperand::Flags::MOVolatile)
699 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Volatile);
700 if (Flags & MachineMemOperand::Flags::MONonTemporal)
701 SpvMemOp |= static_cast<uint32_t>(SPIRV::MemoryOperand::Nontemporal);
702
703 if (SpvMemOp != static_cast<uint32_t>(SPIRV::MemoryOperand::None))
704 MIB.addImm(SpvMemOp);
705 }
706
707 bool SPIRVInstructionSelector::selectLoad(Register ResVReg,
708 const SPIRVType *ResType,
709 MachineInstr &I) const {
710 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
711 Register Ptr = I.getOperand(1 + OpOffset).getReg();
712 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
713 .addDef(ResVReg)
714 .addUse(GR.getSPIRVTypeID(ResType))
715 .addUse(Ptr);
716 if (!I.getNumMemOperands()) {
717 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
718 I.getOpcode() ==
719 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
720 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
721 } else {
722 addMemoryOperands(*I.memoperands_begin(), MIB);
723 }
724 return MIB.constrainAllUses(TII, TRI, RBI);
725 }
726
727 bool SPIRVInstructionSelector::selectStore(MachineInstr &I) const {
728 unsigned OpOffset = isa<GIntrinsic>(I) ? 1 : 0;
729 Register StoreVal = I.getOperand(0 + OpOffset).getReg();
730 Register Ptr = I.getOperand(1 + OpOffset).getReg();
731 MachineBasicBlock &BB = *I.getParent();
732 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpStore))
733 .addUse(Ptr)
734 .addUse(StoreVal);
735 if (!I.getNumMemOperands()) {
736 assert(I.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS ||
737 I.getOpcode() ==
738 TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS);
739 addMemoryOperands(I.getOperand(2 + OpOffset).getImm(), MIB);
740 } else {
741 addMemoryOperands(*I.memoperands_begin(), MIB);
742 }
743 return MIB.constrainAllUses(TII, TRI, RBI);
744 }
745
746 bool SPIRVInstructionSelector::selectStackSave(Register ResVReg,
747 const SPIRVType *ResType,
748 MachineInstr &I) const {
749 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
750 report_fatal_error(
751 "llvm.stacksave intrinsic: this instruction requires the following "
752 "SPIR-V extension: SPV_INTEL_variable_length_array",
753 false);
754 MachineBasicBlock &BB = *I.getParent();
755 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSaveMemoryINTEL))
756 .addDef(ResVReg)
757 .addUse(GR.getSPIRVTypeID(ResType))
758 .constrainAllUses(TII, TRI, RBI);
759 }
760
761 bool SPIRVInstructionSelector::selectStackRestore(MachineInstr &I) const {
762 if (!STI.canUseExtension(SPIRV::Extension::SPV_INTEL_variable_length_array))
763 report_fatal_error(
764 "llvm.stackrestore intrinsic: this instruction requires the following "
765 "SPIR-V extension: SPV_INTEL_variable_length_array",
766 false);
767 if (!I.getOperand(0).isReg())
768 return false;
769 MachineBasicBlock &BB = *I.getParent();
770 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpRestoreMemoryINTEL))
771 .addUse(I.getOperand(0).getReg())
772 .constrainAllUses(TII, TRI, RBI);
773 }
774
775 bool SPIRVInstructionSelector::selectMemOperation(Register ResVReg,
776 MachineInstr &I) const {
777 MachineBasicBlock &BB = *I.getParent();
778 Register SrcReg = I.getOperand(1).getReg();
779 if (I.getOpcode() == TargetOpcode::G_MEMSET) {
780 assert(I.getOperand(1).isReg() && I.getOperand(2).isReg());
781 unsigned Val = getIConstVal(I.getOperand(1).getReg(), MRI);
782 unsigned Num = getIConstVal(I.getOperand(2).getReg(), MRI);
783 SPIRVType *ValTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
784 SPIRVType *ArrTy = GR.getOrCreateSPIRVArrayType(ValTy, Num, I, TII);
785 Register Const = GR.getOrCreateConsIntArray(Val, I, ArrTy, TII);
786 SPIRVType *VarTy = GR.getOrCreateSPIRVPointerType(
787 ArrTy, I, TII, SPIRV::StorageClass::UniformConstant);
788 // TODO: check if we have such GV, add init, use buildGlobalVariable.
789 Function &CurFunction = GR.CurMF->getFunction();
790 Type *LLVMArrTy =
791 ArrayType::get(IntegerType::get(CurFunction.getContext(), 8), Num);
792 // Module takes ownership of the global var.
793 GlobalVariable *GV = new GlobalVariable(*CurFunction.getParent(), LLVMArrTy,
794 true, GlobalValue::InternalLinkage,
795 Constant::getNullValue(LLVMArrTy));
796 Register VarReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
797 GR.add(GV, GR.CurMF, VarReg);
798
799 buildOpDecorate(VarReg, I, TII, SPIRV::Decoration::Constant, {});
800 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpVariable))
801 .addDef(VarReg)
802 .addUse(GR.getSPIRVTypeID(VarTy))
803 .addImm(SPIRV::StorageClass::UniformConstant)
804 .addUse(Const)
805 .constrainAllUses(TII, TRI, RBI);
806 SPIRVType *SourceTy = GR.getOrCreateSPIRVPointerType(
807 ValTy, I, TII, SPIRV::StorageClass::UniformConstant);
808 SrcReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
809 selectUnOpWithSrc(SrcReg, SourceTy, I, VarReg, SPIRV::OpBitcast);
810 }
811 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCopyMemorySized))
812 .addUse(I.getOperand(0).getReg())
813 .addUse(SrcReg)
814 .addUse(I.getOperand(2).getReg());
815 if (I.getNumMemOperands())
816 addMemoryOperands(*I.memoperands_begin(), MIB);
817 bool Result = MIB.constrainAllUses(TII, TRI, RBI);
818 if (ResVReg.isValid() && ResVReg != MIB->getOperand(0).getReg())
819 BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY), ResVReg)
820 .addUse(MIB->getOperand(0).getReg());
821 return Result;
822 }
823
824 bool SPIRVInstructionSelector::selectAtomicRMW(Register ResVReg,
825 const SPIRVType *ResType,
826 MachineInstr &I,
827 unsigned NewOpcode,
828 unsigned NegateOpcode) const {
829 assert(I.hasOneMemOperand());
830 const MachineMemOperand *MemOp = *I.memoperands_begin();
831 uint32_t Scope =
832 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI));
833 Register ScopeReg = buildI32Constant(Scope, I);
834
835 Register Ptr = I.getOperand(1).getReg();
836 // TODO: Changed as it's implemented in the translator. See test/atomicrmw.ll
837 // auto ScSem =
838 // getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr));
839 AtomicOrdering AO = MemOp->getSuccessOrdering();
840 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
841 Register MemSemReg = buildI32Constant(MemSem /*| ScSem*/, I);
842
843 bool Result = false;
844 Register ValueReg = I.getOperand(2).getReg();
845 if (NegateOpcode != 0) {
846 // Translation with negative value operand is requested
847 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
848 Result |= selectUnOpWithSrc(TmpReg, ResType, I, ValueReg, NegateOpcode);
849 ValueReg = TmpReg;
850 }
851
852 Result |= BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(NewOpcode))
853 .addDef(ResVReg)
854 .addUse(GR.getSPIRVTypeID(ResType))
855 .addUse(Ptr)
856 .addUse(ScopeReg)
857 .addUse(MemSemReg)
858 .addUse(ValueReg)
859 .constrainAllUses(TII, TRI, RBI);
860 return Result;
861 }
862
863 bool SPIRVInstructionSelector::selectUnmergeValues(MachineInstr &I) const {
864 unsigned ArgI = I.getNumOperands() - 1;
865 Register SrcReg =
866 I.getOperand(ArgI).isReg() ? I.getOperand(ArgI).getReg() : Register(0);
867 SPIRVType *DefType =
868 SrcReg.isValid() ? GR.getSPIRVTypeForVReg(SrcReg) : nullptr;
869 if (!DefType || DefType->getOpcode() != SPIRV::OpTypeVector)
870 report_fatal_error(
871 "cannot select G_UNMERGE_VALUES with a non-vector argument");
872
873 SPIRVType *ScalarType =
874 GR.getSPIRVTypeForVReg(DefType->getOperand(1).getReg());
875 MachineBasicBlock &BB = *I.getParent();
876 bool Res = false;
877 for (unsigned i = 0; i < I.getNumDefs(); ++i) {
878 Register ResVReg = I.getOperand(i).getReg();
879 SPIRVType *ResType = GR.getSPIRVTypeForVReg(ResVReg);
880 if (!ResType) {
881 // There was no "assign type" actions, let's fix this now
882 ResType = ScalarType;
883 MRI->setRegClass(ResVReg, &SPIRV::IDRegClass);
884 MRI->setType(ResVReg, LLT::scalar(GR.getScalarOrVectorBitWidth(ResType)));
885 GR.assignSPIRVTypeToVReg(ResType, ResVReg, *GR.CurMF);
886 }
887 auto MIB =
888 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
889 .addDef(ResVReg)
890 .addUse(GR.getSPIRVTypeID(ResType))
891 .addUse(SrcReg)
892 .addImm(static_cast<int64_t>(i));
893 Res |= MIB.constrainAllUses(TII, TRI, RBI);
894 }
895 return Res;
896 }
897
898 bool SPIRVInstructionSelector::selectFence(MachineInstr &I) const {
899 AtomicOrdering AO = AtomicOrdering(I.getOperand(0).getImm());
900 uint32_t MemSem = static_cast<uint32_t>(getMemSemantics(AO));
901 Register MemSemReg = buildI32Constant(MemSem, I);
902 SyncScope::ID Ord = SyncScope::ID(I.getOperand(1).getImm());
903 uint32_t Scope = static_cast<uint32_t>(getScope(Ord, MMI));
904 Register ScopeReg = buildI32Constant(Scope, I);
905 MachineBasicBlock &BB = *I.getParent();
906 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpMemoryBarrier))
907 .addUse(ScopeReg)
908 .addUse(MemSemReg)
909 .constrainAllUses(TII, TRI, RBI);
910 }
911
912 bool SPIRVInstructionSelector::selectAtomicCmpXchg(Register ResVReg,
913 const SPIRVType *ResType,
914 MachineInstr &I) const {
915 Register ScopeReg;
916 Register MemSemEqReg;
917 Register MemSemNeqReg;
918 Register Ptr = I.getOperand(2).getReg();
919 if (!isa<GIntrinsic>(I)) {
920 assert(I.hasOneMemOperand());
921 const MachineMemOperand *MemOp = *I.memoperands_begin();
922 unsigned Scope =
923 static_cast<uint32_t>(getScope(MemOp->getSyncScopeID(), MMI));
924 ScopeReg = buildI32Constant(Scope, I);
925
926 unsigned ScSem = static_cast<uint32_t>(
927 getMemSemanticsForStorageClass(GR.getPointerStorageClass(Ptr)));
928 AtomicOrdering AO = MemOp->getSuccessOrdering();
929 unsigned MemSemEq = static_cast<uint32_t>(getMemSemantics(AO)) | ScSem;
930 MemSemEqReg = buildI32Constant(MemSemEq, I);
931 AtomicOrdering FO = MemOp->getFailureOrdering();
932 unsigned MemSemNeq = static_cast<uint32_t>(getMemSemantics(FO)) | ScSem;
933 MemSemNeqReg =
934 MemSemEq == MemSemNeq ? MemSemEqReg : buildI32Constant(MemSemNeq, I);
935 } else {
936 ScopeReg = I.getOperand(5).getReg();
937 MemSemEqReg = I.getOperand(6).getReg();
938 MemSemNeqReg = I.getOperand(7).getReg();
939 }
940
941 Register Cmp = I.getOperand(3).getReg();
942 Register Val = I.getOperand(4).getReg();
943 SPIRVType *SpvValTy = GR.getSPIRVTypeForVReg(Val);
944 Register ACmpRes = MRI->createVirtualRegister(&SPIRV::IDRegClass);
945 const DebugLoc &DL = I.getDebugLoc();
946 bool Result =
947 BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpAtomicCompareExchange))
948 .addDef(ACmpRes)
949 .addUse(GR.getSPIRVTypeID(SpvValTy))
950 .addUse(Ptr)
951 .addUse(ScopeReg)
952 .addUse(MemSemEqReg)
953 .addUse(MemSemNeqReg)
954 .addUse(Val)
955 .addUse(Cmp)
956 .constrainAllUses(TII, TRI, RBI);
957 Register CmpSuccReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
958 SPIRVType *BoolTy = GR.getOrCreateSPIRVBoolType(I, TII);
959 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpIEqual))
960 .addDef(CmpSuccReg)
961 .addUse(GR.getSPIRVTypeID(BoolTy))
962 .addUse(ACmpRes)
963 .addUse(Cmp)
964 .constrainAllUses(TII, TRI, RBI);
965 Register TmpReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
966 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
967 .addDef(TmpReg)
968 .addUse(GR.getSPIRVTypeID(ResType))
969 .addUse(ACmpRes)
970 .addUse(GR.getOrCreateUndef(I, ResType, TII))
971 .addImm(0)
972 .constrainAllUses(TII, TRI, RBI);
973 Result |= BuildMI(*I.getParent(), I, DL, TII.get(SPIRV::OpCompositeInsert))
974 .addDef(ResVReg)
975 .addUse(GR.getSPIRVTypeID(ResType))
976 .addUse(CmpSuccReg)
977 .addUse(TmpReg)
978 .addImm(1)
979 .constrainAllUses(TII, TRI, RBI);
980 return Result;
981 }
982
983 static bool isGenericCastablePtr(SPIRV::StorageClass::StorageClass SC) {
984 switch (SC) {
985 case SPIRV::StorageClass::Workgroup:
986 case SPIRV::StorageClass::CrossWorkgroup:
987 case SPIRV::StorageClass::Function:
988 return true;
989 default:
990 return false;
991 }
992 }
993
994 static bool isUSMStorageClass(SPIRV::StorageClass::StorageClass SC) {
995 switch (SC) {
996 case SPIRV::StorageClass::DeviceOnlyINTEL:
997 case SPIRV::StorageClass::HostOnlyINTEL:
998 return true;
999 default:
1000 return false;
1001 }
1002 }
1003
1004 // In SPIR-V address space casting can only happen to and from the Generic
1005 // storage class. We can also only cast Workgroup, CrossWorkgroup, or Function
1006 // pointers to and from Generic pointers. As such, we can convert e.g. from
1007 // Workgroup to Function by going via a Generic pointer as an intermediary. All
1008 // other combinations can only be done by a bitcast, and are probably not safe.
1009 bool SPIRVInstructionSelector::selectAddrSpaceCast(Register ResVReg,
1010 const SPIRVType *ResType,
1011 MachineInstr &I) const {
1012 // If the AddrSpaceCast user is single and in OpConstantComposite or
1013 // OpVariable, we should select OpSpecConstantOp.
1014 auto UIs = MRI->use_instructions(ResVReg);
1015 if (!UIs.empty() && ++UIs.begin() == UIs.end() &&
1016 (UIs.begin()->getOpcode() == SPIRV::OpConstantComposite ||
1017 UIs.begin()->getOpcode() == SPIRV::OpVariable ||
1018 isSpvIntrinsic(*UIs.begin(), Intrinsic::spv_init_global))) {
1019 Register NewReg = I.getOperand(1).getReg();
1020 MachineBasicBlock &BB = *I.getParent();
1021 SPIRVType *SpvBaseTy = GR.getOrCreateSPIRVIntegerType(8, I, TII);
1022 ResType = GR.getOrCreateSPIRVPointerType(SpvBaseTy, I, TII,
1023 SPIRV::StorageClass::Generic);
1024 bool Result =
1025 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1026 .addDef(ResVReg)
1027 .addUse(GR.getSPIRVTypeID(ResType))
1028 .addImm(static_cast<uint32_t>(SPIRV::Opcode::PtrCastToGeneric))
1029 .addUse(NewReg)
1030 .constrainAllUses(TII, TRI, RBI);
1031 return Result;
1032 }
1033 Register SrcPtr = I.getOperand(1).getReg();
1034 SPIRVType *SrcPtrTy = GR.getSPIRVTypeForVReg(SrcPtr);
1035 SPIRV::StorageClass::StorageClass SrcSC = GR.getPointerStorageClass(SrcPtr);
1036 SPIRV::StorageClass::StorageClass DstSC = GR.getPointerStorageClass(ResVReg);
1037
1038 // don't generate a cast between identical storage classes
1039 if (SrcSC == DstSC)
1040 return true;
1041
1042 // Casting from an eligible pointer to Generic.
1043 if (DstSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(SrcSC))
1044 return selectUnOp(ResVReg, ResType, I, SPIRV::OpPtrCastToGeneric);
1045 // Casting from Generic to an eligible pointer.
1046 if (SrcSC == SPIRV::StorageClass::Generic && isGenericCastablePtr(DstSC))
1047 return selectUnOp(ResVReg, ResType, I, SPIRV::OpGenericCastToPtr);
1048 // Casting between 2 eligible pointers using Generic as an intermediary.
1049 if (isGenericCastablePtr(SrcSC) && isGenericCastablePtr(DstSC)) {
1050 Register Tmp = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1051 SPIRVType *GenericPtrTy = GR.getOrCreateSPIRVPointerType(
1052 SrcPtrTy, I, TII, SPIRV::StorageClass::Generic);
1053 MachineBasicBlock &BB = *I.getParent();
1054 const DebugLoc &DL = I.getDebugLoc();
1055 bool Success = BuildMI(BB, I, DL, TII.get(SPIRV::OpPtrCastToGeneric))
1056 .addDef(Tmp)
1057 .addUse(GR.getSPIRVTypeID(GenericPtrTy))
1058 .addUse(SrcPtr)
1059 .constrainAllUses(TII, TRI, RBI);
1060 return Success && BuildMI(BB, I, DL, TII.get(SPIRV::OpGenericCastToPtr))
1061 .addDef(ResVReg)
1062 .addUse(GR.getSPIRVTypeID(ResType))
1063 .addUse(Tmp)
1064 .constrainAllUses(TII, TRI, RBI);
1065 }
1066
1067 // Check if instructions from the SPV_INTEL_usm_storage_classes extension may
1068 // be applied
1069 if (isUSMStorageClass(SrcSC) && DstSC == SPIRV::StorageClass::CrossWorkgroup)
1070 return selectUnOp(ResVReg, ResType, I,
1071 SPIRV::OpPtrCastToCrossWorkgroupINTEL);
1072 if (SrcSC == SPIRV::StorageClass::CrossWorkgroup && isUSMStorageClass(DstSC))
1073 return selectUnOp(ResVReg, ResType, I,
1074 SPIRV::OpCrossWorkgroupCastToPtrINTEL);
1075
1076 // TODO Should this case just be disallowed completely?
1077 // We're casting 2 other arbitrary address spaces, so have to bitcast.
1078 return selectUnOp(ResVReg, ResType, I, SPIRV::OpBitcast);
1079 }
1080
1081 static unsigned getFCmpOpcode(unsigned PredNum) {
1082 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1083 switch (Pred) {
1084 case CmpInst::FCMP_OEQ:
1085 return SPIRV::OpFOrdEqual;
1086 case CmpInst::FCMP_OGE:
1087 return SPIRV::OpFOrdGreaterThanEqual;
1088 case CmpInst::FCMP_OGT:
1089 return SPIRV::OpFOrdGreaterThan;
1090 case CmpInst::FCMP_OLE:
1091 return SPIRV::OpFOrdLessThanEqual;
1092 case CmpInst::FCMP_OLT:
1093 return SPIRV::OpFOrdLessThan;
1094 case CmpInst::FCMP_ONE:
1095 return SPIRV::OpFOrdNotEqual;
1096 case CmpInst::FCMP_ORD:
1097 return SPIRV::OpOrdered;
1098 case CmpInst::FCMP_UEQ:
1099 return SPIRV::OpFUnordEqual;
1100 case CmpInst::FCMP_UGE:
1101 return SPIRV::OpFUnordGreaterThanEqual;
1102 case CmpInst::FCMP_UGT:
1103 return SPIRV::OpFUnordGreaterThan;
1104 case CmpInst::FCMP_ULE:
1105 return SPIRV::OpFUnordLessThanEqual;
1106 case CmpInst::FCMP_ULT:
1107 return SPIRV::OpFUnordLessThan;
1108 case CmpInst::FCMP_UNE:
1109 return SPIRV::OpFUnordNotEqual;
1110 case CmpInst::FCMP_UNO:
1111 return SPIRV::OpUnordered;
1112 default:
1113 llvm_unreachable("Unknown predicate type for FCmp");
1114 }
1115 }
1116
1117 static unsigned getICmpOpcode(unsigned PredNum) {
1118 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1119 switch (Pred) {
1120 case CmpInst::ICMP_EQ:
1121 return SPIRV::OpIEqual;
1122 case CmpInst::ICMP_NE:
1123 return SPIRV::OpINotEqual;
1124 case CmpInst::ICMP_SGE:
1125 return SPIRV::OpSGreaterThanEqual;
1126 case CmpInst::ICMP_SGT:
1127 return SPIRV::OpSGreaterThan;
1128 case CmpInst::ICMP_SLE:
1129 return SPIRV::OpSLessThanEqual;
1130 case CmpInst::ICMP_SLT:
1131 return SPIRV::OpSLessThan;
1132 case CmpInst::ICMP_UGE:
1133 return SPIRV::OpUGreaterThanEqual;
1134 case CmpInst::ICMP_UGT:
1135 return SPIRV::OpUGreaterThan;
1136 case CmpInst::ICMP_ULE:
1137 return SPIRV::OpULessThanEqual;
1138 case CmpInst::ICMP_ULT:
1139 return SPIRV::OpULessThan;
1140 default:
1141 llvm_unreachable("Unknown predicate type for ICmp");
1142 }
1143 }
1144
1145 static unsigned getPtrCmpOpcode(unsigned Pred) {
1146 switch (static_cast<CmpInst::Predicate>(Pred)) {
1147 case CmpInst::ICMP_EQ:
1148 return SPIRV::OpPtrEqual;
1149 case CmpInst::ICMP_NE:
1150 return SPIRV::OpPtrNotEqual;
1151 default:
1152 llvm_unreachable("Unknown predicate type for pointer comparison");
1153 }
1154 }
1155
1156 // Return the logical operation, or abort if none exists.
1157 static unsigned getBoolCmpOpcode(unsigned PredNum) {
1158 auto Pred = static_cast<CmpInst::Predicate>(PredNum);
1159 switch (Pred) {
1160 case CmpInst::ICMP_EQ:
1161 return SPIRV::OpLogicalEqual;
1162 case CmpInst::ICMP_NE:
1163 return SPIRV::OpLogicalNotEqual;
1164 default:
1165 llvm_unreachable("Unknown predicate type for Bool comparison");
1166 }
1167 }
1168
1169 bool SPIRVInstructionSelector::selectAnyOrAll(Register ResVReg,
1170 const SPIRVType *ResType,
1171 MachineInstr &I,
1172 unsigned OpAnyOrAll) const {
1173 assert(I.getNumOperands() == 3);
1174 assert(I.getOperand(2).isReg());
1175 MachineBasicBlock &BB = *I.getParent();
1176 Register InputRegister = I.getOperand(2).getReg();
1177 SPIRVType *InputType = GR.getSPIRVTypeForVReg(InputRegister);
1178
1179 if (!InputType)
1180 report_fatal_error("Input Type could not be determined.");
1181
1182 bool IsBoolTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeBool);
1183 bool IsVectorTy = InputType->getOpcode() == SPIRV::OpTypeVector;
1184 if (IsBoolTy && !IsVectorTy) {
1185 assert(ResVReg == I.getOperand(0).getReg());
1186 return BuildMI(*I.getParent(), I, I.getDebugLoc(),
1187 TII.get(TargetOpcode::COPY))
1188 .addDef(ResVReg)
1189 .addUse(InputRegister)
1190 .constrainAllUses(TII, TRI, RBI);
1191 }
1192
1193 bool IsFloatTy = GR.isScalarOrVectorOfType(InputRegister, SPIRV::OpTypeFloat);
1194 unsigned SpirvNotEqualId =
1195 IsFloatTy ? SPIRV::OpFOrdNotEqual : SPIRV::OpINotEqual;
1196 SPIRVType *SpvBoolScalarTy = GR.getOrCreateSPIRVBoolType(I, TII);
1197 SPIRVType *SpvBoolTy = SpvBoolScalarTy;
1198 Register NotEqualReg = ResVReg;
1199
1200 if (IsVectorTy) {
1201 NotEqualReg = IsBoolTy ? InputRegister
1202 : MRI->createVirtualRegister(&SPIRV::IDRegClass);
1203 const unsigned NumElts = InputType->getOperand(2).getImm();
1204 SpvBoolTy = GR.getOrCreateSPIRVVectorType(SpvBoolTy, NumElts, I, TII);
1205 }
1206
1207 if (!IsBoolTy) {
1208 Register ConstZeroReg =
1209 IsFloatTy ? buildZerosValF(InputType, I) : buildZerosVal(InputType, I);
1210
1211 BuildMI(BB, I, I.getDebugLoc(), TII.get(SpirvNotEqualId))
1212 .addDef(NotEqualReg)
1213 .addUse(GR.getSPIRVTypeID(SpvBoolTy))
1214 .addUse(InputRegister)
1215 .addUse(ConstZeroReg)
1216 .constrainAllUses(TII, TRI, RBI);
1217 }
1218
1219 if (!IsVectorTy)
1220 return true;
1221
1222 return BuildMI(BB, I, I.getDebugLoc(), TII.get(OpAnyOrAll))
1223 .addDef(ResVReg)
1224 .addUse(GR.getSPIRVTypeID(SpvBoolScalarTy))
1225 .addUse(NotEqualReg)
1226 .constrainAllUses(TII, TRI, RBI);
1227 }
1228
1229 bool SPIRVInstructionSelector::selectAll(Register ResVReg,
1230 const SPIRVType *ResType,
1231 MachineInstr &I) const {
1232 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAll);
1233 }
1234
1235 bool SPIRVInstructionSelector::selectAny(Register ResVReg,
1236 const SPIRVType *ResType,
1237 MachineInstr &I) const {
1238 return selectAnyOrAll(ResVReg, ResType, I, SPIRV::OpAny);
1239 }
1240
1241 bool SPIRVInstructionSelector::selectBitreverse(Register ResVReg,
1242 const SPIRVType *ResType,
1243 MachineInstr &I) const {
1244 MachineBasicBlock &BB = *I.getParent();
1245 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpBitReverse))
1246 .addDef(ResVReg)
1247 .addUse(GR.getSPIRVTypeID(ResType))
1248 .addUse(I.getOperand(1).getReg())
1249 .constrainAllUses(TII, TRI, RBI);
1250 }
1251
1252 bool SPIRVInstructionSelector::selectFreeze(Register ResVReg,
1253 const SPIRVType *ResType,
1254 MachineInstr &I) const {
1255 // There is no way to implement `freeze` correctly without support on SPIR-V
1256 // standard side, but we may at least address a simple (static) case when
1257 // undef/poison value presence is obvious. The main benefit of even
1258 // incomplete `freeze` support is preventing of translation from crashing due
1259 // to lack of support on legalization and instruction selection steps.
1260 if (!I.getOperand(0).isReg() || !I.getOperand(1).isReg())
1261 return false;
1262 Register OpReg = I.getOperand(1).getReg();
1263 if (MachineInstr *Def = MRI->getVRegDef(OpReg)) {
1264 Register Reg;
1265 switch (Def->getOpcode()) {
1266 case SPIRV::ASSIGN_TYPE:
1267 if (MachineInstr *AssignToDef =
1268 MRI->getVRegDef(Def->getOperand(1).getReg())) {
1269 if (AssignToDef->getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1270 Reg = Def->getOperand(2).getReg();
1271 }
1272 break;
1273 case SPIRV::OpUndef:
1274 Reg = Def->getOperand(1).getReg();
1275 break;
1276 }
1277 unsigned DestOpCode;
1278 if (Reg.isValid()) {
1279 DestOpCode = SPIRV::OpConstantNull;
1280 } else {
1281 DestOpCode = TargetOpcode::COPY;
1282 Reg = OpReg;
1283 }
1284 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(DestOpCode))
1285 .addDef(I.getOperand(0).getReg())
1286 .addUse(Reg)
1287 .constrainAllUses(TII, TRI, RBI);
1288 }
1289 return false;
1290 }
1291
1292 bool SPIRVInstructionSelector::selectConstVector(Register ResVReg,
1293 const SPIRVType *ResType,
1294 MachineInstr &I) const {
1295 // TODO: only const case is supported for now.
1296 assert(std::all_of(
1297 I.operands_begin(), I.operands_end(), [this](const MachineOperand &MO) {
1298 if (MO.isDef())
1299 return true;
1300 if (!MO.isReg())
1301 return false;
1302 SPIRVType *ConstTy = this->MRI->getVRegDef(MO.getReg());
1303 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE &&
1304 ConstTy->getOperand(1).isReg());
1305 Register ConstReg = ConstTy->getOperand(1).getReg();
1306 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg);
1307 assert(Const);
1308 return (Const->getOpcode() == TargetOpcode::G_CONSTANT ||
1309 Const->getOpcode() == TargetOpcode::G_FCONSTANT);
1310 }));
1311
1312 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1313 TII.get(SPIRV::OpConstantComposite))
1314 .addDef(ResVReg)
1315 .addUse(GR.getSPIRVTypeID(ResType));
1316 for (unsigned i = I.getNumExplicitDefs(); i < I.getNumExplicitOperands(); ++i)
1317 MIB.addUse(I.getOperand(i).getReg());
1318 return MIB.constrainAllUses(TII, TRI, RBI);
1319 }
1320
1321 static unsigned getArrayComponentCount(MachineRegisterInfo *MRI,
1322 const SPIRVType *ResType) {
1323 Register OpReg = ResType->getOperand(2).getReg();
1324 SPIRVType *OpDef = MRI->getVRegDef(OpReg);
1325 if (!OpDef)
1326 return 0;
1327 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
1328 OpDef->getOperand(1).isReg()) {
1329 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
1330 OpDef = RefDef;
1331 }
1332 unsigned N = OpDef->getOpcode() == TargetOpcode::G_CONSTANT
1333 ? OpDef->getOperand(1).getCImm()->getValue().getZExtValue()
1334 : 0;
1335 return N;
1336 }
1337
1338 // Return true if the type represents a constant register
1339 static bool isConstReg(MachineRegisterInfo *MRI, SPIRVType *OpDef) {
1340 if (OpDef->getOpcode() == SPIRV::ASSIGN_TYPE &&
1341 OpDef->getOperand(1).isReg()) {
1342 if (SPIRVType *RefDef = MRI->getVRegDef(OpDef->getOperand(1).getReg()))
1343 OpDef = RefDef;
1344 }
1345 return OpDef->getOpcode() == TargetOpcode::G_CONSTANT ||
1346 OpDef->getOpcode() == TargetOpcode::G_FCONSTANT;
1347 }
1348
1349 // Return true if the virtual register represents a constant
1350 static bool isConstReg(MachineRegisterInfo *MRI, Register OpReg) {
1351 if (SPIRVType *OpDef = MRI->getVRegDef(OpReg))
1352 return isConstReg(MRI, OpDef);
1353 return false;
1354 }
1355
1356 bool SPIRVInstructionSelector::selectSplatVector(Register ResVReg,
1357 const SPIRVType *ResType,
1358 MachineInstr &I) const {
1359 unsigned N = 0;
1360 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1361 N = GR.getScalarOrVectorComponentCount(ResType);
1362 else if (ResType->getOpcode() == SPIRV::OpTypeArray)
1363 N = getArrayComponentCount(MRI, ResType);
1364 else
1365 report_fatal_error("Cannot select G_SPLAT_VECTOR with a non-vector result");
1366
1367 unsigned OpIdx = I.getNumExplicitDefs();
1368 if (!I.getOperand(OpIdx).isReg())
1369 report_fatal_error("Unexpected argument in G_SPLAT_VECTOR");
1370
1371 // check if we may construct a constant vector
1372 Register OpReg = I.getOperand(OpIdx).getReg();
1373 bool IsConst = isConstReg(MRI, OpReg);
1374
1375 if (!IsConst && N < 2)
1376 report_fatal_error(
1377 "There must be at least two constituent operands in a vector");
1378
1379 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(),
1380 TII.get(IsConst ? SPIRV::OpConstantComposite
1381 : SPIRV::OpCompositeConstruct))
1382 .addDef(ResVReg)
1383 .addUse(GR.getSPIRVTypeID(ResType));
1384 for (unsigned i = 0; i < N; ++i)
1385 MIB.addUse(OpReg);
1386 return MIB.constrainAllUses(TII, TRI, RBI);
1387 }
1388
1389 bool SPIRVInstructionSelector::selectCmp(Register ResVReg,
1390 const SPIRVType *ResType,
1391 unsigned CmpOpc,
1392 MachineInstr &I) const {
1393 Register Cmp0 = I.getOperand(2).getReg();
1394 Register Cmp1 = I.getOperand(3).getReg();
1395 assert(GR.getSPIRVTypeForVReg(Cmp0)->getOpcode() ==
1396 GR.getSPIRVTypeForVReg(Cmp1)->getOpcode() &&
1397 "CMP operands should have the same type");
1398 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(CmpOpc))
1399 .addDef(ResVReg)
1400 .addUse(GR.getSPIRVTypeID(ResType))
1401 .addUse(Cmp0)
1402 .addUse(Cmp1)
1403 .constrainAllUses(TII, TRI, RBI);
1404 }
1405
1406 bool SPIRVInstructionSelector::selectICmp(Register ResVReg,
1407 const SPIRVType *ResType,
1408 MachineInstr &I) const {
1409 auto Pred = I.getOperand(1).getPredicate();
1410 unsigned CmpOpc;
1411
1412 Register CmpOperand = I.getOperand(2).getReg();
1413 if (GR.isScalarOfType(CmpOperand, SPIRV::OpTypePointer))
1414 CmpOpc = getPtrCmpOpcode(Pred);
1415 else if (GR.isScalarOrVectorOfType(CmpOperand, SPIRV::OpTypeBool))
1416 CmpOpc = getBoolCmpOpcode(Pred);
1417 else
1418 CmpOpc = getICmpOpcode(Pred);
1419 return selectCmp(ResVReg, ResType, CmpOpc, I);
1420 }
1421
1422 void SPIRVInstructionSelector::renderFImm32(MachineInstrBuilder &MIB,
1423 const MachineInstr &I,
1424 int OpIdx) const {
1425 assert(I.getOpcode() == TargetOpcode::G_FCONSTANT && OpIdx == -1 &&
1426 "Expected G_FCONSTANT");
1427 const ConstantFP *FPImm = I.getOperand(1).getFPImm();
1428 addNumImm(FPImm->getValueAPF().bitcastToAPInt(), MIB);
1429 }
1430
1431 void SPIRVInstructionSelector::renderImm32(MachineInstrBuilder &MIB,
1432 const MachineInstr &I,
1433 int OpIdx) const {
1434 assert(I.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 &&
1435 "Expected G_CONSTANT");
1436 addNumImm(I.getOperand(1).getCImm()->getValue(), MIB);
1437 }
1438
1439 Register
1440 SPIRVInstructionSelector::buildI32Constant(uint32_t Val, MachineInstr &I,
1441 const SPIRVType *ResType) const {
1442 Type *LLVMTy = IntegerType::get(GR.CurMF->getFunction().getContext(), 32);
1443 const SPIRVType *SpvI32Ty =
1444 ResType ? ResType : GR.getOrCreateSPIRVIntegerType(32, I, TII);
1445 // Find a constant in DT or build a new one.
1446 auto ConstInt = ConstantInt::get(LLVMTy, Val);
1447 Register NewReg = GR.find(ConstInt, GR.CurMF);
1448 if (!NewReg.isValid()) {
1449 NewReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
1450 GR.add(ConstInt, GR.CurMF, NewReg);
1451 MachineInstr *MI;
1452 MachineBasicBlock &BB = *I.getParent();
1453 if (Val == 0) {
1454 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1455 .addDef(NewReg)
1456 .addUse(GR.getSPIRVTypeID(SpvI32Ty));
1457 } else {
1458 MI = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1459 .addDef(NewReg)
1460 .addUse(GR.getSPIRVTypeID(SpvI32Ty))
1461 .addImm(APInt(32, Val).getZExtValue());
1462 }
1463 constrainSelectedInstRegOperands(*MI, TII, TRI, RBI);
1464 }
1465 return NewReg;
1466 }
1467
1468 bool SPIRVInstructionSelector::selectFCmp(Register ResVReg,
1469 const SPIRVType *ResType,
1470 MachineInstr &I) const {
1471 unsigned CmpOp = getFCmpOpcode(I.getOperand(1).getPredicate());
1472 return selectCmp(ResVReg, ResType, CmpOp, I);
1473 }
1474
1475 Register SPIRVInstructionSelector::buildZerosVal(const SPIRVType *ResType,
1476 MachineInstr &I) const {
1477 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
1478 bool ZeroAsNull = STI.isOpenCLEnv();
1479 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1480 return GR.getOrCreateConstVector(0UL, I, ResType, TII, ZeroAsNull);
1481 return GR.getOrCreateConstInt(0, I, ResType, TII, ZeroAsNull);
1482 }
1483
1484 static APFloat getZeroFP(const Type *LLVMFloatTy) {
1485 if (!LLVMFloatTy)
1486 return APFloat::getZero(APFloat::IEEEsingle());
1487 switch (LLVMFloatTy->getScalarType()->getTypeID()) {
1488 case Type::HalfTyID:
1489 return APFloat::getZero(APFloat::IEEEhalf());
1490 default:
1491 case Type::FloatTyID:
1492 return APFloat::getZero(APFloat::IEEEsingle());
1493 case Type::DoubleTyID:
1494 return APFloat::getZero(APFloat::IEEEdouble());
1495 }
1496 }
1497
1498 Register SPIRVInstructionSelector::buildZerosValF(const SPIRVType *ResType,
1499 MachineInstr &I) const {
1500 // OpenCL uses nulls for Zero. In HLSL we don't use null constants.
1501 bool ZeroAsNull = STI.isOpenCLEnv();
1502 APFloat VZero = getZeroFP(GR.getTypeForSPIRVType(ResType));
1503 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1504 return GR.getOrCreateConstVector(VZero, I, ResType, TII, ZeroAsNull);
1505 return GR.getOrCreateConstFP(VZero, I, ResType, TII, ZeroAsNull);
1506 }
1507
1508 Register SPIRVInstructionSelector::buildOnesVal(bool AllOnes,
1509 const SPIRVType *ResType,
1510 MachineInstr &I) const {
1511 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1512 APInt One =
1513 AllOnes ? APInt::getAllOnes(BitWidth) : APInt::getOneBitSet(BitWidth, 0);
1514 if (ResType->getOpcode() == SPIRV::OpTypeVector)
1515 return GR.getOrCreateConstVector(One.getZExtValue(), I, ResType, TII);
1516 return GR.getOrCreateConstInt(One.getZExtValue(), I, ResType, TII);
1517 }
1518
1519 bool SPIRVInstructionSelector::selectSelect(Register ResVReg,
1520 const SPIRVType *ResType,
1521 MachineInstr &I,
1522 bool IsSigned) const {
1523 // To extend a bool, we need to use OpSelect between constants.
1524 Register ZeroReg = buildZerosVal(ResType, I);
1525 Register OneReg = buildOnesVal(IsSigned, ResType, I);
1526 bool IsScalarBool =
1527 GR.isScalarOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool);
1528 unsigned Opcode =
1529 IsScalarBool ? SPIRV::OpSelectSISCond : SPIRV::OpSelectSIVCond;
1530 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1531 .addDef(ResVReg)
1532 .addUse(GR.getSPIRVTypeID(ResType))
1533 .addUse(I.getOperand(1).getReg())
1534 .addUse(OneReg)
1535 .addUse(ZeroReg)
1536 .constrainAllUses(TII, TRI, RBI);
1537 }
1538
1539 bool SPIRVInstructionSelector::selectIToF(Register ResVReg,
1540 const SPIRVType *ResType,
1541 MachineInstr &I, bool IsSigned,
1542 unsigned Opcode) const {
1543 Register SrcReg = I.getOperand(1).getReg();
1544 // We can convert bool value directly to float type without OpConvert*ToF,
1545 // however the translator generates OpSelect+OpConvert*ToF, so we do the same.
1546 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool)) {
1547 unsigned BitWidth = GR.getScalarOrVectorBitWidth(ResType);
1548 SPIRVType *TmpType = GR.getOrCreateSPIRVIntegerType(BitWidth, I, TII);
1549 if (ResType->getOpcode() == SPIRV::OpTypeVector) {
1550 const unsigned NumElts = ResType->getOperand(2).getImm();
1551 TmpType = GR.getOrCreateSPIRVVectorType(TmpType, NumElts, I, TII);
1552 }
1553 SrcReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1554 selectSelect(SrcReg, TmpType, I, false);
1555 }
1556 return selectUnOpWithSrc(ResVReg, ResType, I, SrcReg, Opcode);
1557 }
1558
1559 bool SPIRVInstructionSelector::selectExt(Register ResVReg,
1560 const SPIRVType *ResType,
1561 MachineInstr &I, bool IsSigned) const {
1562 if (GR.isScalarOrVectorOfType(I.getOperand(1).getReg(), SPIRV::OpTypeBool))
1563 return selectSelect(ResVReg, ResType, I, IsSigned);
1564 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1565 return selectUnOp(ResVReg, ResType, I, Opcode);
1566 }
1567
1568 bool SPIRVInstructionSelector::selectIntToBool(Register IntReg,
1569 Register ResVReg,
1570 MachineInstr &I,
1571 const SPIRVType *IntTy,
1572 const SPIRVType *BoolTy) const {
1573 // To truncate to a bool, we use OpBitwiseAnd 1 and OpINotEqual to zero.
1574 Register BitIntReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1575 bool IsVectorTy = IntTy->getOpcode() == SPIRV::OpTypeVector;
1576 unsigned Opcode = IsVectorTy ? SPIRV::OpBitwiseAndV : SPIRV::OpBitwiseAndS;
1577 Register Zero = buildZerosVal(IntTy, I);
1578 Register One = buildOnesVal(false, IntTy, I);
1579 MachineBasicBlock &BB = *I.getParent();
1580 BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1581 .addDef(BitIntReg)
1582 .addUse(GR.getSPIRVTypeID(IntTy))
1583 .addUse(IntReg)
1584 .addUse(One)
1585 .constrainAllUses(TII, TRI, RBI);
1586 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpINotEqual))
1587 .addDef(ResVReg)
1588 .addUse(GR.getSPIRVTypeID(BoolTy))
1589 .addUse(BitIntReg)
1590 .addUse(Zero)
1591 .constrainAllUses(TII, TRI, RBI);
1592 }
1593
1594 bool SPIRVInstructionSelector::selectTrunc(Register ResVReg,
1595 const SPIRVType *ResType,
1596 MachineInstr &I) const {
1597 if (GR.isScalarOrVectorOfType(ResVReg, SPIRV::OpTypeBool)) {
1598 Register IntReg = I.getOperand(1).getReg();
1599 const SPIRVType *ArgType = GR.getSPIRVTypeForVReg(IntReg);
1600 return selectIntToBool(IntReg, ResVReg, I, ArgType, ResType);
1601 }
1602 bool IsSigned = GR.isScalarOrVectorSigned(ResType);
1603 unsigned Opcode = IsSigned ? SPIRV::OpSConvert : SPIRV::OpUConvert;
1604 return selectUnOp(ResVReg, ResType, I, Opcode);
1605 }
1606
1607 bool SPIRVInstructionSelector::selectConst(Register ResVReg,
1608 const SPIRVType *ResType,
1609 const APInt &Imm,
1610 MachineInstr &I) const {
1611 unsigned TyOpcode = ResType->getOpcode();
1612 assert(TyOpcode != SPIRV::OpTypePointer || Imm.isZero());
1613 MachineBasicBlock &BB = *I.getParent();
1614 if ((TyOpcode == SPIRV::OpTypePointer || TyOpcode == SPIRV::OpTypeEvent) &&
1615 Imm.isZero())
1616 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
1617 .addDef(ResVReg)
1618 .addUse(GR.getSPIRVTypeID(ResType))
1619 .constrainAllUses(TII, TRI, RBI);
1620 if (TyOpcode == SPIRV::OpTypeInt) {
1621 assert(Imm.getBitWidth() <= 64 && "Unsupported integer width!");
1622 Register Reg = GR.getOrCreateConstInt(Imm.getZExtValue(), I, ResType, TII);
1623 if (Reg == ResVReg)
1624 return true;
1625 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
1626 .addDef(ResVReg)
1627 .addUse(Reg)
1628 .constrainAllUses(TII, TRI, RBI);
1629 }
1630 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantI))
1631 .addDef(ResVReg)
1632 .addUse(GR.getSPIRVTypeID(ResType));
1633 // <=32-bit integers should be caught by the sdag pattern.
1634 assert(Imm.getBitWidth() > 32);
1635 addNumImm(Imm, MIB);
1636 return MIB.constrainAllUses(TII, TRI, RBI);
1637 }
1638
1639 bool SPIRVInstructionSelector::selectOpUndef(Register ResVReg,
1640 const SPIRVType *ResType,
1641 MachineInstr &I) const {
1642 return BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1643 .addDef(ResVReg)
1644 .addUse(GR.getSPIRVTypeID(ResType))
1645 .constrainAllUses(TII, TRI, RBI);
1646 }
1647
1648 static bool isImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1649 assert(MO.isReg());
1650 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1651 if (TypeInst->getOpcode() != SPIRV::ASSIGN_TYPE)
1652 return false;
1653 assert(TypeInst->getOperand(1).isReg());
1654 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1655 return ImmInst->getOpcode() == TargetOpcode::G_CONSTANT;
1656 }
1657
1658 static int64_t foldImm(const MachineOperand &MO, MachineRegisterInfo *MRI) {
1659 const SPIRVType *TypeInst = MRI->getVRegDef(MO.getReg());
1660 MachineInstr *ImmInst = MRI->getVRegDef(TypeInst->getOperand(1).getReg());
1661 assert(ImmInst->getOpcode() == TargetOpcode::G_CONSTANT);
1662 return ImmInst->getOperand(1).getCImm()->getZExtValue();
1663 }
1664
1665 bool SPIRVInstructionSelector::selectInsertVal(Register ResVReg,
1666 const SPIRVType *ResType,
1667 MachineInstr &I) const {
1668 MachineBasicBlock &BB = *I.getParent();
1669 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeInsert))
1670 .addDef(ResVReg)
1671 .addUse(GR.getSPIRVTypeID(ResType))
1672 // object to insert
1673 .addUse(I.getOperand(3).getReg())
1674 // composite to insert into
1675 .addUse(I.getOperand(2).getReg());
1676 for (unsigned i = 4; i < I.getNumOperands(); i++)
1677 MIB.addImm(foldImm(I.getOperand(i), MRI));
1678 return MIB.constrainAllUses(TII, TRI, RBI);
1679 }
1680
1681 bool SPIRVInstructionSelector::selectExtractVal(Register ResVReg,
1682 const SPIRVType *ResType,
1683 MachineInstr &I) const {
1684 MachineBasicBlock &BB = *I.getParent();
1685 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
1686 .addDef(ResVReg)
1687 .addUse(GR.getSPIRVTypeID(ResType))
1688 .addUse(I.getOperand(2).getReg());
1689 for (unsigned i = 3; i < I.getNumOperands(); i++)
1690 MIB.addImm(foldImm(I.getOperand(i), MRI));
1691 return MIB.constrainAllUses(TII, TRI, RBI);
1692 }
1693
1694 bool SPIRVInstructionSelector::selectInsertElt(Register ResVReg,
1695 const SPIRVType *ResType,
1696 MachineInstr &I) const {
1697 if (isImm(I.getOperand(4), MRI))
1698 return selectInsertVal(ResVReg, ResType, I);
1699 MachineBasicBlock &BB = *I.getParent();
1700 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorInsertDynamic))
1701 .addDef(ResVReg)
1702 .addUse(GR.getSPIRVTypeID(ResType))
1703 .addUse(I.getOperand(2).getReg())
1704 .addUse(I.getOperand(3).getReg())
1705 .addUse(I.getOperand(4).getReg())
1706 .constrainAllUses(TII, TRI, RBI);
1707 }
1708
1709 bool SPIRVInstructionSelector::selectExtractElt(Register ResVReg,
1710 const SPIRVType *ResType,
1711 MachineInstr &I) const {
1712 if (isImm(I.getOperand(3), MRI))
1713 return selectExtractVal(ResVReg, ResType, I);
1714 MachineBasicBlock &BB = *I.getParent();
1715 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpVectorExtractDynamic))
1716 .addDef(ResVReg)
1717 .addUse(GR.getSPIRVTypeID(ResType))
1718 .addUse(I.getOperand(2).getReg())
1719 .addUse(I.getOperand(3).getReg())
1720 .constrainAllUses(TII, TRI, RBI);
1721 }
1722
1723 bool SPIRVInstructionSelector::selectGEP(Register ResVReg,
1724 const SPIRVType *ResType,
1725 MachineInstr &I) const {
1726 const bool IsGEPInBounds = I.getOperand(2).getImm();
1727
1728 // OpAccessChain could be used for OpenCL, but the SPIRV-LLVM Translator only
1729 // relies on PtrAccessChain, so we'll try not to deviate. For Vulkan however,
1730 // we have to use Op[InBounds]AccessChain.
1731 const unsigned Opcode = STI.isVulkanEnv()
1732 ? (IsGEPInBounds ? SPIRV::OpInBoundsAccessChain
1733 : SPIRV::OpAccessChain)
1734 : (IsGEPInBounds ? SPIRV::OpInBoundsPtrAccessChain
1735 : SPIRV::OpPtrAccessChain);
1736
1737 auto Res = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode))
1738 .addDef(ResVReg)
1739 .addUse(GR.getSPIRVTypeID(ResType))
1740 // Object to get a pointer to.
1741 .addUse(I.getOperand(3).getReg());
1742 // Adding indices.
1743 const unsigned StartingIndex =
1744 (Opcode == SPIRV::OpAccessChain || Opcode == SPIRV::OpInBoundsAccessChain)
1745 ? 5
1746 : 4;
1747 for (unsigned i = StartingIndex; i < I.getNumExplicitOperands(); ++i)
1748 Res.addUse(I.getOperand(i).getReg());
1749 return Res.constrainAllUses(TII, TRI, RBI);
1750 }
1751
1752 // Maybe wrap a value into OpSpecConstantOp
1753 bool SPIRVInstructionSelector::wrapIntoSpecConstantOp(
1754 MachineInstr &I, SmallVector<Register> &CompositeArgs) const {
1755 bool Result = true;
1756 unsigned Lim = I.getNumExplicitOperands();
1757 for (unsigned i = I.getNumExplicitDefs() + 1; i < Lim; ++i) {
1758 Register OpReg = I.getOperand(i).getReg();
1759 SPIRVType *OpDefine = MRI->getVRegDef(OpReg);
1760 SPIRVType *OpType = GR.getSPIRVTypeForVReg(OpReg);
1761 if (!OpDefine || !OpType || isConstReg(MRI, OpDefine) ||
1762 OpDefine->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST) {
1763 // The case of G_ADDRSPACE_CAST inside spv_const_composite() is processed
1764 // by selectAddrSpaceCast()
1765 CompositeArgs.push_back(OpReg);
1766 continue;
1767 }
1768 MachineFunction *MF = I.getMF();
1769 Register WrapReg = GR.find(OpDefine, MF);
1770 if (WrapReg.isValid()) {
1771 CompositeArgs.push_back(WrapReg);
1772 continue;
1773 }
1774 // Create a new register for the wrapper
1775 WrapReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
1776 GR.add(OpDefine, MF, WrapReg);
1777 CompositeArgs.push_back(WrapReg);
1778 // Decorate the wrapper register and generate a new instruction
1779 MRI->setType(WrapReg, LLT::pointer(0, 32));
1780 GR.assignSPIRVTypeToVReg(OpType, WrapReg, *MF);
1781 MachineBasicBlock &BB = *I.getParent();
1782 Result = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSpecConstantOp))
1783 .addDef(WrapReg)
1784 .addUse(GR.getSPIRVTypeID(OpType))
1785 .addImm(static_cast<uint32_t>(SPIRV::Opcode::Bitcast))
1786 .addUse(OpReg)
1787 .constrainAllUses(TII, TRI, RBI);
1788 if (!Result)
1789 break;
1790 }
1791 return Result;
1792 }
1793
1794 bool SPIRVInstructionSelector::selectIntrinsic(Register ResVReg,
1795 const SPIRVType *ResType,
1796 MachineInstr &I) const {
1797 MachineBasicBlock &BB = *I.getParent();
1798 Intrinsic::ID IID = cast<GIntrinsic>(I).getIntrinsicID();
1799 switch (IID) {
1800 case Intrinsic::spv_load:
1801 return selectLoad(ResVReg, ResType, I);
1802 case Intrinsic::spv_store:
1803 return selectStore(I);
1804 case Intrinsic::spv_extractv:
1805 return selectExtractVal(ResVReg, ResType, I);
1806 case Intrinsic::spv_insertv:
1807 return selectInsertVal(ResVReg, ResType, I);
1808 case Intrinsic::spv_extractelt:
1809 return selectExtractElt(ResVReg, ResType, I);
1810 case Intrinsic::spv_insertelt:
1811 return selectInsertElt(ResVReg, ResType, I);
1812 case Intrinsic::spv_gep:
1813 return selectGEP(ResVReg, ResType, I);
1814 case Intrinsic::spv_unref_global:
1815 case Intrinsic::spv_init_global: {
1816 MachineInstr *MI = MRI->getVRegDef(I.getOperand(1).getReg());
1817 MachineInstr *Init = I.getNumExplicitOperands() > 2
1818 ? MRI->getVRegDef(I.getOperand(2).getReg())
1819 : nullptr;
1820 assert(MI);
1821 return selectGlobalValue(MI->getOperand(0).getReg(), *MI, Init);
1822 }
1823 case Intrinsic::spv_undef: {
1824 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUndef))
1825 .addDef(ResVReg)
1826 .addUse(GR.getSPIRVTypeID(ResType));
1827 return MIB.constrainAllUses(TII, TRI, RBI);
1828 }
1829 case Intrinsic::spv_const_composite: {
1830 // If no values are attached, the composite is null constant.
1831 bool IsNull = I.getNumExplicitDefs() + 1 == I.getNumExplicitOperands();
1832 // Select a proper instruction.
1833 unsigned Opcode = SPIRV::OpConstantNull;
1834 SmallVector<Register> CompositeArgs;
1835 if (!IsNull) {
1836 Opcode = SPIRV::OpConstantComposite;
1837 if (!wrapIntoSpecConstantOp(I, CompositeArgs))
1838 return false;
1839 }
1840 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
1841 .addDef(ResVReg)
1842 .addUse(GR.getSPIRVTypeID(ResType));
1843 // skip type MD node we already used when generated assign.type for this
1844 if (!IsNull) {
1845 for (Register OpReg : CompositeArgs)
1846 MIB.addUse(OpReg);
1847 }
1848 return MIB.constrainAllUses(TII, TRI, RBI);
1849 }
1850 case Intrinsic::spv_assign_name: {
1851 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpName));
1852 MIB.addUse(I.getOperand(I.getNumExplicitDefs() + 1).getReg());
1853 for (unsigned i = I.getNumExplicitDefs() + 2;
1854 i < I.getNumExplicitOperands(); ++i) {
1855 MIB.addImm(I.getOperand(i).getImm());
1856 }
1857 return MIB.constrainAllUses(TII, TRI, RBI);
1858 }
1859 case Intrinsic::spv_switch: {
1860 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpSwitch));
1861 for (unsigned i = 1; i < I.getNumExplicitOperands(); ++i) {
1862 if (I.getOperand(i).isReg())
1863 MIB.addReg(I.getOperand(i).getReg());
1864 else if (I.getOperand(i).isCImm())
1865 addNumImm(I.getOperand(i).getCImm()->getValue(), MIB);
1866 else if (I.getOperand(i).isMBB())
1867 MIB.addMBB(I.getOperand(i).getMBB());
1868 else
1869 llvm_unreachable("Unexpected OpSwitch operand");
1870 }
1871 return MIB.constrainAllUses(TII, TRI, RBI);
1872 }
1873 case Intrinsic::spv_cmpxchg:
1874 return selectAtomicCmpXchg(ResVReg, ResType, I);
1875 case Intrinsic::spv_unreachable:
1876 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpUnreachable));
1877 break;
1878 case Intrinsic::spv_alloca:
1879 return selectFrameIndex(ResVReg, ResType, I);
1880 case Intrinsic::spv_alloca_array:
1881 return selectAllocaArray(ResVReg, ResType, I);
1882 case Intrinsic::spv_assume:
1883 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1884 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpAssumeTrueKHR))
1885 .addUse(I.getOperand(1).getReg());
1886 break;
1887 case Intrinsic::spv_expect:
1888 if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
1889 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExpectKHR))
1890 .addDef(ResVReg)
1891 .addUse(GR.getSPIRVTypeID(ResType))
1892 .addUse(I.getOperand(2).getReg())
1893 .addUse(I.getOperand(3).getReg());
1894 break;
1895 case Intrinsic::spv_thread_id:
1896 return selectSpvThreadId(ResVReg, ResType, I);
1897 case Intrinsic::spv_all:
1898 return selectAll(ResVReg, ResType, I);
1899 case Intrinsic::spv_any:
1900 return selectAny(ResVReg, ResType, I);
1901 case Intrinsic::spv_lifetime_start:
1902 case Intrinsic::spv_lifetime_end: {
1903 unsigned Op = IID == Intrinsic::spv_lifetime_start ? SPIRV::OpLifetimeStart
1904 : SPIRV::OpLifetimeStop;
1905 int64_t Size = I.getOperand(I.getNumExplicitDefs() + 1).getImm();
1906 Register PtrReg = I.getOperand(I.getNumExplicitDefs() + 2).getReg();
1907 unsigned PonteeOpType = GR.getPointeeTypeOp(PtrReg);
1908 bool IsNonvoidPtr = PonteeOpType != 0 && PonteeOpType != SPIRV::OpTypeVoid;
1909 if (Size == -1 || IsNonvoidPtr)
1910 Size = 0;
1911 BuildMI(BB, I, I.getDebugLoc(), TII.get(Op)).addUse(PtrReg).addImm(Size);
1912 } break;
1913 default: {
1914 std::string DiagMsg;
1915 raw_string_ostream OS(DiagMsg);
1916 I.print(OS);
1917 DiagMsg = "Intrinsic selection not implemented: " + DiagMsg;
1918 report_fatal_error(DiagMsg.c_str(), false);
1919 }
1920 }
1921 return true;
1922 }
1923
1924 bool SPIRVInstructionSelector::selectAllocaArray(Register ResVReg,
1925 const SPIRVType *ResType,
1926 MachineInstr &I) const {
1927 // there was an allocation size parameter to the allocation instruction
1928 // that is not 1
1929 MachineBasicBlock &BB = *I.getParent();
1930 return BuildMI(BB, I, I.getDebugLoc(),
1931 TII.get(SPIRV::OpVariableLengthArrayINTEL))
1932 .addDef(ResVReg)
1933 .addUse(GR.getSPIRVTypeID(ResType))
1934 .addUse(I.getOperand(2).getReg())
1935 .constrainAllUses(TII, TRI, RBI);
1936 }
1937
1938 bool SPIRVInstructionSelector::selectFrameIndex(Register ResVReg,
1939 const SPIRVType *ResType,
1940 MachineInstr &I) const {
1941 // Change order of instructions if needed: all OpVariable instructions in a
1942 // function must be the first instructions in the first block
1943 MachineFunction *MF = I.getParent()->getParent();
1944 MachineBasicBlock *MBB = &MF->front();
1945 auto It = MBB->SkipPHIsAndLabels(MBB->begin()), E = MBB->end();
1946 bool IsHeader = false;
1947 unsigned Opcode;
1948 for (; It != E && It != I; ++It) {
1949 Opcode = It->getOpcode();
1950 if (Opcode == SPIRV::OpFunction || Opcode == SPIRV::OpFunctionParameter) {
1951 IsHeader = true;
1952 } else if (IsHeader &&
1953 !(Opcode == SPIRV::ASSIGN_TYPE || Opcode == SPIRV::OpLabel)) {
1954 ++It;
1955 break;
1956 }
1957 }
1958 return BuildMI(*MBB, It, It->getDebugLoc(), TII.get(SPIRV::OpVariable))
1959 .addDef(ResVReg)
1960 .addUse(GR.getSPIRVTypeID(ResType))
1961 .addImm(static_cast<uint32_t>(SPIRV::StorageClass::Function))
1962 .constrainAllUses(TII, TRI, RBI);
1963 }
1964
1965 bool SPIRVInstructionSelector::selectBranch(MachineInstr &I) const {
1966 // InstructionSelector walks backwards through the instructions. We can use
1967 // both a G_BR and a G_BRCOND to create an OpBranchConditional. We hit G_BR
1968 // first, so can generate an OpBranchConditional here. If there is no
1969 // G_BRCOND, we just use OpBranch for a regular unconditional branch.
1970 const MachineInstr *PrevI = I.getPrevNode();
1971 MachineBasicBlock &MBB = *I.getParent();
1972 if (PrevI != nullptr && PrevI->getOpcode() == TargetOpcode::G_BRCOND) {
1973 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
1974 .addUse(PrevI->getOperand(0).getReg())
1975 .addMBB(PrevI->getOperand(1).getMBB())
1976 .addMBB(I.getOperand(0).getMBB())
1977 .constrainAllUses(TII, TRI, RBI);
1978 }
1979 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranch))
1980 .addMBB(I.getOperand(0).getMBB())
1981 .constrainAllUses(TII, TRI, RBI);
1982 }
1983
1984 bool SPIRVInstructionSelector::selectBranchCond(MachineInstr &I) const {
1985 // InstructionSelector walks backwards through the instructions. For an
1986 // explicit conditional branch with no fallthrough, we use both a G_BR and a
1987 // G_BRCOND to create an OpBranchConditional. We should hit G_BR first, and
1988 // generate the OpBranchConditional in selectBranch above.
1989 //
1990 // If an OpBranchConditional has been generated, we simply return, as the work
1991 // is alread done. If there is no OpBranchConditional, LLVM must be relying on
1992 // implicit fallthrough to the next basic block, so we need to create an
1993 // OpBranchConditional with an explicit "false" argument pointing to the next
1994 // basic block that LLVM would fall through to.
1995 const MachineInstr *NextI = I.getNextNode();
1996 // Check if this has already been successfully selected.
1997 if (NextI != nullptr && NextI->getOpcode() == SPIRV::OpBranchConditional)
1998 return true;
1999 // Must be relying on implicit block fallthrough, so generate an
2000 // OpBranchConditional with the "next" basic block as the "false" target.
2001 MachineBasicBlock &MBB = *I.getParent();
2002 unsigned NextMBBNum = MBB.getNextNode()->getNumber();
2003 MachineBasicBlock *NextMBB = I.getMF()->getBlockNumbered(NextMBBNum);
2004 return BuildMI(MBB, I, I.getDebugLoc(), TII.get(SPIRV::OpBranchConditional))
2005 .addUse(I.getOperand(0).getReg())
2006 .addMBB(I.getOperand(1).getMBB())
2007 .addMBB(NextMBB)
2008 .constrainAllUses(TII, TRI, RBI);
2009 }
2010
2011 bool SPIRVInstructionSelector::selectPhi(Register ResVReg,
2012 const SPIRVType *ResType,
2013 MachineInstr &I) const {
2014 auto MIB = BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpPhi))
2015 .addDef(ResVReg)
2016 .addUse(GR.getSPIRVTypeID(ResType));
2017 const unsigned NumOps = I.getNumOperands();
2018 for (unsigned i = 1; i < NumOps; i += 2) {
2019 MIB.addUse(I.getOperand(i + 0).getReg());
2020 MIB.addMBB(I.getOperand(i + 1).getMBB());
2021 }
2022 return MIB.constrainAllUses(TII, TRI, RBI);
2023 }
2024
2025 bool SPIRVInstructionSelector::selectGlobalValue(
2026 Register ResVReg, MachineInstr &I, const MachineInstr *Init) const {
2027 // FIXME: don't use MachineIRBuilder here, replace it with BuildMI.
2028 MachineIRBuilder MIRBuilder(I);
2029 const GlobalValue *GV = I.getOperand(1).getGlobal();
2030 Type *GVType = GR.getDeducedGlobalValueType(GV);
2031 SPIRVType *PointerBaseType;
2032 if (GVType->isArrayTy()) {
2033 SPIRVType *ArrayElementType =
2034 GR.getOrCreateSPIRVType(GVType->getArrayElementType(), MIRBuilder,
2035 SPIRV::AccessQualifier::ReadWrite, false);
2036 PointerBaseType = GR.getOrCreateSPIRVArrayType(
2037 ArrayElementType, GVType->getArrayNumElements(), I, TII);
2038 } else {
2039 PointerBaseType = GR.getOrCreateSPIRVType(
2040 GVType, MIRBuilder, SPIRV::AccessQualifier::ReadWrite, false);
2041 }
2042 SPIRVType *ResType = GR.getOrCreateSPIRVPointerType(
2043 PointerBaseType, I, TII,
2044 addressSpaceToStorageClass(GV->getAddressSpace(), STI));
2045
2046 std::string GlobalIdent;
2047 if (!GV->hasName()) {
2048 unsigned &ID = UnnamedGlobalIDs[GV];
2049 if (ID == 0)
2050 ID = UnnamedGlobalIDs.size();
2051 GlobalIdent = "__unnamed_" + Twine(ID).str();
2052 } else {
2053 GlobalIdent = GV->getGlobalIdentifier();
2054 }
2055
2056 // Behaviour of functions as operands depends on availability of the
2057 // corresponding extension (SPV_INTEL_function_pointers):
2058 // - If there is an extension to operate with functions as operands:
2059 // We create a proper constant operand and evaluate a correct type for a
2060 // function pointer.
2061 // - Without the required extension:
2062 // We have functions as operands in tests with blocks of instruction e.g. in
2063 // transcoding/global_block.ll. These operands are not used and should be
2064 // substituted by zero constants. Their type is expected to be always
2065 // OpTypePointer Function %uchar.
2066 if (isa<Function>(GV)) {
2067 const Constant *ConstVal = GV;
2068 MachineBasicBlock &BB = *I.getParent();
2069 Register NewReg = GR.find(ConstVal, GR.CurMF);
2070 if (!NewReg.isValid()) {
2071 Register NewReg = ResVReg;
2072 GR.add(ConstVal, GR.CurMF, NewReg);
2073 const Function *GVFun =
2074 STI.canUseExtension(SPIRV::Extension::SPV_INTEL_function_pointers)
2075 ? dyn_cast<Function>(GV)
2076 : nullptr;
2077 if (GVFun) {
2078 // References to a function via function pointers generate virtual
2079 // registers without a definition. We will resolve it later, during
2080 // module analysis stage.
2081 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2082 Register FuncVReg = MRI->createGenericVirtualRegister(LLT::scalar(32));
2083 MRI->setRegClass(FuncVReg, &SPIRV::IDRegClass);
2084 MachineInstrBuilder MB =
2085 BuildMI(BB, I, I.getDebugLoc(),
2086 TII.get(SPIRV::OpConstantFunctionPointerINTEL))
2087 .addDef(NewReg)
2088 .addUse(GR.getSPIRVTypeID(ResType))
2089 .addUse(FuncVReg);
2090 // mapping the function pointer to the used Function
2091 GR.recordFunctionPointer(&MB.getInstr()->getOperand(2), GVFun);
2092 return MB.constrainAllUses(TII, TRI, RBI);
2093 }
2094 return BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpConstantNull))
2095 .addDef(NewReg)
2096 .addUse(GR.getSPIRVTypeID(ResType))
2097 .constrainAllUses(TII, TRI, RBI);
2098 }
2099 assert(NewReg != ResVReg);
2100 return BuildMI(BB, I, I.getDebugLoc(), TII.get(TargetOpcode::COPY))
2101 .addDef(ResVReg)
2102 .addUse(NewReg)
2103 .constrainAllUses(TII, TRI, RBI);
2104 }
2105 auto GlobalVar = cast<GlobalVariable>(GV);
2106 assert(GlobalVar->getName() != "llvm.global.annotations");
2107
2108 bool HasInit = GlobalVar->hasInitializer() &&
2109 !isa<UndefValue>(GlobalVar->getInitializer());
2110 // Skip empty declaration for GVs with initilaizers till we get the decl with
2111 // passed initializer.
2112 if (HasInit && !Init)
2113 return true;
2114
2115 unsigned AddrSpace = GV->getAddressSpace();
2116 SPIRV::StorageClass::StorageClass Storage =
2117 addressSpaceToStorageClass(AddrSpace, STI);
2118 bool HasLnkTy = GV->getLinkage() != GlobalValue::InternalLinkage &&
2119 Storage != SPIRV::StorageClass::Function;
2120 SPIRV::LinkageType::LinkageType LnkType =
2121 (GV->isDeclaration() || GV->hasAvailableExternallyLinkage())
2122 ? SPIRV::LinkageType::Import
2123 : (GV->getLinkage() == GlobalValue::LinkOnceODRLinkage &&
2124 STI.canUseExtension(SPIRV::Extension::SPV_KHR_linkonce_odr)
2125 ? SPIRV::LinkageType::LinkOnceODR
2126 : SPIRV::LinkageType::Export);
2127
2128 Register Reg = GR.buildGlobalVariable(ResVReg, ResType, GlobalIdent, GV,
2129 Storage, Init, GlobalVar->isConstant(),
2130 HasLnkTy, LnkType, MIRBuilder, true);
2131 return Reg.isValid();
2132 }
2133
2134 bool SPIRVInstructionSelector::selectLog10(Register ResVReg,
2135 const SPIRVType *ResType,
2136 MachineInstr &I) const {
2137 if (STI.canUseExtInstSet(SPIRV::InstructionSet::OpenCL_std)) {
2138 return selectExtInst(ResVReg, ResType, I, CL::log10);
2139 }
2140
2141 // There is no log10 instruction in the GLSL Extended Instruction set, so it
2142 // is implemented as:
2143 // log10(x) = log2(x) * (1 / log2(10))
2144 // = log2(x) * 0.30103
2145
2146 MachineIRBuilder MIRBuilder(I);
2147 MachineBasicBlock &BB = *I.getParent();
2148
2149 // Build log2(x).
2150 Register VarReg = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2151 bool Result =
2152 BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpExtInst))
2153 .addDef(VarReg)
2154 .addUse(GR.getSPIRVTypeID(ResType))
2155 .addImm(static_cast<uint32_t>(SPIRV::InstructionSet::GLSL_std_450))
2156 .addImm(GL::Log2)
2157 .add(I.getOperand(1))
2158 .constrainAllUses(TII, TRI, RBI);
2159
2160 // Build 0.30103.
2161 assert(ResType->getOpcode() == SPIRV::OpTypeVector ||
2162 ResType->getOpcode() == SPIRV::OpTypeFloat);
2163 // TODO: Add matrix implementation once supported by the HLSL frontend.
2164 const SPIRVType *SpirvScalarType =
2165 ResType->getOpcode() == SPIRV::OpTypeVector
2166 ? GR.getSPIRVTypeForVReg(ResType->getOperand(1).getReg())
2167 : ResType;
2168 Register ScaleReg =
2169 GR.buildConstantFP(APFloat(0.30103f), MIRBuilder, SpirvScalarType);
2170
2171 // Multiply log2(x) by 0.30103 to get log10(x) result.
2172 auto Opcode = ResType->getOpcode() == SPIRV::OpTypeVector
2173 ? SPIRV::OpVectorTimesScalar
2174 : SPIRV::OpFMulS;
2175 Result &= BuildMI(BB, I, I.getDebugLoc(), TII.get(Opcode))
2176 .addDef(ResVReg)
2177 .addUse(GR.getSPIRVTypeID(ResType))
2178 .addUse(VarReg)
2179 .addUse(ScaleReg)
2180 .constrainAllUses(TII, TRI, RBI);
2181
2182 return Result;
2183 }
2184
2185 bool SPIRVInstructionSelector::selectSpvThreadId(Register ResVReg,
2186 const SPIRVType *ResType,
2187 MachineInstr &I) const {
2188 // DX intrinsic: @llvm.dx.thread.id(i32)
2189 // ID Name Description
2190 // 93 ThreadId reads the thread ID
2191
2192 MachineIRBuilder MIRBuilder(I);
2193 const SPIRVType *U32Type = GR.getOrCreateSPIRVIntegerType(32, MIRBuilder);
2194 const SPIRVType *Vec3Ty =
2195 GR.getOrCreateSPIRVVectorType(U32Type, 3, MIRBuilder);
2196 const SPIRVType *PtrType = GR.getOrCreateSPIRVPointerType(
2197 Vec3Ty, MIRBuilder, SPIRV::StorageClass::Input);
2198
2199 // Create new register for GlobalInvocationID builtin variable.
2200 Register NewRegister =
2201 MIRBuilder.getMRI()->createVirtualRegister(&SPIRV::IDRegClass);
2202 MIRBuilder.getMRI()->setType(NewRegister, LLT::pointer(0, 32));
2203 GR.assignSPIRVTypeToVReg(PtrType, NewRegister, MIRBuilder.getMF());
2204
2205 // Build GlobalInvocationID global variable with the necessary decorations.
2206 Register Variable = GR.buildGlobalVariable(
2207 NewRegister, PtrType,
2208 getLinkStringForBuiltIn(SPIRV::BuiltIn::GlobalInvocationId), nullptr,
2209 SPIRV::StorageClass::Input, nullptr, true, true,
2210 SPIRV::LinkageType::Import, MIRBuilder, false);
2211
2212 // Create new register for loading value.
2213 MachineRegisterInfo *MRI = MIRBuilder.getMRI();
2214 Register LoadedRegister = MRI->createVirtualRegister(&SPIRV::IDRegClass);
2215 MIRBuilder.getMRI()->setType(LoadedRegister, LLT::pointer(0, 32));
2216 GR.assignSPIRVTypeToVReg(Vec3Ty, LoadedRegister, MIRBuilder.getMF());
2217
2218 // Load v3uint value from the global variable.
2219 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(SPIRV::OpLoad))
2220 .addDef(LoadedRegister)
2221 .addUse(GR.getSPIRVTypeID(Vec3Ty))
2222 .addUse(Variable);
2223
2224 // Get Thread ID index. Expecting operand is a constant immediate value,
2225 // wrapped in a type assignment.
2226 assert(I.getOperand(2).isReg());
2227 Register ThreadIdReg = I.getOperand(2).getReg();
2228 SPIRVType *ConstTy = this->MRI->getVRegDef(ThreadIdReg);
2229 assert(ConstTy && ConstTy->getOpcode() == SPIRV::ASSIGN_TYPE &&
2230 ConstTy->getOperand(1).isReg());
2231 Register ConstReg = ConstTy->getOperand(1).getReg();
2232 const MachineInstr *Const = this->MRI->getVRegDef(ConstReg);
2233 assert(Const && Const->getOpcode() == TargetOpcode::G_CONSTANT);
2234 const llvm::APInt &Val = Const->getOperand(1).getCImm()->getValue();
2235 const uint32_t ThreadId = Val.getZExtValue();
2236
2237 // Extract the thread ID from the loaded vector value.
2238 MachineBasicBlock &BB = *I.getParent();
2239 auto MIB = BuildMI(BB, I, I.getDebugLoc(), TII.get(SPIRV::OpCompositeExtract))
2240 .addDef(ResVReg)
2241 .addUse(GR.getSPIRVTypeID(ResType))
2242 .addUse(LoadedRegister)
2243 .addImm(ThreadId);
2244 return MIB.constrainAllUses(TII, TRI, RBI);
2245 }
2246
2247 namespace llvm {
2248 InstructionSelector *
2249 createSPIRVInstructionSelector(const SPIRVTargetMachine &TM,
2250 const SPIRVSubtarget &Subtarget,
2251 const RegisterBankInfo &RBI) {
2252 return new SPIRVInstructionSelector(TM, Subtarget, RBI);
2253 }
2254 } // namespace llvm
LLVM monorepo