lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp

   1 //===-- NVPTXReplaceImageHandles.cpp - Replace image handles for Fermi ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // On Fermi, image handles are not supported. To work around this, we traverse
  10 // the machine code and replace image handles with concrete symbols. For this
  11 // to work reliably, inlining of all function call must be performed.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "NVPTX.h"
  16 #include "NVPTXMachineFunctionInfo.h"
  17 #include "NVPTXSubtarget.h"
  18 #include "NVPTXTargetMachine.h"
  19 #include "MCTargetDesc/NVPTXBaseInfo.h"
  20 #include "llvm/ADT/DenseSet.h"
  21 #include "llvm/CodeGen/MachineFunction.h"
  22 #include "llvm/CodeGen/MachineFunctionPass.h"
  23 #include "llvm/CodeGen/MachineRegisterInfo.h"
  24 #include "llvm/Support/raw_ostream.h"
  25
  26 using namespace llvm;
  27
  28 namespace {
  29 class NVPTXReplaceImageHandles : public MachineFunctionPass {
  30 private:
  31   static char ID;
  32   DenseSet<MachineInstr *> InstrsToRemove;
  33
  34 public:
  35   NVPTXReplaceImageHandles();
  36
  37   bool runOnMachineFunction(MachineFunction &MF) override;
  38
  39   StringRef getPassName() const override {
  40     return "NVPTX Replace Image Handles";
  41   }
  42 private:
  43   bool processInstr(MachineInstr &MI);
  44   void replaceImageHandle(MachineOperand &Op, MachineFunction &MF);
  45   bool findIndexForHandle(MachineOperand &Op, MachineFunction &MF,
  46                           unsigned &Idx);
  47 };
  48 }
  49
  50 char NVPTXReplaceImageHandles::ID = 0;
  51
  52 NVPTXReplaceImageHandles::NVPTXReplaceImageHandles()
  53   : MachineFunctionPass(ID) {}
  54
  55 bool NVPTXReplaceImageHandles::runOnMachineFunction(MachineFunction &MF) {
  56   bool Changed = false;
  57   InstrsToRemove.clear();
  58
  59   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end(); BI != BE;
  60        ++BI) {
  61     for (MachineBasicBlock::iterator I = (*BI).begin(), E = (*BI).end();
  62          I != E; ++I) {
  63       MachineInstr &MI = *I;
  64       Changed |= processInstr(MI);
  65     }
  66   }
  67
  68   // Now clean up any handle-access instructions
  69   // This is needed in debug mode when code cleanup passes are not executed,
  70   // but we need the handle access to be eliminated because they are not
  71   // valid instructions when image handles are disabled.
  72   for (DenseSet<MachineInstr *>::iterator I = InstrsToRemove.begin(),
  73        E = InstrsToRemove.end(); I != E; ++I) {
  74     (*I)->eraseFromParent();
  75   }
  76   return Changed;
  77 }
  78
  79 bool NVPTXReplaceImageHandles::processInstr(MachineInstr &MI) {
  80   MachineFunction &MF = *MI.getParent()->getParent();
  81   const MCInstrDesc &MCID = MI.getDesc();
  82
  83   if (MCID.TSFlags & NVPTXII::IsTexFlag) {
  84     // This is a texture fetch, so operand 4 is a texref and operand 5 is
  85     // a samplerref
  86     MachineOperand &TexHandle = MI.getOperand(4);
  87     replaceImageHandle(TexHandle, MF);
  88
  89     if (!(MCID.TSFlags & NVPTXII::IsTexModeUnifiedFlag)) {
  90       MachineOperand &SampHandle = MI.getOperand(5);
  91       replaceImageHandle(SampHandle, MF);
  92     }
  93
  94     return true;
  95   } else if (MCID.TSFlags & NVPTXII::IsSuldMask) {
  96     unsigned VecSize =
  97       1 << (((MCID.TSFlags & NVPTXII::IsSuldMask) >> NVPTXII::IsSuldShift) - 1);
  98
  99     // For a surface load of vector size N, the Nth operand will be the surfref
 100     MachineOperand &SurfHandle = MI.getOperand(VecSize);
 101
 102     replaceImageHandle(SurfHandle, MF);
 103
 104     return true;
 105   } else if (MCID.TSFlags & NVPTXII::IsSustFlag) {
 106     // This is a surface store, so operand 0 is a surfref
 107     MachineOperand &SurfHandle = MI.getOperand(0);
 108
 109     replaceImageHandle(SurfHandle, MF);
 110
 111     return true;
 112   } else if (MCID.TSFlags & NVPTXII::IsSurfTexQueryFlag) {
 113     // This is a query, so operand 1 is a surfref/texref
 114     MachineOperand &Handle = MI.getOperand(1);
 115
 116     replaceImageHandle(Handle, MF);
 117
 118     return true;
 119   }
 120
 121   return false;
 122 }
 123
 124 void NVPTXReplaceImageHandles::
 125 replaceImageHandle(MachineOperand &Op, MachineFunction &MF) {
 126   unsigned Idx;
 127   if (findIndexForHandle(Op, MF, Idx)) {
 128     Op.ChangeToImmediate(Idx);
 129   }
 130 }
 131
 132 bool NVPTXReplaceImageHandles::
 133 findIndexForHandle(MachineOperand &Op, MachineFunction &MF, unsigned &Idx) {
 134   const MachineRegisterInfo &MRI = MF.getRegInfo();
 135   NVPTXMachineFunctionInfo *MFI = MF.getInfo<NVPTXMachineFunctionInfo>();
 136
 137   assert(Op.isReg() && "Handle is not in a reg?");
 138
 139   // Which instruction defines the handle?
 140   MachineInstr &TexHandleDef = *MRI.getVRegDef(Op.getReg());
 141
 142   switch (TexHandleDef.getOpcode()) {
 143   case NVPTX::LD_i64_avar: {
 144     // The handle is a parameter value being loaded, replace with the
 145     // parameter symbol
 146     const NVPTXTargetMachine &TM =
 147         static_cast<const NVPTXTargetMachine &>(MF.getTarget());
 148     if (TM.getDrvInterface() == NVPTX::CUDA) {
 149       // For CUDA, we preserve the param loads coming from function arguments
 150       return false;
 151     }
 152
 153     assert(TexHandleDef.getOperand(6).isSymbol() && "Load is not a symbol!");
 154     StringRef Sym = TexHandleDef.getOperand(6).getSymbolName();
 155     std::string ParamBaseName = MF.getName();
 156     ParamBaseName += "_param_";
 157     assert(Sym.startswith(ParamBaseName) && "Invalid symbol reference");
 158     unsigned Param = atoi(Sym.data()+ParamBaseName.size());
 159     std::string NewSym;
 160     raw_string_ostream NewSymStr(NewSym);
 161     NewSymStr << MF.getName() << "_param_" << Param;
 162
 163     InstrsToRemove.insert(&TexHandleDef);
 164     Idx = MFI->getImageHandleSymbolIndex(NewSymStr.str().c_str());
 165     return true;
 166   }
 167   case NVPTX::texsurf_handles: {
 168     // The handle is a global variable, replace with the global variable name
 169     assert(TexHandleDef.getOperand(1).isGlobal() && "Load is not a global!");
 170     const GlobalValue *GV = TexHandleDef.getOperand(1).getGlobal();
 171     assert(GV->hasName() && "Global sampler must be named!");
 172     InstrsToRemove.insert(&TexHandleDef);
 173     Idx = MFI->getImageHandleSymbolIndex(GV->getName().data());
 174     return true;
 175   }
 176   case NVPTX::nvvm_move_i64:
 177   case TargetOpcode::COPY: {
 178     bool Res = findIndexForHandle(TexHandleDef.getOperand(1), MF, Idx);
 179     if (Res) {
 180       InstrsToRemove.insert(&TexHandleDef);
 181     }
 182     return Res;
 183   }
 184   default:
 185     llvm_unreachable("Unknown instruction operating on handle");
 186   }
 187 }
 188
 189 MachineFunctionPass *llvm::createNVPTXReplaceImageHandlesPass() {
 190   return new NVPTXReplaceImageHandles();
 191 }