llvm/lib/Target/NVPTX/NVPTXUtilities.cpp

   1 //===- NVPTXUtilities.cpp - Utility Functions -----------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains miscellaneous utility functions
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "NVPTXUtilities.h"
  14 #include "NVPTX.h"
  15 #include "NVPTXTargetMachine.h"
  16 #include "llvm/IR/Constants.h"
  17 #include "llvm/IR/Function.h"
  18 #include "llvm/IR/GlobalVariable.h"
  19 #include "llvm/IR/InstIterator.h"
  20 #include "llvm/IR/Module.h"
  21 #include "llvm/IR/Operator.h"
  22 #include "llvm/Support/Mutex.h"
  23 #include <algorithm>
  24 #include <cstring>
  25 #include <map>
  26 #include <mutex>
  27 #include <string>
  28 #include <vector>
  29
  30 namespace llvm {
  31
  32 namespace {
  33 typedef std::map<std::string, std::vector<unsigned> > key_val_pair_t;
  34 typedef std::map<const GlobalValue *, key_val_pair_t> global_val_annot_t;
  35
  36 struct AnnotationCache {
  37   sys::Mutex Lock;
  38   std::map<const Module *, global_val_annot_t> Cache;
  39 };
  40
  41 AnnotationCache &getAnnotationCache() {
  42   static AnnotationCache AC;
  43   return AC;
  44 }
  45 } // anonymous namespace
  46
  47 void clearAnnotationCache(const Module *Mod) {
  48   auto &AC = getAnnotationCache();
  49   std::lock_guard<sys::Mutex> Guard(AC.Lock);
  50   AC.Cache.erase(Mod);
  51 }
  52
  53 static void cacheAnnotationFromMD(const MDNode *md, key_val_pair_t &retval) {
  54   auto &AC = getAnnotationCache();
  55   std::lock_guard<sys::Mutex> Guard(AC.Lock);
  56   assert(md && "Invalid mdnode for annotation");
  57   assert((md->getNumOperands() % 2) == 1 && "Invalid number of operands");
  58   // start index = 1, to skip the global variable key
  59   // increment = 2, to skip the value for each property-value pairs
  60   for (unsigned i = 1, e = md->getNumOperands(); i != e; i += 2) {
  61     // property
  62     const MDString *prop = dyn_cast<MDString>(md->getOperand(i));
  63     assert(prop && "Annotation property not a string");
  64
  65     // value
  66     ConstantInt *Val = mdconst::dyn_extract<ConstantInt>(md->getOperand(i + 1));
  67     assert(Val && "Value operand not a constant int");
  68
  69     std::string keyname = prop->getString().str();
  70     if (retval.find(keyname) != retval.end())
  71       retval[keyname].push_back(Val->getZExtValue());
  72     else {
  73       std::vector<unsigned> tmp;
  74       tmp.push_back(Val->getZExtValue());
  75       retval[keyname] = tmp;
  76     }
  77   }
  78 }
  79
  80 static void cacheAnnotationFromMD(const Module *m, const GlobalValue *gv) {
  81   auto &AC = getAnnotationCache();
  82   std::lock_guard<sys::Mutex> Guard(AC.Lock);
  83   NamedMDNode *NMD = m->getNamedMetadata("nvvm.annotations");
  84   if (!NMD)
  85     return;
  86   key_val_pair_t tmp;
  87   for (unsigned i = 0, e = NMD->getNumOperands(); i != e; ++i) {
  88     const MDNode *elem = NMD->getOperand(i);
  89
  90     GlobalValue *entity =
  91         mdconst::dyn_extract_or_null<GlobalValue>(elem->getOperand(0));
  92     // entity may be null due to DCE
  93     if (!entity)
  94       continue;
  95     if (entity != gv)
  96       continue;
  97
  98     // accumulate annotations for entity in tmp
  99     cacheAnnotationFromMD(elem, tmp);
 100   }
 101
 102   if (tmp.empty()) // no annotations for this gv
 103     return;
 104
 105   if (AC.Cache.find(m) != AC.Cache.end())
 106     AC.Cache[m][gv] = std::move(tmp);
 107   else {
 108     global_val_annot_t tmp1;
 109     tmp1[gv] = std::move(tmp);
 110     AC.Cache[m] = std::move(tmp1);
 111   }
 112 }
 113
 114 bool findOneNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
 115                            unsigned &retval) {
 116   auto &AC = getAnnotationCache();
 117   std::lock_guard<sys::Mutex> Guard(AC.Lock);
 118   const Module *m = gv->getParent();
 119   if (AC.Cache.find(m) == AC.Cache.end())
 120     cacheAnnotationFromMD(m, gv);
 121   else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
 122     cacheAnnotationFromMD(m, gv);
 123   if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
 124     return false;
 125   retval = AC.Cache[m][gv][prop][0];
 126   return true;
 127 }
 128
 129 bool findAllNVVMAnnotation(const GlobalValue *gv, const std::string &prop,
 130                            std::vector<unsigned> &retval) {
 131   auto &AC = getAnnotationCache();
 132   std::lock_guard<sys::Mutex> Guard(AC.Lock);
 133   const Module *m = gv->getParent();
 134   if (AC.Cache.find(m) == AC.Cache.end())
 135     cacheAnnotationFromMD(m, gv);
 136   else if (AC.Cache[m].find(gv) == AC.Cache[m].end())
 137     cacheAnnotationFromMD(m, gv);
 138   if (AC.Cache[m][gv].find(prop) == AC.Cache[m][gv].end())
 139     return false;
 140   retval = AC.Cache[m][gv][prop];
 141   return true;
 142 }
 143
 144 bool isTexture(const Value &val) {
 145   if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
 146     unsigned annot;
 147     if (findOneNVVMAnnotation(gv, "texture", annot)) {
 148       assert((annot == 1) && "Unexpected annotation on a texture symbol");
 149       return true;
 150     }
 151   }
 152   return false;
 153 }
 154
 155 bool isSurface(const Value &val) {
 156   if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
 157     unsigned annot;
 158     if (findOneNVVMAnnotation(gv, "surface", annot)) {
 159       assert((annot == 1) && "Unexpected annotation on a surface symbol");
 160       return true;
 161     }
 162   }
 163   return false;
 164 }
 165
 166 bool isSampler(const Value &val) {
 167   const char *AnnotationName = "sampler";
 168
 169   if (const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
 170     unsigned annot;
 171     if (findOneNVVMAnnotation(gv, AnnotationName, annot)) {
 172       assert((annot == 1) && "Unexpected annotation on a sampler symbol");
 173       return true;
 174     }
 175   }
 176   if (const Argument *arg = dyn_cast<Argument>(&val)) {
 177     const Function *func = arg->getParent();
 178     std::vector<unsigned> annot;
 179     if (findAllNVVMAnnotation(func, AnnotationName, annot)) {
 180       if (is_contained(annot, arg->getArgNo()))
 181         return true;
 182     }
 183   }
 184   return false;
 185 }
 186
 187 bool isImageReadOnly(const Value &val) {
 188   if (const Argument *arg = dyn_cast<Argument>(&val)) {
 189     const Function *func = arg->getParent();
 190     std::vector<unsigned> annot;
 191     if (findAllNVVMAnnotation(func, "rdoimage", annot)) {
 192       if (is_contained(annot, arg->getArgNo()))
 193         return true;
 194     }
 195   }
 196   return false;
 197 }
 198
 199 bool isImageWriteOnly(const Value &val) {
 200   if (const Argument *arg = dyn_cast<Argument>(&val)) {
 201     const Function *func = arg->getParent();
 202     std::vector<unsigned> annot;
 203     if (findAllNVVMAnnotation(func, "wroimage", annot)) {
 204       if (is_contained(annot, arg->getArgNo()))
 205         return true;
 206     }
 207   }
 208   return false;
 209 }
 210
 211 bool isImageReadWrite(const Value &val) {
 212   if (const Argument *arg = dyn_cast<Argument>(&val)) {
 213     const Function *func = arg->getParent();
 214     std::vector<unsigned> annot;
 215     if (findAllNVVMAnnotation(func, "rdwrimage", annot)) {
 216       if (is_contained(annot, arg->getArgNo()))
 217         return true;
 218     }
 219   }
 220   return false;
 221 }
 222
 223 bool isImage(const Value &val) {
 224   return isImageReadOnly(val) || isImageWriteOnly(val) || isImageReadWrite(val);
 225 }
 226
 227 bool isManaged(const Value &val) {
 228   if(const GlobalValue *gv = dyn_cast<GlobalValue>(&val)) {
 229     unsigned annot;
 230     if (findOneNVVMAnnotation(gv, "managed", annot)) {
 231       assert((annot == 1) && "Unexpected annotation on a managed symbol");
 232       return true;
 233     }
 234   }
 235   return false;
 236 }
 237
 238 std::string getTextureName(const Value &val) {
 239   assert(val.hasName() && "Found texture variable with no name");
 240   return std::string(val.getName());
 241 }
 242
 243 std::string getSurfaceName(const Value &val) {
 244   assert(val.hasName() && "Found surface variable with no name");
 245   return std::string(val.getName());
 246 }
 247
 248 std::string getSamplerName(const Value &val) {
 249   assert(val.hasName() && "Found sampler variable with no name");
 250   return std::string(val.getName());
 251 }
 252
 253 bool getMaxNTIDx(const Function &F, unsigned &x) {
 254   return findOneNVVMAnnotation(&F, "maxntidx", x);
 255 }
 256
 257 bool getMaxNTIDy(const Function &F, unsigned &y) {
 258   return findOneNVVMAnnotation(&F, "maxntidy", y);
 259 }
 260
 261 bool getMaxNTIDz(const Function &F, unsigned &z) {
 262   return findOneNVVMAnnotation(&F, "maxntidz", z);
 263 }
 264
 265 bool getMaxClusterRank(const Function &F, unsigned &x) {
 266   return findOneNVVMAnnotation(&F, "maxclusterrank", x);
 267 }
 268
 269 bool getReqNTIDx(const Function &F, unsigned &x) {
 270   return findOneNVVMAnnotation(&F, "reqntidx", x);
 271 }
 272
 273 bool getReqNTIDy(const Function &F, unsigned &y) {
 274   return findOneNVVMAnnotation(&F, "reqntidy", y);
 275 }
 276
 277 bool getReqNTIDz(const Function &F, unsigned &z) {
 278   return findOneNVVMAnnotation(&F, "reqntidz", z);
 279 }
 280
 281 bool getMinCTASm(const Function &F, unsigned &x) {
 282   return findOneNVVMAnnotation(&F, "minctasm", x);
 283 }
 284
 285 bool getMaxNReg(const Function &F, unsigned &x) {
 286   return findOneNVVMAnnotation(&F, "maxnreg", x);
 287 }
 288
 289 bool isKernelFunction(const Function &F) {
 290   unsigned x = 0;
 291   bool retval = findOneNVVMAnnotation(&F, "kernel", x);
 292   if (!retval) {
 293     // There is no NVVM metadata, check the calling convention
 294     return F.getCallingConv() == CallingConv::PTX_Kernel;
 295   }
 296   return (x == 1);
 297 }
 298
 299 bool getAlign(const Function &F, unsigned index, unsigned &align) {
 300   std::vector<unsigned> Vs;
 301   bool retval = findAllNVVMAnnotation(&F, "align", Vs);
 302   if (!retval)
 303     return false;
 304   for (unsigned v : Vs) {
 305     if ((v >> 16) == index) {
 306       align = v & 0xFFFF;
 307       return true;
 308     }
 309   }
 310   return false;
 311 }
 312
 313 bool getAlign(const CallInst &I, unsigned index, unsigned &align) {
 314   if (MDNode *alignNode = I.getMetadata("callalign")) {
 315     for (int i = 0, n = alignNode->getNumOperands(); i < n; i++) {
 316       if (const ConstantInt *CI =
 317               mdconst::dyn_extract<ConstantInt>(alignNode->getOperand(i))) {
 318         unsigned v = CI->getZExtValue();
 319         if ((v >> 16) == index) {
 320           align = v & 0xFFFF;
 321           return true;
 322         }
 323         if ((v >> 16) > index) {
 324           return false;
 325         }
 326       }
 327     }
 328   }
 329   return false;
 330 }
 331
 332 Function *getMaybeBitcastedCallee(const CallBase *CB) {
 333   return dyn_cast<Function>(CB->getCalledOperand()->stripPointerCasts());
 334 }
 335
 336 bool shouldEmitPTXNoReturn(const Value *V, const TargetMachine &TM) {
 337   const auto &ST =
 338       *static_cast<const NVPTXTargetMachine &>(TM).getSubtargetImpl();
 339   if (!ST.hasNoReturn())
 340     return false;
 341
 342   assert((isa<Function>(V) || isa<CallInst>(V)) &&
 343          "Expect either a call instruction or a function");
 344
 345   if (const CallInst *CallI = dyn_cast<CallInst>(V))
 346     return CallI->doesNotReturn() &&
 347            CallI->getFunctionType()->getReturnType()->isVoidTy();
 348
 349   const Function *F = cast<Function>(V);
 350   return F->doesNotReturn() &&
 351          F->getFunctionType()->getReturnType()->isVoidTy() &&
 352          !isKernelFunction(*F);
 353 }
 354
 355 bool Isv2x16VT(EVT VT) {
 356   return (VT == MVT::v2f16 || VT == MVT::v2bf16 || VT == MVT::v2i16);
 357 }
 358
 359 } // namespace llvm