llvm/lib/Target/NVPTX/NVPTXCtorDtorLowering.cpp

   1 //===-- NVPTXCtorDtorLowering.cpp - Handle global ctors and dtors --------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This pass creates a unified init and fini kernel with the required metadata
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "NVPTXCtorDtorLowering.h"
  14 #include "MCTargetDesc/NVPTXBaseInfo.h"
  15 #include "NVPTX.h"
  16 #include "llvm/ADT/StringExtras.h"
  17 #include "llvm/IR/Constants.h"
  18 #include "llvm/IR/Function.h"
  19 #include "llvm/IR/GlobalVariable.h"
  20 #include "llvm/IR/IRBuilder.h"
  21 #include "llvm/IR/Module.h"
  22 #include "llvm/IR/Value.h"
  23 #include "llvm/Pass.h"
  24 #include "llvm/Support/CommandLine.h"
  25 #include "llvm/Support/MD5.h"
  26 #include "llvm/Transforms/Utils/ModuleUtils.h"
  27
  28 using namespace llvm;
  29
  30 #define DEBUG_TYPE "nvptx-lower-ctor-dtor"
  31
  32 static cl::opt<std::string>
  33     GlobalStr("nvptx-lower-global-ctor-dtor-id",
  34               cl::desc("Override unique ID of ctor/dtor globals."),
  35               cl::init(""), cl::Hidden);
  36
  37 static cl::opt<bool>
  38     CreateKernels("nvptx-emit-init-fini-kernel",
  39                   cl::desc("Emit kernels to call ctor/dtor globals."),
  40                   cl::init(true), cl::Hidden);
  41
  42 namespace {
  43
  44 static std::string getHash(StringRef Str) {
  45   llvm::MD5 Hasher;
  46   llvm::MD5::MD5Result Hash;
  47   Hasher.update(Str);
  48   Hasher.final(Hash);
  49   return llvm::utohexstr(Hash.low(), /*LowerCase=*/true);
  50 }
  51
  52 static void addKernelMetadata(Module &M, GlobalValue *GV) {
  53   llvm::LLVMContext &Ctx = M.getContext();
  54
  55   // Get "nvvm.annotations" metadata node.
  56   llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("nvvm.annotations");
  57
  58   llvm::Metadata *KernelMDVals[] = {
  59       llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, "kernel"),
  60       llvm::ConstantAsMetadata::get(
  61           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
  62
  63   // This kernel is only to be called single-threaded.
  64   llvm::Metadata *ThreadXMDVals[] = {
  65       llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, "maxntidx"),
  66       llvm::ConstantAsMetadata::get(
  67           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
  68   llvm::Metadata *ThreadYMDVals[] = {
  69       llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, "maxntidy"),
  70       llvm::ConstantAsMetadata::get(
  71           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
  72   llvm::Metadata *ThreadZMDVals[] = {
  73       llvm::ConstantAsMetadata::get(GV), llvm::MDString::get(Ctx, "maxntidz"),
  74       llvm::ConstantAsMetadata::get(
  75           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
  76
  77   llvm::Metadata *BlockMDVals[] = {
  78       llvm::ConstantAsMetadata::get(GV),
  79       llvm::MDString::get(Ctx, "maxclusterrank"),
  80       llvm::ConstantAsMetadata::get(
  81           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
  82
  83   // Append metadata to nvvm.annotations.
  84   MD->addOperand(llvm::MDNode::get(Ctx, KernelMDVals));
  85   MD->addOperand(llvm::MDNode::get(Ctx, ThreadXMDVals));
  86   MD->addOperand(llvm::MDNode::get(Ctx, ThreadYMDVals));
  87   MD->addOperand(llvm::MDNode::get(Ctx, ThreadZMDVals));
  88   MD->addOperand(llvm::MDNode::get(Ctx, BlockMDVals));
  89 }
  90
  91 static Function *createInitOrFiniKernelFunction(Module &M, bool IsCtor) {
  92   StringRef InitOrFiniKernelName =
  93       IsCtor ? "nvptx$device$init" : "nvptx$device$fini";
  94   if (M.getFunction(InitOrFiniKernelName))
  95     return nullptr;
  96
  97   Function *InitOrFiniKernel = Function::createWithDefaultAttr(
  98       FunctionType::get(Type::getVoidTy(M.getContext()), false),
  99       GlobalValue::WeakODRLinkage, 0, InitOrFiniKernelName, &M);
 100   addKernelMetadata(M, InitOrFiniKernel);
 101
 102   return InitOrFiniKernel;
 103 }
 104
 105 // We create the IR required to call each callback in this section. This is
 106 // equivalent to the following code. Normally, the linker would provide us with
 107 // the definitions of the init and fini array sections. The 'nvlink' linker does
 108 // not do this so initializing these values is done by the runtime.
 109 //
 110 // extern "C" void **__init_array_start = nullptr;
 111 // extern "C" void **__init_array_end = nullptr;
 112 // extern "C" void **__fini_array_start = nullptr;
 113 // extern "C" void **__fini_array_end = nullptr;
 114 //
 115 // using InitCallback = void();
 116 // using FiniCallback = void();
 117 //
 118 // void call_init_array_callbacks() {
 119 //   for (auto start = __init_array_start; start != __init_array_end; ++start)
 120 //     reinterpret_cast<InitCallback *>(*start)();
 121 // }
 122 //
 123 // void call_init_array_callbacks() {
 124 //   size_t fini_array_size = __fini_array_end - __fini_array_start;
 125 //   for (size_t i = fini_array_size; i > 0; --i)
 126 //     reinterpret_cast<FiniCallback *>(__fini_array_start[i - 1])();
 127 // }
 128 static void createInitOrFiniCalls(Function &F, bool IsCtor) {
 129   Module &M = *F.getParent();
 130   LLVMContext &C = M.getContext();
 131
 132   IRBuilder<> IRB(BasicBlock::Create(C, "entry", &F));
 133   auto *LoopBB = BasicBlock::Create(C, "while.entry", &F);
 134   auto *ExitBB = BasicBlock::Create(C, "while.end", &F);
 135   Type *PtrTy = IRB.getPtrTy(llvm::ADDRESS_SPACE_GLOBAL);
 136
 137   auto *Begin = M.getOrInsertGlobal(
 138       IsCtor ? "__init_array_start" : "__fini_array_start",
 139       PointerType::get(C, 0), [&]() {
 140         auto *GV = new GlobalVariable(
 141             M, PointerType::get(C, 0),
 142             /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
 143             Constant::getNullValue(PointerType::get(C, 0)),
 144             IsCtor ? "__init_array_start" : "__fini_array_start",
 145             /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
 146             /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
 147         GV->setVisibility(GlobalVariable::ProtectedVisibility);
 148         return GV;
 149       });
 150   auto *End = M.getOrInsertGlobal(
 151       IsCtor ? "__init_array_end" : "__fini_array_end", PointerType::get(C, 0),
 152       [&]() {
 153         auto *GV = new GlobalVariable(
 154             M, PointerType::get(C, 0),
 155             /*isConstant=*/false, GlobalValue::WeakAnyLinkage,
 156             Constant::getNullValue(PointerType::get(C, 0)),
 157             IsCtor ? "__init_array_end" : "__fini_array_end",
 158             /*InsertBefore=*/nullptr, GlobalVariable::NotThreadLocal,
 159             /*AddressSpace=*/llvm::ADDRESS_SPACE_GLOBAL);
 160         GV->setVisibility(GlobalVariable::ProtectedVisibility);
 161         return GV;
 162       });
 163
 164   // The constructor type is suppoed to allow using the argument vectors, but
 165   // for now we just call them with no arguments.
 166   auto *CallBackTy = FunctionType::get(IRB.getVoidTy(), {});
 167
 168   // The destructor array must be called in reverse order. Get an expression to
 169   // the end of the array and iterate backwards in that case.
 170   Value *BeginVal = IRB.CreateLoad(Begin->getType(), Begin, "begin");
 171   Value *EndVal = IRB.CreateLoad(Begin->getType(), End, "stop");
 172   if (!IsCtor) {
 173     auto *BeginInt = IRB.CreatePtrToInt(BeginVal, IntegerType::getInt64Ty(C));
 174     auto *EndInt = IRB.CreatePtrToInt(EndVal, IntegerType::getInt64Ty(C));
 175     auto *SubInst = IRB.CreateSub(EndInt, BeginInt);
 176     auto *Offset = IRB.CreateAShr(
 177         SubInst, ConstantInt::get(IntegerType::getInt64Ty(C), 3), "offset",
 178         /*IsExact=*/true);
 179     auto *ValuePtr = IRB.CreateGEP(PointerType::get(C, 0), BeginVal,
 180                                    ArrayRef<Value *>({Offset}));
 181     EndVal = BeginVal;
 182     BeginVal = IRB.CreateInBoundsGEP(
 183         PointerType::get(C, 0), ValuePtr,
 184         ArrayRef<Value *>(ConstantInt::get(IntegerType::getInt64Ty(C), -1)),
 185         "start");
 186   }
 187   IRB.CreateCondBr(
 188       IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_NE : ICmpInst::ICMP_UGT, BeginVal,
 189                     EndVal),
 190       LoopBB, ExitBB);
 191   IRB.SetInsertPoint(LoopBB);
 192   auto *CallBackPHI = IRB.CreatePHI(PtrTy, 2, "ptr");
 193   auto *CallBack = IRB.CreateLoad(IRB.getPtrTy(F.getAddressSpace()),
 194                                   CallBackPHI, "callback");
 195   IRB.CreateCall(CallBackTy, CallBack);
 196   auto *NewCallBack =
 197       IRB.CreateConstGEP1_64(PtrTy, CallBackPHI, IsCtor ? 1 : -1, "next");
 198   auto *EndCmp = IRB.CreateCmp(IsCtor ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_ULT,
 199                                NewCallBack, EndVal, "end");
 200   CallBackPHI->addIncoming(BeginVal, &F.getEntryBlock());
 201   CallBackPHI->addIncoming(NewCallBack, LoopBB);
 202   IRB.CreateCondBr(EndCmp, ExitBB, LoopBB);
 203   IRB.SetInsertPoint(ExitBB);
 204   IRB.CreateRetVoid();
 205 }
 206
 207 static bool createInitOrFiniGlobals(Module &M, GlobalVariable *GV,
 208                                     bool IsCtor) {
 209   ConstantArray *GA = dyn_cast<ConstantArray>(GV->getInitializer());
 210   if (!GA || GA->getNumOperands() == 0)
 211     return false;
 212
 213   // NVPTX has no way to emit variables at specific sections or support for
 214   // the traditional constructor sections. Instead, we emit mangled global
 215   // names so the runtime can build the list manually.
 216   for (Value *V : GA->operands()) {
 217     auto *CS = cast<ConstantStruct>(V);
 218     auto *F = cast<Constant>(CS->getOperand(1));
 219     uint64_t Priority = cast<ConstantInt>(CS->getOperand(0))->getSExtValue();
 220     std::string PriorityStr = "." + std::to_string(Priority);
 221     // We append a semi-unique hash and the priority to the global name.
 222     std::string GlobalID =
 223         !GlobalStr.empty() ? GlobalStr : getHash(M.getSourceFileName());
 224     std::string NameStr =
 225         ((IsCtor ? "__init_array_object_" : "__fini_array_object_") +
 226          F->getName() + "_" + GlobalID + "_" + std::to_string(Priority))
 227             .str();
 228     // PTX does not support exported names with '.' in them.
 229     llvm::transform(NameStr, NameStr.begin(),
 230                     [](char c) { return c == '.' ? '_' : c; });
 231
 232     auto *GV = new GlobalVariable(M, F->getType(), /*IsConstant=*/true,
 233                                   GlobalValue::ExternalLinkage, F, NameStr,
 234                                   nullptr, GlobalValue::NotThreadLocal,
 235                                   /*AddressSpace=*/4);
 236     // This isn't respected by Nvidia, simply put here for clarity.
 237     GV->setSection(IsCtor ? ".init_array" + PriorityStr
 238                           : ".fini_array" + PriorityStr);
 239     GV->setVisibility(GlobalVariable::ProtectedVisibility);
 240     appendToUsed(M, {GV});
 241   }
 242
 243   return true;
 244 }
 245
 246 static bool createInitOrFiniKernel(Module &M, StringRef GlobalName,
 247                                    bool IsCtor) {
 248   GlobalVariable *GV = M.getGlobalVariable(GlobalName);
 249   if (!GV || !GV->hasInitializer())
 250     return false;
 251
 252   if (!createInitOrFiniGlobals(M, GV, IsCtor))
 253     return false;
 254
 255   if (!CreateKernels)
 256     return true;
 257
 258   Function *InitOrFiniKernel = createInitOrFiniKernelFunction(M, IsCtor);
 259   if (!InitOrFiniKernel)
 260     return false;
 261
 262   createInitOrFiniCalls(*InitOrFiniKernel, IsCtor);
 263
 264   GV->eraseFromParent();
 265   return true;
 266 }
 267
 268 static bool lowerCtorsAndDtors(Module &M) {
 269   bool Modified = false;
 270   Modified |= createInitOrFiniKernel(M, "llvm.global_ctors", /*IsCtor =*/true);
 271   Modified |= createInitOrFiniKernel(M, "llvm.global_dtors", /*IsCtor =*/false);
 272   return Modified;
 273 }
 274
 275 class NVPTXCtorDtorLoweringLegacy final : public ModulePass {
 276 public:
 277   static char ID;
 278   NVPTXCtorDtorLoweringLegacy() : ModulePass(ID) {}
 279   bool runOnModule(Module &M) override { return lowerCtorsAndDtors(M); }
 280 };
 281
 282 } // End anonymous namespace
 283
 284 PreservedAnalyses NVPTXCtorDtorLoweringPass::run(Module &M,
 285                                                  ModuleAnalysisManager &AM) {
 286   return lowerCtorsAndDtors(M) ? PreservedAnalyses::none()
 287                                : PreservedAnalyses::all();
 288 }
 289
 290 char NVPTXCtorDtorLoweringLegacy::ID = 0;
 291 char &llvm::NVPTXCtorDtorLoweringLegacyPassID = NVPTXCtorDtorLoweringLegacy::ID;
 292 INITIALIZE_PASS(NVPTXCtorDtorLoweringLegacy, DEBUG_TYPE,
 293                 "Lower ctors and dtors for NVPTX", false, false)
 294
 295 ModulePass *llvm::createNVPTXCtorDtorLoweringLegacyPass() {
 296   return new NVPTXCtorDtorLoweringLegacy();
 297 }