llvm/lib/Target/AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp

   1 //===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // \file
  10 // This post-linking pass replaces the function pointer of enqueued
  11 // block kernel with a global variable (runtime handle) and adds
  12 // "runtime-handle" attribute to the enqueued block kernel.
  13 //
  14 // In LLVM CodeGen the runtime-handle metadata will be translated to
  15 // RuntimeHandle metadata in code object. Runtime allocates a global buffer
  16 // for each kernel with RuntimeHandle metadata and saves the kernel address
  17 // required for the AQL packet into the buffer. __enqueue_kernel function
  18 // in device library knows that the invoke function pointer in the block
  19 // literal is actually runtime handle and loads the kernel address from it
  20 // and put it into AQL packet for dispatching.
  21 //
  22 // This cannot be done in FE since FE cannot create a unique global variable
  23 // with external linkage across LLVM modules. The global variable with internal
  24 // linkage does not work since optimization passes will try to replace loads
  25 // of the global variable with its initialization value.
  26 //
  27 // It also identifies the kernels directly or indirectly enqueues kernels
  28 // and adds "calls-enqueue-kernel" function attribute to them, which will
  29 // be used to determine whether to emit runtime metadata for the kernel
  30 // enqueue related hidden kernel arguments.
  31 //
  32 //===----------------------------------------------------------------------===//
  33
  34 #include "AMDGPU.h"
  35 #include "llvm/ADT/DenseSet.h"
  36 #include "llvm/ADT/SmallString.h"
  37 #include "llvm/IR/Constants.h"
  38 #include "llvm/IR/Instructions.h"
  39 #include "llvm/IR/Mangler.h"
  40 #include "llvm/IR/Module.h"
  41 #include "llvm/Pass.h"
  42 #include "llvm/Support/Debug.h"
  43
  44 #define DEBUG_TYPE "amdgpu-lower-enqueued-block"
  45
  46 using namespace llvm;
  47
  48 namespace {
  49
  50 /// Lower enqueued blocks.
  51 class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass {
  52 public:
  53   static char ID;
  54
  55   explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {}
  56
  57 private:
  58   bool runOnModule(Module &M) override;
  59 };
  60
  61 } // end anonymous namespace
  62
  63 char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0;
  64
  65 char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID =
  66     AMDGPUOpenCLEnqueuedBlockLowering::ID;
  67
  68 INITIALIZE_PASS(AMDGPUOpenCLEnqueuedBlockLowering, DEBUG_TYPE,
  69                 "Lower OpenCL enqueued blocks", false, false)
  70
  71 ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() {
  72   return new AMDGPUOpenCLEnqueuedBlockLowering();
  73 }
  74
  75 bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) {
  76   DenseSet<Function *> Callers;
  77   auto &C = M.getContext();
  78   bool Changed = false;
  79
  80   // ptr kernel_object, i32 private_segment_size, i32 group_segment_size
  81   StructType *HandleTy = nullptr;
  82
  83   for (auto &F : M.functions()) {
  84     if (F.hasFnAttribute("enqueued-block")) {
  85       if (!F.hasName()) {
  86         SmallString<64> Name;
  87         Mangler::getNameWithPrefix(Name, "__amdgpu_enqueued_kernel",
  88                                    M.getDataLayout());
  89         F.setName(Name);
  90       }
  91       LLVM_DEBUG(dbgs() << "found enqueued kernel: " << F.getName() << '\n');
  92       auto RuntimeHandle = (F.getName() + ".runtime_handle").str();
  93       if (!HandleTy) {
  94         Type *Int32 = Type::getInt32Ty(C);
  95         HandleTy =
  96             StructType::create(C, {PointerType::getUnqual(C), Int32, Int32},
  97                                "block.runtime.handle.t");
  98       }
  99
 100       auto *GV = new GlobalVariable(
 101           M, HandleTy,
 102           /*isConstant=*/true, GlobalValue::ExternalLinkage,
 103           /*Initializer=*/Constant::getNullValue(HandleTy), RuntimeHandle,
 104           /*InsertBefore=*/nullptr, GlobalValue::NotThreadLocal,
 105           AMDGPUAS::GLOBAL_ADDRESS,
 106           /*isExternallyInitialized=*/true);
 107       LLVM_DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
 108
 109       F.replaceAllUsesWith(ConstantExpr::getAddrSpaceCast(GV, F.getType()));
 110       F.addFnAttr("runtime-handle", RuntimeHandle);
 111       F.setLinkage(GlobalValue::ExternalLinkage);
 112       Changed = true;
 113     }
 114   }
 115
 116   return Changed;
 117 }