llvm/lib/Target/SPIRV/SPIRVPrepareFunctions.cpp

   1 //===-- SPIRVPrepareFunctions.cpp - modify function signatures --*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This pass modifies function signatures containing aggregate arguments
  10 // and/or return value before IRTranslator. Information about the original
  11 // signatures is stored in metadata. It is used during call lowering to
  12 // restore correct SPIR-V types of function arguments and return values.
  13 // This pass also substitutes some llvm intrinsic calls with calls to newly
  14 // generated functions (as the Khronos LLVM/SPIR-V Translator does).
  15 //
  16 // NOTE: this pass is a module-level one due to the necessity to modify
  17 // GVs/functions.
  18 //
  19 //===----------------------------------------------------------------------===//
  20
  21 #include "SPIRV.h"
  22 #include "SPIRVSubtarget.h"
  23 #include "SPIRVTargetMachine.h"
  24 #include "SPIRVUtils.h"
  25 #include "llvm/Analysis/ValueTracking.h"
  26 #include "llvm/CodeGen/IntrinsicLowering.h"
  27 #include "llvm/IR/IRBuilder.h"
  28 #include "llvm/IR/IntrinsicInst.h"
  29 #include "llvm/IR/Intrinsics.h"
  30 #include "llvm/IR/IntrinsicsSPIRV.h"
  31 #include "llvm/Transforms/Utils/Cloning.h"
  32 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
  33 #include <charconv>
  34 #include <regex>
  35
  36 using namespace llvm;
  37
  38 namespace llvm {
  39 void initializeSPIRVPrepareFunctionsPass(PassRegistry &);
  40 }
  41
  42 namespace {
  43
  44 class SPIRVPrepareFunctions : public ModulePass {
  45   const SPIRVTargetMachine &TM;
  46   bool substituteIntrinsicCalls(Function *F);
  47   Function *removeAggregateTypesFromSignature(Function *F);
  48
  49 public:
  50   static char ID;
  51   SPIRVPrepareFunctions(const SPIRVTargetMachine &TM) : ModulePass(ID), TM(TM) {
  52     initializeSPIRVPrepareFunctionsPass(*PassRegistry::getPassRegistry());
  53   }
  54
  55   bool runOnModule(Module &M) override;
  56
  57   StringRef getPassName() const override { return "SPIRV prepare functions"; }
  58
  59   void getAnalysisUsage(AnalysisUsage &AU) const override {
  60     ModulePass::getAnalysisUsage(AU);
  61   }
  62 };
  63
  64 } // namespace
  65
  66 char SPIRVPrepareFunctions::ID = 0;
  67
  68 INITIALIZE_PASS(SPIRVPrepareFunctions, "prepare-functions",
  69                 "SPIRV prepare functions", false, false)
  70
  71 std::string lowerLLVMIntrinsicName(IntrinsicInst *II) {
  72   Function *IntrinsicFunc = II->getCalledFunction();
  73   assert(IntrinsicFunc && "Missing function");
  74   std::string FuncName = IntrinsicFunc->getName().str();
  75   std::replace(FuncName.begin(), FuncName.end(), '.', '_');
  76   FuncName = "spirv." + FuncName;
  77   return FuncName;
  78 }
  79
  80 static Function *getOrCreateFunction(Module *M, Type *RetTy,
  81                                      ArrayRef<Type *> ArgTypes,
  82                                      StringRef Name) {
  83   FunctionType *FT = FunctionType::get(RetTy, ArgTypes, false);
  84   Function *F = M->getFunction(Name);
  85   if (F && F->getFunctionType() == FT)
  86     return F;
  87   Function *NewF = Function::Create(FT, GlobalValue::ExternalLinkage, Name, M);
  88   if (F)
  89     NewF->setDSOLocal(F->isDSOLocal());
  90   NewF->setCallingConv(CallingConv::SPIR_FUNC);
  91   return NewF;
  92 }
  93
  94 static bool lowerIntrinsicToFunction(IntrinsicInst *Intrinsic) {
  95   // For @llvm.memset.* intrinsic cases with constant value and length arguments
  96   // are emulated via "storing" a constant array to the destination. For other
  97   // cases we wrap the intrinsic in @spirv.llvm_memset_* function and expand the
  98   // intrinsic to a loop via expandMemSetAsLoop().
  99   if (auto *MSI = dyn_cast<MemSetInst>(Intrinsic))
 100     if (isa<Constant>(MSI->getValue()) && isa<ConstantInt>(MSI->getLength()))
 101       return false; // It is handled later using OpCopyMemorySized.
 102
 103   Module *M = Intrinsic->getModule();
 104   std::string FuncName = lowerLLVMIntrinsicName(Intrinsic);
 105   if (Intrinsic->isVolatile())
 106     FuncName += ".volatile";
 107   // Redirect @llvm.intrinsic.* call to @spirv.llvm_intrinsic_*
 108   Function *F = M->getFunction(FuncName);
 109   if (F) {
 110     Intrinsic->setCalledFunction(F);
 111     return true;
 112   }
 113   // TODO copy arguments attributes: nocapture writeonly.
 114   FunctionCallee FC =
 115       M->getOrInsertFunction(FuncName, Intrinsic->getFunctionType());
 116   auto IntrinsicID = Intrinsic->getIntrinsicID();
 117   Intrinsic->setCalledFunction(FC);
 118
 119   F = dyn_cast<Function>(FC.getCallee());
 120   assert(F && "Callee must be a function");
 121
 122   switch (IntrinsicID) {
 123   case Intrinsic::memset: {
 124     auto *MSI = static_cast<MemSetInst *>(Intrinsic);
 125     Argument *Dest = F->getArg(0);
 126     Argument *Val = F->getArg(1);
 127     Argument *Len = F->getArg(2);
 128     Argument *IsVolatile = F->getArg(3);
 129     Dest->setName("dest");
 130     Val->setName("val");
 131     Len->setName("len");
 132     IsVolatile->setName("isvolatile");
 133     BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F);
 134     IRBuilder<> IRB(EntryBB);
 135     auto *MemSet = IRB.CreateMemSet(Dest, Val, Len, MSI->getDestAlign(),
 136                                     MSI->isVolatile());
 137     IRB.CreateRetVoid();
 138     expandMemSetAsLoop(cast<MemSetInst>(MemSet));
 139     MemSet->eraseFromParent();
 140     break;
 141   }
 142   case Intrinsic::bswap: {
 143     BasicBlock *EntryBB = BasicBlock::Create(M->getContext(), "entry", F);
 144     IRBuilder<> IRB(EntryBB);
 145     auto *BSwap = IRB.CreateIntrinsic(Intrinsic::bswap, Intrinsic->getType(),
 146                                       F->getArg(0));
 147     IRB.CreateRet(BSwap);
 148     IntrinsicLowering IL(M->getDataLayout());
 149     IL.LowerIntrinsicCall(BSwap);
 150     break;
 151   }
 152   default:
 153     break;
 154   }
 155   return true;
 156 }
 157
 158 static std::string getAnnotation(Value *AnnoVal, Value *OptAnnoVal) {
 159   if (auto *Ref = dyn_cast_or_null<GetElementPtrInst>(AnnoVal))
 160     AnnoVal = Ref->getOperand(0);
 161   if (auto *Ref = dyn_cast_or_null<BitCastInst>(OptAnnoVal))
 162     OptAnnoVal = Ref->getOperand(0);
 163
 164   std::string Anno;
 165   if (auto *C = dyn_cast_or_null<Constant>(AnnoVal)) {
 166     StringRef Str;
 167     if (getConstantStringInfo(C, Str))
 168       Anno = Str;
 169   }
 170   // handle optional annotation parameter in a way that Khronos Translator do
 171   // (collect integers wrapped in a struct)
 172   if (auto *C = dyn_cast_or_null<Constant>(OptAnnoVal);
 173       C && C->getNumOperands()) {
 174     Value *MaybeStruct = C->getOperand(0);
 175     if (auto *Struct = dyn_cast<ConstantStruct>(MaybeStruct)) {
 176       for (unsigned I = 0, E = Struct->getNumOperands(); I != E; ++I) {
 177         if (auto *CInt = dyn_cast<ConstantInt>(Struct->getOperand(I)))
 178           Anno += (I == 0 ? ": " : ", ") +
 179                   std::to_string(CInt->getType()->getIntegerBitWidth() == 1
 180                                      ? CInt->getZExtValue()
 181                                      : CInt->getSExtValue());
 182       }
 183     } else if (auto *Struct = dyn_cast<ConstantAggregateZero>(MaybeStruct)) {
 184       // { i32 i32 ... } zeroinitializer
 185       for (unsigned I = 0, E = Struct->getType()->getStructNumElements();
 186            I != E; ++I)
 187         Anno += I == 0 ? ": 0" : ", 0";
 188     }
 189   }
 190   return Anno;
 191 }
 192
 193 static SmallVector<Metadata *> parseAnnotation(Value *I,
 194                                                const std::string &Anno,
 195                                                LLVMContext &Ctx,
 196                                                Type *Int32Ty) {
 197   // Try to parse the annotation string according to the following rules:
 198   // annotation := ({kind} | {kind:value,value,...})+
 199   // kind := number
 200   // value := number | string
 201   static const std::regex R(
 202       "\\{(\\d+)(?:[:,](\\d+|\"[^\"]*\")(?:,(\\d+|\"[^\"]*\"))*)?\\}");
 203   SmallVector<Metadata *> MDs;
 204   int Pos = 0;
 205   for (std::sregex_iterator
 206            It = std::sregex_iterator(Anno.begin(), Anno.end(), R),
 207            ItEnd = std::sregex_iterator();
 208        It != ItEnd; ++It) {
 209     if (It->position() != Pos)
 210       return SmallVector<Metadata *>{};
 211     Pos = It->position() + It->length();
 212     std::smatch Match = *It;
 213     SmallVector<Metadata *> MDsItem;
 214     for (std::size_t i = 1; i < Match.size(); ++i) {
 215       std::ssub_match SMatch = Match[i];
 216       std::string Item = SMatch.str();
 217       if (Item.length() == 0)
 218         break;
 219       if (Item[0] == '"') {
 220         Item = Item.substr(1, Item.length() - 2);
 221         // Acceptable format of the string snippet is:
 222         static const std::regex RStr("^(\\d+)(?:,(\\d+))*$");
 223         if (std::smatch MatchStr; std::regex_match(Item, MatchStr, RStr)) {
 224           for (std::size_t SubIdx = 1; SubIdx < MatchStr.size(); ++SubIdx)
 225             if (std::string SubStr = MatchStr[SubIdx].str(); SubStr.length())
 226               MDsItem.push_back(ConstantAsMetadata::get(
 227                   ConstantInt::get(Int32Ty, std::stoi(SubStr))));
 228         } else {
 229           MDsItem.push_back(MDString::get(Ctx, Item));
 230         }
 231       } else if (int32_t Num;
 232                  std::from_chars(Item.data(), Item.data() + Item.size(), Num)
 233                      .ec == std::errc{}) {
 234         MDsItem.push_back(
 235             ConstantAsMetadata::get(ConstantInt::get(Int32Ty, Num)));
 236       } else {
 237         MDsItem.push_back(MDString::get(Ctx, Item));
 238       }
 239     }
 240     if (MDsItem.size() == 0)
 241       return SmallVector<Metadata *>{};
 242     MDs.push_back(MDNode::get(Ctx, MDsItem));
 243   }
 244   return Pos == static_cast<int>(Anno.length()) ? MDs
 245                                                 : SmallVector<Metadata *>{};
 246 }
 247
 248 static void lowerPtrAnnotation(IntrinsicInst *II) {
 249   LLVMContext &Ctx = II->getContext();
 250   Type *Int32Ty = Type::getInt32Ty(Ctx);
 251
 252   // Retrieve an annotation string from arguments.
 253   Value *PtrArg = nullptr;
 254   if (auto *BI = dyn_cast<BitCastInst>(II->getArgOperand(0)))
 255     PtrArg = BI->getOperand(0);
 256   else
 257     PtrArg = II->getOperand(0);
 258   std::string Anno =
 259       getAnnotation(II->getArgOperand(1),
 260                     4 < II->arg_size() ? II->getArgOperand(4) : nullptr);
 261
 262   // Parse the annotation.
 263   SmallVector<Metadata *> MDs = parseAnnotation(II, Anno, Ctx, Int32Ty);
 264
 265   // If the annotation string is not parsed successfully we don't know the
 266   // format used and output it as a general UserSemantic decoration.
 267   // Otherwise MDs is a Metadata tuple (a decoration list) in the format
 268   // expected by `spirv.Decorations`.
 269   if (MDs.size() == 0) {
 270     auto UserSemantic = ConstantAsMetadata::get(ConstantInt::get(
 271         Int32Ty, static_cast<uint32_t>(SPIRV::Decoration::UserSemantic)));
 272     MDs.push_back(MDNode::get(Ctx, {UserSemantic, MDString::get(Ctx, Anno)}));
 273   }
 274
 275   // Build the internal intrinsic function.
 276   IRBuilder<> IRB(II->getParent());
 277   IRB.SetInsertPoint(II);
 278   IRB.CreateIntrinsic(
 279       Intrinsic::spv_assign_decoration, {PtrArg->getType()},
 280       {PtrArg, MetadataAsValue::get(Ctx, MDNode::get(Ctx, MDs))});
 281   II->replaceAllUsesWith(II->getOperand(0));
 282 }
 283
 284 static void lowerFunnelShifts(IntrinsicInst *FSHIntrinsic) {
 285   // Get a separate function - otherwise, we'd have to rework the CFG of the
 286   // current one. Then simply replace the intrinsic uses with a call to the new
 287   // function.
 288   // Generate LLVM IR for  i* @spirv.llvm_fsh?_i* (i* %a, i* %b, i* %c)
 289   Module *M = FSHIntrinsic->getModule();
 290   FunctionType *FSHFuncTy = FSHIntrinsic->getFunctionType();
 291   Type *FSHRetTy = FSHFuncTy->getReturnType();
 292   const std::string FuncName = lowerLLVMIntrinsicName(FSHIntrinsic);
 293   Function *FSHFunc =
 294       getOrCreateFunction(M, FSHRetTy, FSHFuncTy->params(), FuncName);
 295
 296   if (!FSHFunc->empty()) {
 297     FSHIntrinsic->setCalledFunction(FSHFunc);
 298     return;
 299   }
 300   BasicBlock *RotateBB = BasicBlock::Create(M->getContext(), "rotate", FSHFunc);
 301   IRBuilder<> IRB(RotateBB);
 302   Type *Ty = FSHFunc->getReturnType();
 303   // Build the actual funnel shift rotate logic.
 304   // In the comments, "int" is used interchangeably with "vector of int
 305   // elements".
 306   FixedVectorType *VectorTy = dyn_cast<FixedVectorType>(Ty);
 307   Type *IntTy = VectorTy ? VectorTy->getElementType() : Ty;
 308   unsigned BitWidth = IntTy->getIntegerBitWidth();
 309   ConstantInt *BitWidthConstant = IRB.getInt({BitWidth, BitWidth});
 310   Value *BitWidthForInsts =
 311       VectorTy
 312           ? IRB.CreateVectorSplat(VectorTy->getNumElements(), BitWidthConstant)
 313           : BitWidthConstant;
 314   Value *RotateModVal =
 315       IRB.CreateURem(/*Rotate*/ FSHFunc->getArg(2), BitWidthForInsts);
 316   Value *FirstShift = nullptr, *SecShift = nullptr;
 317   if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
 318     // Shift the less significant number right, the "rotate" number of bits
 319     // will be 0-filled on the left as a result of this regular shift.
 320     FirstShift = IRB.CreateLShr(FSHFunc->getArg(1), RotateModVal);
 321   } else {
 322     // Shift the more significant number left, the "rotate" number of bits
 323     // will be 0-filled on the right as a result of this regular shift.
 324     FirstShift = IRB.CreateShl(FSHFunc->getArg(0), RotateModVal);
 325   }
 326   // We want the "rotate" number of the more significant int's LSBs (MSBs) to
 327   // occupy the leftmost (rightmost) "0 space" left by the previous operation.
 328   // Therefore, subtract the "rotate" number from the integer bitsize...
 329   Value *SubRotateVal = IRB.CreateSub(BitWidthForInsts, RotateModVal);
 330   if (FSHIntrinsic->getIntrinsicID() == Intrinsic::fshr) {
 331     // ...and left-shift the more significant int by this number, zero-filling
 332     // the LSBs.
 333     SecShift = IRB.CreateShl(FSHFunc->getArg(0), SubRotateVal);
 334   } else {
 335     // ...and right-shift the less significant int by this number, zero-filling
 336     // the MSBs.
 337     SecShift = IRB.CreateLShr(FSHFunc->getArg(1), SubRotateVal);
 338   }
 339   // A simple binary addition of the shifted ints yields the final result.
 340   IRB.CreateRet(IRB.CreateOr(FirstShift, SecShift));
 341
 342   FSHIntrinsic->setCalledFunction(FSHFunc);
 343 }
 344
 345 static void lowerExpectAssume(IntrinsicInst *II) {
 346   // If we cannot use the SPV_KHR_expect_assume extension, then we need to
 347   // ignore the intrinsic and move on. It should be removed later on by LLVM.
 348   // Otherwise we should lower the intrinsic to the corresponding SPIR-V
 349   // instruction.
 350   // For @llvm.assume we have OpAssumeTrueKHR.
 351   // For @llvm.expect we have OpExpectKHR.
 352   //
 353   // We need to lower this into a builtin and then the builtin into a SPIR-V
 354   // instruction.
 355   if (II->getIntrinsicID() == Intrinsic::assume) {
 356     Function *F = Intrinsic::getOrInsertDeclaration(
 357         II->getModule(), Intrinsic::SPVIntrinsics::spv_assume);
 358     II->setCalledFunction(F);
 359   } else if (II->getIntrinsicID() == Intrinsic::expect) {
 360     Function *F = Intrinsic::getOrInsertDeclaration(
 361         II->getModule(), Intrinsic::SPVIntrinsics::spv_expect,
 362         {II->getOperand(0)->getType()});
 363     II->setCalledFunction(F);
 364   } else {
 365     llvm_unreachable("Unknown intrinsic");
 366   }
 367
 368   return;
 369 }
 370
 371 static bool toSpvOverloadedIntrinsic(IntrinsicInst *II, Intrinsic::ID NewID,
 372                                      ArrayRef<unsigned> OpNos) {
 373   Function *F = nullptr;
 374   if (OpNos.empty()) {
 375     F = Intrinsic::getOrInsertDeclaration(II->getModule(), NewID);
 376   } else {
 377     SmallVector<Type *, 4> Tys;
 378     for (unsigned OpNo : OpNos)
 379       Tys.push_back(II->getOperand(OpNo)->getType());
 380     F = Intrinsic::getOrInsertDeclaration(II->getModule(), NewID, Tys);
 381   }
 382   II->setCalledFunction(F);
 383   return true;
 384 }
 385
 386 // Substitutes calls to LLVM intrinsics with either calls to SPIR-V intrinsics
 387 // or calls to proper generated functions. Returns True if F was modified.
 388 bool SPIRVPrepareFunctions::substituteIntrinsicCalls(Function *F) {
 389   bool Changed = false;
 390   for (BasicBlock &BB : *F) {
 391     for (Instruction &I : BB) {
 392       auto Call = dyn_cast<CallInst>(&I);
 393       if (!Call)
 394         continue;
 395       Function *CF = Call->getCalledFunction();
 396       if (!CF || !CF->isIntrinsic())
 397         continue;
 398       auto *II = cast<IntrinsicInst>(Call);
 399       switch (II->getIntrinsicID()) {
 400       case Intrinsic::memset:
 401       case Intrinsic::bswap:
 402         Changed |= lowerIntrinsicToFunction(II);
 403         break;
 404       case Intrinsic::fshl:
 405       case Intrinsic::fshr:
 406         lowerFunnelShifts(II);
 407         Changed = true;
 408         break;
 409       case Intrinsic::assume:
 410       case Intrinsic::expect: {
 411         const SPIRVSubtarget &STI = TM.getSubtarget<SPIRVSubtarget>(*F);
 412         if (STI.canUseExtension(SPIRV::Extension::SPV_KHR_expect_assume))
 413           lowerExpectAssume(II);
 414         Changed = true;
 415       } break;
 416       case Intrinsic::lifetime_start:
 417         Changed |= toSpvOverloadedIntrinsic(
 418             II, Intrinsic::SPVIntrinsics::spv_lifetime_start, {1});
 419         break;
 420       case Intrinsic::lifetime_end:
 421         Changed |= toSpvOverloadedIntrinsic(
 422             II, Intrinsic::SPVIntrinsics::spv_lifetime_end, {1});
 423         break;
 424       case Intrinsic::ptr_annotation:
 425         lowerPtrAnnotation(II);
 426         Changed = true;
 427         break;
 428       }
 429     }
 430   }
 431   return Changed;
 432 }
 433
 434 // Returns F if aggregate argument/return types are not present or cloned F
 435 // function with the types replaced by i32 types. The change in types is
 436 // noted in 'spv.cloned_funcs' metadata for later restoration.
 437 Function *
 438 SPIRVPrepareFunctions::removeAggregateTypesFromSignature(Function *F) {
 439   bool IsRetAggr = F->getReturnType()->isAggregateType();
 440   // Allow intrinsics with aggregate return type to reach GlobalISel
 441   if (F->isIntrinsic() && IsRetAggr)
 442     return F;
 443
 444   IRBuilder<> B(F->getContext());
 445
 446   bool HasAggrArg =
 447       std::any_of(F->arg_begin(), F->arg_end(), [](Argument &Arg) {
 448         return Arg.getType()->isAggregateType();
 449       });
 450   bool DoClone = IsRetAggr || HasAggrArg;
 451   if (!DoClone)
 452     return F;
 453   SmallVector<std::pair<int, Type *>, 4> ChangedTypes;
 454   Type *RetType = IsRetAggr ? B.getInt32Ty() : F->getReturnType();
 455   if (IsRetAggr)
 456     ChangedTypes.push_back(std::pair<int, Type *>(-1, F->getReturnType()));
 457   SmallVector<Type *, 4> ArgTypes;
 458   for (const auto &Arg : F->args()) {
 459     if (Arg.getType()->isAggregateType()) {
 460       ArgTypes.push_back(B.getInt32Ty());
 461       ChangedTypes.push_back(
 462           std::pair<int, Type *>(Arg.getArgNo(), Arg.getType()));
 463     } else
 464       ArgTypes.push_back(Arg.getType());
 465   }
 466   FunctionType *NewFTy =
 467       FunctionType::get(RetType, ArgTypes, F->getFunctionType()->isVarArg());
 468   Function *NewF =
 469       Function::Create(NewFTy, F->getLinkage(), F->getName(), *F->getParent());
 470
 471   ValueToValueMapTy VMap;
 472   auto NewFArgIt = NewF->arg_begin();
 473   for (auto &Arg : F->args()) {
 474     StringRef ArgName = Arg.getName();
 475     NewFArgIt->setName(ArgName);
 476     VMap[&Arg] = &(*NewFArgIt++);
 477   }
 478   SmallVector<ReturnInst *, 8> Returns;
 479
 480   CloneFunctionInto(NewF, F, VMap, CloneFunctionChangeType::LocalChangesOnly,
 481                     Returns);
 482   NewF->takeName(F);
 483
 484   NamedMDNode *FuncMD =
 485       F->getParent()->getOrInsertNamedMetadata("spv.cloned_funcs");
 486   SmallVector<Metadata *, 2> MDArgs;
 487   MDArgs.push_back(MDString::get(B.getContext(), NewF->getName()));
 488   for (auto &ChangedTyP : ChangedTypes)
 489     MDArgs.push_back(MDNode::get(
 490         B.getContext(),
 491         {ConstantAsMetadata::get(B.getInt32(ChangedTyP.first)),
 492          ValueAsMetadata::get(Constant::getNullValue(ChangedTyP.second))}));
 493   MDNode *ThisFuncMD = MDNode::get(B.getContext(), MDArgs);
 494   FuncMD->addOperand(ThisFuncMD);
 495
 496   for (auto *U : make_early_inc_range(F->users())) {
 497     if (auto *CI = dyn_cast<CallInst>(U))
 498       CI->mutateFunctionType(NewF->getFunctionType());
 499     U->replaceUsesOfWith(F, NewF);
 500   }
 501
 502   // register the mutation
 503   if (RetType != F->getReturnType())
 504     TM.getSubtarget<SPIRVSubtarget>(*F).getSPIRVGlobalRegistry()->addMutated(
 505         NewF, F->getReturnType());
 506   return NewF;
 507 }
 508
 509 bool SPIRVPrepareFunctions::runOnModule(Module &M) {
 510   bool Changed = false;
 511   for (Function &F : M) {
 512     Changed |= substituteIntrinsicCalls(&F);
 513     Changed |= sortBlocks(F);
 514   }
 515
 516   std::vector<Function *> FuncsWorklist;
 517   for (auto &F : M)
 518     FuncsWorklist.push_back(&F);
 519
 520   for (auto *F : FuncsWorklist) {
 521     Function *NewF = removeAggregateTypesFromSignature(F);
 522
 523     if (NewF != F) {
 524       F->eraseFromParent();
 525       Changed = true;
 526     }
 527   }
 528   return Changed;
 529 }
 530
 531 ModulePass *
 532 llvm::createSPIRVPrepareFunctionsPass(const SPIRVTargetMachine &TM) {
 533   return new SPIRVPrepareFunctions(TM);
 534 }