lib/IR/IRBuilder.cpp

   1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file implements the IRBuilder class, which is used as a convenient way
  11 // to create LLVM instructions with a consistent and simplified interface.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "llvm/IR/IRBuilder.h"
  16 #include "llvm/ADT/ArrayRef.h"
  17 #include "llvm/ADT/None.h"
  18 #include "llvm/IR/Constant.h"
  19 #include "llvm/IR/Constants.h"
  20 #include "llvm/IR/DerivedTypes.h"
  21 #include "llvm/IR/Function.h"
  22 #include "llvm/IR/GlobalValue.h"
  23 #include "llvm/IR/GlobalVariable.h"
  24 #include "llvm/IR/IntrinsicInst.h"
  25 #include "llvm/IR/Intrinsics.h"
  26 #include "llvm/IR/LLVMContext.h"
  27 #include "llvm/IR/Operator.h"
  28 #include "llvm/IR/Statepoint.h"
  29 #include "llvm/IR/Type.h"
  30 #include "llvm/IR/Value.h"
  31 #include "llvm/Support/Casting.h"
  32 #include "llvm/Support/MathExtras.h"
  33 #include <cassert>
  34 #include <cstdint>
  35 #include <vector>
  36
  37 using namespace llvm;
  38
  39 /// CreateGlobalString - Make a new global variable with an initializer that
  40 /// has array of i8 type filled in with the nul terminated string value
  41 /// specified.  If Name is specified, it is the name of the global variable
  42 /// created.
  43 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
  44                                                   const Twine &Name,
  45                                                   unsigned AddressSpace) {
  46   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
  47   Module &M = *BB->getParent()->getParent();
  48   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
  49                                 GlobalValue::PrivateLinkage, StrConstant, Name,
  50                                 nullptr, GlobalVariable::NotThreadLocal,
  51                                 AddressSpace);
  52   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
  53   GV->setAlignment(1);
  54   return GV;
  55 }
  56
  57 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
  58   assert(BB && BB->getParent() && "No current function!");
  59   return BB->getParent()->getReturnType();
  60 }
  61
  62 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
  63   auto *PT = cast<PointerType>(Ptr->getType());
  64   if (PT->getElementType()->isIntegerTy(8))
  65     return Ptr;
  66
  67   // Otherwise, we need to insert a bitcast.
  68   PT = getInt8PtrTy(PT->getAddressSpace());
  69   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
  70   BB->getInstList().insert(InsertPt, BCI);
  71   SetInstDebugLocation(BCI);
  72   return BCI;
  73 }
  74
  75 static CallInst *createCallHelper(Value *Callee, ArrayRef<Value *> Ops,
  76                                   IRBuilderBase *Builder,
  77                                   const Twine &Name = "",
  78                                   Instruction *FMFSource = nullptr) {
  79   CallInst *CI = CallInst::Create(Callee, Ops, Name);
  80   if (FMFSource)
  81     CI->copyFastMathFlags(FMFSource);
  82   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
  83   Builder->SetInstDebugLocation(CI);
  84   return CI;
  85 }
  86
  87 static InvokeInst *createInvokeHelper(Value *Invokee, BasicBlock *NormalDest,
  88                                       BasicBlock *UnwindDest,
  89                                       ArrayRef<Value *> Ops,
  90                                       IRBuilderBase *Builder,
  91                                       const Twine &Name = "") {
  92   InvokeInst *II =
  93       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
  94   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
  95                                                   II);
  96   Builder->SetInstDebugLocation(II);
  97   return II;
  98 }
  99
 100 CallInst *IRBuilderBase::
 101 CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
 102              bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
 103              MDNode *NoAliasTag) {
 104   Ptr = getCastedInt8PtrValue(Ptr);
 105   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
 106   Type *Tys[] = { Ptr->getType(), Size->getType() };
 107   Module *M = BB->getParent()->getParent();
 108   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
 109
 110   CallInst *CI = createCallHelper(TheFn, Ops, this);
 111
 112   if (Align > 0)
 113     cast<MemSetInst>(CI)->setDestAlignment(Align);
 114
 115   // Set the TBAA info if present.
 116   if (TBAATag)
 117     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 118
 119   if (ScopeTag)
 120     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 121
 122   if (NoAliasTag)
 123     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 124
 125   return CI;
 126 }
 127
 128 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
 129     Value *Ptr, Value *Val, Value *Size, unsigned Align, uint32_t ElementSize,
 130     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
 131   assert(Align >= ElementSize &&
 132          "Pointer alignment must be at least element size.");
 133
 134   Ptr = getCastedInt8PtrValue(Ptr);
 135   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
 136   Type *Tys[] = {Ptr->getType(), Size->getType()};
 137   Module *M = BB->getParent()->getParent();
 138   Value *TheFn = Intrinsic::getDeclaration(
 139       M, Intrinsic::memset_element_unordered_atomic, Tys);
 140
 141   CallInst *CI = createCallHelper(TheFn, Ops, this);
 142
 143   cast<AtomicMemSetInst>(CI)->setDestAlignment(Align);
 144
 145   // Set the TBAA info if present.
 146   if (TBAATag)
 147     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 148
 149   if (ScopeTag)
 150     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 151
 152   if (NoAliasTag)
 153     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 154
 155   return CI;
 156 }
 157
 158 CallInst *IRBuilderBase::
 159 CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
 160              Value *Size, bool isVolatile, MDNode *TBAATag,
 161              MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
 162   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
 163   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
 164   Dst = getCastedInt8PtrValue(Dst);
 165   Src = getCastedInt8PtrValue(Src);
 166
 167   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
 168   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
 169   Module *M = BB->getParent()->getParent();
 170   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
 171
 172   CallInst *CI = createCallHelper(TheFn, Ops, this);
 173
 174   auto* MCI = cast<MemCpyInst>(CI);
 175   if (DstAlign > 0)
 176     MCI->setDestAlignment(DstAlign);
 177   if (SrcAlign > 0)
 178     MCI->setSourceAlignment(SrcAlign);
 179
 180   // Set the TBAA info if present.
 181   if (TBAATag)
 182     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 183
 184   // Set the TBAA Struct info if present.
 185   if (TBAAStructTag)
 186     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 187
 188   if (ScopeTag)
 189     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 190
 191   if (NoAliasTag)
 192     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 193
 194   return CI;
 195 }
 196
 197 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
 198     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
 199     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
 200     MDNode *ScopeTag, MDNode *NoAliasTag) {
 201   assert(DstAlign >= ElementSize &&
 202          "Pointer alignment must be at least element size");
 203   assert(SrcAlign >= ElementSize &&
 204          "Pointer alignment must be at least element size");
 205   Dst = getCastedInt8PtrValue(Dst);
 206   Src = getCastedInt8PtrValue(Src);
 207
 208   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
 209   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 210   Module *M = BB->getParent()->getParent();
 211   Value *TheFn = Intrinsic::getDeclaration(
 212       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
 213
 214   CallInst *CI = createCallHelper(TheFn, Ops, this);
 215
 216   // Set the alignment of the pointer args.
 217   auto *AMCI = cast<AtomicMemCpyInst>(CI);
 218   AMCI->setDestAlignment(DstAlign);
 219   AMCI->setSourceAlignment(SrcAlign);
 220
 221   // Set the TBAA info if present.
 222   if (TBAATag)
 223     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 224
 225   // Set the TBAA Struct info if present.
 226   if (TBAAStructTag)
 227     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 228
 229   if (ScopeTag)
 230     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 231
 232   if (NoAliasTag)
 233     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 234
 235   return CI;
 236 }
 237
 238 CallInst *IRBuilderBase::
 239 CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
 240               Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
 241               MDNode *NoAliasTag) {
 242   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
 243   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
 244   Dst = getCastedInt8PtrValue(Dst);
 245   Src = getCastedInt8PtrValue(Src);
 246
 247   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
 248   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
 249   Module *M = BB->getParent()->getParent();
 250   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
 251
 252   CallInst *CI = createCallHelper(TheFn, Ops, this);
 253
 254   auto *MMI = cast<MemMoveInst>(CI);
 255   if (DstAlign > 0)
 256     MMI->setDestAlignment(DstAlign);
 257   if (SrcAlign > 0)
 258     MMI->setSourceAlignment(SrcAlign);
 259
 260   // Set the TBAA info if present.
 261   if (TBAATag)
 262     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 263
 264   if (ScopeTag)
 265     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 266
 267   if (NoAliasTag)
 268     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 269
 270   return CI;
 271 }
 272
 273 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
 274     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
 275     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
 276     MDNode *ScopeTag, MDNode *NoAliasTag) {
 277   assert(DstAlign >= ElementSize &&
 278          "Pointer alignment must be at least element size");
 279   assert(SrcAlign >= ElementSize &&
 280          "Pointer alignment must be at least element size");
 281   Dst = getCastedInt8PtrValue(Dst);
 282   Src = getCastedInt8PtrValue(Src);
 283
 284   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
 285   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 286   Module *M = BB->getParent()->getParent();
 287   Value *TheFn = Intrinsic::getDeclaration(
 288       M, Intrinsic::memmove_element_unordered_atomic, Tys);
 289
 290   CallInst *CI = createCallHelper(TheFn, Ops, this);
 291
 292   // Set the alignment of the pointer args.
 293   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
 294   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
 295
 296   // Set the TBAA info if present.
 297   if (TBAATag)
 298     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 299
 300   // Set the TBAA Struct info if present.
 301   if (TBAAStructTag)
 302     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 303
 304   if (ScopeTag)
 305     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 306
 307   if (NoAliasTag)
 308     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 309
 310   return CI;
 311 }
 312
 313 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
 314                                     Value *Src) {
 315   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 316   Value *Ops[] = {Src};
 317   Type *Tys[] = { Src->getType()->getVectorElementType(), Src->getType() };
 318   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
 319   return createCallHelper(Decl, Ops, Builder);
 320 }
 321
 322 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
 323   Module *M = GetInsertBlock()->getParent()->getParent();
 324   Value *Ops[] = {Acc, Src};
 325   Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
 326                  Src->getType()};
 327   auto Decl = Intrinsic::getDeclaration(
 328       M, Intrinsic::experimental_vector_reduce_fadd, Tys);
 329   return createCallHelper(Decl, Ops, this);
 330 }
 331
 332 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
 333   Module *M = GetInsertBlock()->getParent()->getParent();
 334   Value *Ops[] = {Acc, Src};
 335   Type *Tys[] = {Src->getType()->getVectorElementType(), Acc->getType(),
 336                  Src->getType()};
 337   auto Decl = Intrinsic::getDeclaration(
 338       M, Intrinsic::experimental_vector_reduce_fmul, Tys);
 339   return createCallHelper(Decl, Ops, this);
 340 }
 341
 342 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
 343   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
 344                                Src);
 345 }
 346
 347 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
 348   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
 349                                Src);
 350 }
 351
 352 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
 353   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
 354                                Src);
 355 }
 356
 357 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
 358   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
 359                                Src);
 360 }
 361
 362 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
 363   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
 364                                Src);
 365 }
 366
 367 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
 368   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
 369                      : Intrinsic::experimental_vector_reduce_umax;
 370   return getReductionIntrinsic(this, ID, Src);
 371 }
 372
 373 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
 374   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
 375                      : Intrinsic::experimental_vector_reduce_umin;
 376   return getReductionIntrinsic(this, ID, Src);
 377 }
 378
 379 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
 380   auto Rdx = getReductionIntrinsic(
 381       this, Intrinsic::experimental_vector_reduce_fmax, Src);
 382   if (NoNaN) {
 383     FastMathFlags FMF;
 384     FMF.setNoNaNs();
 385     Rdx->setFastMathFlags(FMF);
 386   }
 387   return Rdx;
 388 }
 389
 390 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
 391   auto Rdx = getReductionIntrinsic(
 392       this, Intrinsic::experimental_vector_reduce_fmin, Src);
 393   if (NoNaN) {
 394     FastMathFlags FMF;
 395     FMF.setNoNaNs();
 396     Rdx->setFastMathFlags(FMF);
 397   }
 398   return Rdx;
 399 }
 400
 401 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
 402   assert(isa<PointerType>(Ptr->getType()) &&
 403          "lifetime.start only applies to pointers.");
 404   Ptr = getCastedInt8PtrValue(Ptr);
 405   if (!Size)
 406     Size = getInt64(-1);
 407   else
 408     assert(Size->getType() == getInt64Ty() &&
 409            "lifetime.start requires the size to be an i64");
 410   Value *Ops[] = { Size, Ptr };
 411   Module *M = BB->getParent()->getParent();
 412   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_start,
 413                                            { Ptr->getType() });
 414   return createCallHelper(TheFn, Ops, this);
 415 }
 416
 417 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
 418   assert(isa<PointerType>(Ptr->getType()) &&
 419          "lifetime.end only applies to pointers.");
 420   Ptr = getCastedInt8PtrValue(Ptr);
 421   if (!Size)
 422     Size = getInt64(-1);
 423   else
 424     assert(Size->getType() == getInt64Ty() &&
 425            "lifetime.end requires the size to be an i64");
 426   Value *Ops[] = { Size, Ptr };
 427   Module *M = BB->getParent()->getParent();
 428   Value *TheFn = Intrinsic::getDeclaration(M, Intrinsic::lifetime_end,
 429                                            { Ptr->getType() });
 430   return createCallHelper(TheFn, Ops, this);
 431 }
 432
 433 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
 434
 435   assert(isa<PointerType>(Ptr->getType()) &&
 436          "invariant.start only applies to pointers.");
 437   Ptr = getCastedInt8PtrValue(Ptr);
 438   if (!Size)
 439     Size = getInt64(-1);
 440   else
 441     assert(Size->getType() == getInt64Ty() &&
 442            "invariant.start requires the size to be an i64");
 443
 444   Value *Ops[] = {Size, Ptr};
 445   // Fill in the single overloaded type: memory object type.
 446   Type *ObjectPtr[1] = {Ptr->getType()};
 447   Module *M = BB->getParent()->getParent();
 448   Value *TheFn =
 449       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
 450   return createCallHelper(TheFn, Ops, this);
 451 }
 452
 453 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
 454   assert(Cond->getType() == getInt1Ty() &&
 455          "an assumption condition must be of type i1");
 456
 457   Value *Ops[] = { Cond };
 458   Module *M = BB->getParent()->getParent();
 459   Value *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
 460   return createCallHelper(FnAssume, Ops, this);
 461 }
 462
 463 /// Create a call to a Masked Load intrinsic.
 464 /// \p Ptr      - base pointer for the load
 465 /// \p Align    - alignment of the source location
 466 /// \p Mask     - vector of booleans which indicates what vector lanes should
 467 ///               be accessed in memory
 468 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
 469 ///               of the result
 470 /// \p Name     - name of the result variable
 471 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
 472                                           Value *Mask, Value *PassThru,
 473                                           const Twine &Name) {
 474   auto *PtrTy = cast<PointerType>(Ptr->getType());
 475   Type *DataTy = PtrTy->getElementType();
 476   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
 477   assert(Mask && "Mask should not be all-ones (null)");
 478   if (!PassThru)
 479     PassThru = UndefValue::get(DataTy);
 480   Type *OverloadedTypes[] = { DataTy, PtrTy };
 481   Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
 482   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
 483                                OverloadedTypes, Name);
 484 }
 485
 486 /// Create a call to a Masked Store intrinsic.
 487 /// \p Val   - data to be stored,
 488 /// \p Ptr   - base pointer for the store
 489 /// \p Align - alignment of the destination location
 490 /// \p Mask  - vector of booleans which indicates what vector lanes should
 491 ///            be accessed in memory
 492 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
 493                                            unsigned Align, Value *Mask) {
 494   auto *PtrTy = cast<PointerType>(Ptr->getType());
 495   Type *DataTy = PtrTy->getElementType();
 496   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
 497   assert(Mask && "Mask should not be all-ones (null)");
 498   Type *OverloadedTypes[] = { DataTy, PtrTy };
 499   Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
 500   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
 501 }
 502
 503 /// Create a call to a Masked intrinsic, with given intrinsic Id,
 504 /// an array of operands - Ops, and an array of overloaded types -
 505 /// OverloadedTypes.
 506 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
 507                                                ArrayRef<Value *> Ops,
 508                                                ArrayRef<Type *> OverloadedTypes,
 509                                                const Twine &Name) {
 510   Module *M = BB->getParent()->getParent();
 511   Value *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
 512   return createCallHelper(TheFn, Ops, this, Name);
 513 }
 514
 515 /// Create a call to a Masked Gather intrinsic.
 516 /// \p Ptrs     - vector of pointers for loading
 517 /// \p Align    - alignment for one element
 518 /// \p Mask     - vector of booleans which indicates what vector lanes should
 519 ///               be accessed in memory
 520 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
 521 ///               of the result
 522 /// \p Name     - name of the result variable
 523 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
 524                                             Value *Mask,  Value *PassThru,
 525                                             const Twine& Name) {
 526   auto PtrsTy = cast<VectorType>(Ptrs->getType());
 527   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
 528   unsigned NumElts = PtrsTy->getVectorNumElements();
 529   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
 530
 531   if (!Mask)
 532     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
 533                                      NumElts));
 534
 535   if (!PassThru)
 536     PassThru = UndefValue::get(DataTy);
 537
 538   Type *OverloadedTypes[] = {DataTy, PtrsTy};
 539   Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
 540
 541   // We specify only one type when we create this intrinsic. Types of other
 542   // arguments are derived from this type.
 543   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
 544                                Name);
 545 }
 546
 547 /// Create a call to a Masked Scatter intrinsic.
 548 /// \p Data  - data to be stored,
 549 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
 550 ///            stored
 551 /// \p Align - alignment for one element
 552 /// \p Mask  - vector of booleans which indicates what vector lanes should
 553 ///            be accessed in memory
 554 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
 555                                              unsigned Align, Value *Mask) {
 556   auto PtrsTy = cast<VectorType>(Ptrs->getType());
 557   auto DataTy = cast<VectorType>(Data->getType());
 558   unsigned NumElts = PtrsTy->getVectorNumElements();
 559
 560 #ifndef NDEBUG
 561   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
 562   assert(NumElts == DataTy->getVectorNumElements() &&
 563          PtrTy->getElementType() == DataTy->getElementType() &&
 564          "Incompatible pointer and data types");
 565 #endif
 566
 567   if (!Mask)
 568     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
 569                                      NumElts));
 570
 571   Type *OverloadedTypes[] = {DataTy, PtrsTy};
 572   Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
 573
 574   // We specify only one type when we create this intrinsic. Types of other
 575   // arguments are derived from this type.
 576   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
 577 }
 578
 579 template <typename T0, typename T1, typename T2, typename T3>
 580 static std::vector<Value *>
 581 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
 582                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
 583                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
 584                   ArrayRef<T3> GCArgs) {
 585   std::vector<Value *> Args;
 586   Args.push_back(B.getInt64(ID));
 587   Args.push_back(B.getInt32(NumPatchBytes));
 588   Args.push_back(ActualCallee);
 589   Args.push_back(B.getInt32(CallArgs.size()));
 590   Args.push_back(B.getInt32(Flags));
 591   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
 592   Args.push_back(B.getInt32(TransitionArgs.size()));
 593   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
 594   Args.push_back(B.getInt32(DeoptArgs.size()));
 595   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
 596   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
 597
 598   return Args;
 599 }
 600
 601 template <typename T0, typename T1, typename T2, typename T3>
 602 static CallInst *CreateGCStatepointCallCommon(
 603     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
 604     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
 605     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
 606     const Twine &Name) {
 607   // Extract out the type of the callee.
 608   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
 609   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
 610          "actual callee must be a callable value");
 611
 612   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 613   // Fill in the one generic type'd argument (the function is also vararg)
 614   Type *ArgTypes[] = { FuncPtrType };
 615   Function *FnStatepoint =
 616     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
 617                               ArgTypes);
 618
 619   std::vector<Value *> Args =
 620       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
 621                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
 622   return createCallHelper(FnStatepoint, Args, Builder, Name);
 623 }
 624
 625 CallInst *IRBuilderBase::CreateGCStatepointCall(
 626     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
 627     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
 628     ArrayRef<Value *> GCArgs, const Twine &Name) {
 629   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
 630       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
 631       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
 632 }
 633
 634 CallInst *IRBuilderBase::CreateGCStatepointCall(
 635     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
 636     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
 637     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 638   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
 639       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
 640       DeoptArgs, GCArgs, Name);
 641 }
 642
 643 CallInst *IRBuilderBase::CreateGCStatepointCall(
 644     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
 645     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
 646     ArrayRef<Value *> GCArgs, const Twine &Name) {
 647   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
 648       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
 649       CallArgs, None, DeoptArgs, GCArgs, Name);
 650 }
 651
 652 template <typename T0, typename T1, typename T2, typename T3>
 653 static InvokeInst *CreateGCStatepointInvokeCommon(
 654     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
 655     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
 656     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
 657     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
 658   // Extract out the type of the callee.
 659   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
 660   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
 661          "actual callee must be a callable value");
 662
 663   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 664   // Fill in the one generic type'd argument (the function is also vararg)
 665   Function *FnStatepoint = Intrinsic::getDeclaration(
 666       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
 667
 668   std::vector<Value *> Args =
 669       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
 670                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
 671   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
 672                             Name);
 673 }
 674
 675 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 676     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 677     BasicBlock *NormalDest, BasicBlock *UnwindDest,
 678     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
 679     ArrayRef<Value *> GCArgs, const Twine &Name) {
 680   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
 681       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
 682       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
 683       DeoptArgs, GCArgs, Name);
 684 }
 685
 686 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 687     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 688     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
 689     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
 690     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 691   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
 692       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
 693       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
 694 }
 695
 696 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 697     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 698     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
 699     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 700   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
 701       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
 702       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
 703       Name);
 704 }
 705
 706 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
 707                                        Type *ResultType,
 708                                        const Twine &Name) {
 709  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
 710  Module *M = BB->getParent()->getParent();
 711  Type *Types[] = {ResultType};
 712  Value *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
 713
 714  Value *Args[] = {Statepoint};
 715  return createCallHelper(FnGCResult, Args, this, Name);
 716 }
 717
 718 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
 719                                          int BaseOffset,
 720                                          int DerivedOffset,
 721                                          Type *ResultType,
 722                                          const Twine &Name) {
 723  Module *M = BB->getParent()->getParent();
 724  Type *Types[] = {ResultType};
 725  Value *FnGCRelocate =
 726    Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
 727
 728  Value *Args[] = {Statepoint,
 729                   getInt32(BaseOffset),
 730                   getInt32(DerivedOffset)};
 731  return createCallHelper(FnGCRelocate, Args, this, Name);
 732 }
 733
 734 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
 735                                               Instruction *FMFSource,
 736                                               const Twine &Name) {
 737   Module *M = BB->getModule();
 738   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
 739   return createCallHelper(Fn, {V}, this, Name, FMFSource);
 740 }
 741
 742 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
 743                                                Value *RHS,
 744                                                Instruction *FMFSource,
 745                                                const Twine &Name) {
 746   Module *M = BB->getModule();
 747   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
 748   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
 749 }
 750
 751 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
 752                                          ArrayRef<Type *> Types,
 753                                          ArrayRef<Value *> Args,
 754                                          Instruction *FMFSource,
 755                                          const Twine &Name) {
 756   Module *M = BB->getModule();
 757   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
 758   return createCallHelper(Fn, Args, this, Name, FMFSource);
 759 }