lib/IR/IRBuilder.cpp

   1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the IRBuilder class, which is used as a convenient way
  10 // to create LLVM instructions with a consistent and simplified interface.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/IR/IRBuilder.h"
  15 #include "llvm/ADT/ArrayRef.h"
  16 #include "llvm/ADT/None.h"
  17 #include "llvm/IR/Constant.h"
  18 #include "llvm/IR/Constants.h"
  19 #include "llvm/IR/DerivedTypes.h"
  20 #include "llvm/IR/Function.h"
  21 #include "llvm/IR/GlobalValue.h"
  22 #include "llvm/IR/GlobalVariable.h"
  23 #include "llvm/IR/IntrinsicInst.h"
  24 #include "llvm/IR/Intrinsics.h"
  25 #include "llvm/IR/LLVMContext.h"
  26 #include "llvm/IR/Operator.h"
  27 #include "llvm/IR/Statepoint.h"
  28 #include "llvm/IR/Type.h"
  29 #include "llvm/IR/Value.h"
  30 #include "llvm/Support/Casting.h"
  31 #include "llvm/Support/MathExtras.h"
  32 #include <cassert>
  33 #include <cstdint>
  34 #include <vector>
  35
  36 using namespace llvm;
  37
  38 /// CreateGlobalString - Make a new global variable with an initializer that
  39 /// has array of i8 type filled in with the nul terminated string value
  40 /// specified.  If Name is specified, it is the name of the global variable
  41 /// created.
  42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
  43                                                   const Twine &Name,
  44                                                   unsigned AddressSpace) {
  45   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
  46   Module &M = *BB->getParent()->getParent();
  47   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
  48                                 GlobalValue::PrivateLinkage, StrConstant, Name,
  49                                 nullptr, GlobalVariable::NotThreadLocal,
  50                                 AddressSpace);
  51   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
  52   GV->setAlignment(Align::None());
  53   return GV;
  54 }
  55
  56 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
  57   assert(BB && BB->getParent() && "No current function!");
  58   return BB->getParent()->getReturnType();
  59 }
  60
  61 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
  62   auto *PT = cast<PointerType>(Ptr->getType());
  63   if (PT->getElementType()->isIntegerTy(8))
  64     return Ptr;
  65
  66   // Otherwise, we need to insert a bitcast.
  67   PT = getInt8PtrTy(PT->getAddressSpace());
  68   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
  69   BB->getInstList().insert(InsertPt, BCI);
  70   SetInstDebugLocation(BCI);
  71   return BCI;
  72 }
  73
  74 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
  75                                   IRBuilderBase *Builder,
  76                                   const Twine &Name = "",
  77                                   Instruction *FMFSource = nullptr) {
  78   CallInst *CI = CallInst::Create(Callee, Ops, Name);
  79   if (FMFSource)
  80     CI->copyFastMathFlags(FMFSource);
  81   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
  82   Builder->SetInstDebugLocation(CI);
  83   return CI;
  84 }
  85
  86 static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
  87                                       BasicBlock *UnwindDest,
  88                                       ArrayRef<Value *> Ops,
  89                                       IRBuilderBase *Builder,
  90                                       const Twine &Name = "") {
  91   InvokeInst *II =
  92       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
  93   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
  94                                                   II);
  95   Builder->SetInstDebugLocation(II);
  96   return II;
  97 }
  98
  99 CallInst *IRBuilderBase::
 100 CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
 101              bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
 102              MDNode *NoAliasTag) {
 103   Ptr = getCastedInt8PtrValue(Ptr);
 104   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
 105   Type *Tys[] = { Ptr->getType(), Size->getType() };
 106   Module *M = BB->getParent()->getParent();
 107   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
 108
 109   CallInst *CI = createCallHelper(TheFn, Ops, this);
 110
 111   if (Align > 0)
 112     cast<MemSetInst>(CI)->setDestAlignment(Align);
 113
 114   // Set the TBAA info if present.
 115   if (TBAATag)
 116     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 117
 118   if (ScopeTag)
 119     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 120
 121   if (NoAliasTag)
 122     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 123
 124   return CI;
 125 }
 126
 127 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
 128     Value *Ptr, Value *Val, Value *Size, unsigned Align, uint32_t ElementSize,
 129     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
 130   assert(Align >= ElementSize &&
 131          "Pointer alignment must be at least element size.");
 132
 133   Ptr = getCastedInt8PtrValue(Ptr);
 134   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
 135   Type *Tys[] = {Ptr->getType(), Size->getType()};
 136   Module *M = BB->getParent()->getParent();
 137   Function *TheFn = Intrinsic::getDeclaration(
 138       M, Intrinsic::memset_element_unordered_atomic, Tys);
 139
 140   CallInst *CI = createCallHelper(TheFn, Ops, this);
 141
 142   cast<AtomicMemSetInst>(CI)->setDestAlignment(Align);
 143
 144   // Set the TBAA info if present.
 145   if (TBAATag)
 146     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 147
 148   if (ScopeTag)
 149     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 150
 151   if (NoAliasTag)
 152     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 153
 154   return CI;
 155 }
 156
 157 CallInst *IRBuilderBase::
 158 CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
 159              Value *Size, bool isVolatile, MDNode *TBAATag,
 160              MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
 161   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
 162   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
 163   Dst = getCastedInt8PtrValue(Dst);
 164   Src = getCastedInt8PtrValue(Src);
 165
 166   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
 167   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
 168   Module *M = BB->getParent()->getParent();
 169   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
 170
 171   CallInst *CI = createCallHelper(TheFn, Ops, this);
 172
 173   auto* MCI = cast<MemCpyInst>(CI);
 174   if (DstAlign > 0)
 175     MCI->setDestAlignment(DstAlign);
 176   if (SrcAlign > 0)
 177     MCI->setSourceAlignment(SrcAlign);
 178
 179   // Set the TBAA info if present.
 180   if (TBAATag)
 181     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 182
 183   // Set the TBAA Struct info if present.
 184   if (TBAAStructTag)
 185     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 186
 187   if (ScopeTag)
 188     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 189
 190   if (NoAliasTag)
 191     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 192
 193   return CI;
 194 }
 195
 196 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
 197     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
 198     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
 199     MDNode *ScopeTag, MDNode *NoAliasTag) {
 200   assert(DstAlign >= ElementSize &&
 201          "Pointer alignment must be at least element size");
 202   assert(SrcAlign >= ElementSize &&
 203          "Pointer alignment must be at least element size");
 204   Dst = getCastedInt8PtrValue(Dst);
 205   Src = getCastedInt8PtrValue(Src);
 206
 207   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
 208   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 209   Module *M = BB->getParent()->getParent();
 210   Function *TheFn = Intrinsic::getDeclaration(
 211       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
 212
 213   CallInst *CI = createCallHelper(TheFn, Ops, this);
 214
 215   // Set the alignment of the pointer args.
 216   auto *AMCI = cast<AtomicMemCpyInst>(CI);
 217   AMCI->setDestAlignment(DstAlign);
 218   AMCI->setSourceAlignment(SrcAlign);
 219
 220   // Set the TBAA info if present.
 221   if (TBAATag)
 222     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 223
 224   // Set the TBAA Struct info if present.
 225   if (TBAAStructTag)
 226     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 227
 228   if (ScopeTag)
 229     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 230
 231   if (NoAliasTag)
 232     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 233
 234   return CI;
 235 }
 236
 237 CallInst *IRBuilderBase::
 238 CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
 239               Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
 240               MDNode *NoAliasTag) {
 241   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
 242   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
 243   Dst = getCastedInt8PtrValue(Dst);
 244   Src = getCastedInt8PtrValue(Src);
 245
 246   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
 247   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
 248   Module *M = BB->getParent()->getParent();
 249   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
 250
 251   CallInst *CI = createCallHelper(TheFn, Ops, this);
 252
 253   auto *MMI = cast<MemMoveInst>(CI);
 254   if (DstAlign > 0)
 255     MMI->setDestAlignment(DstAlign);
 256   if (SrcAlign > 0)
 257     MMI->setSourceAlignment(SrcAlign);
 258
 259   // Set the TBAA info if present.
 260   if (TBAATag)
 261     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 262
 263   if (ScopeTag)
 264     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 265
 266   if (NoAliasTag)
 267     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 268
 269   return CI;
 270 }
 271
 272 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
 273     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
 274     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
 275     MDNode *ScopeTag, MDNode *NoAliasTag) {
 276   assert(DstAlign >= ElementSize &&
 277          "Pointer alignment must be at least element size");
 278   assert(SrcAlign >= ElementSize &&
 279          "Pointer alignment must be at least element size");
 280   Dst = getCastedInt8PtrValue(Dst);
 281   Src = getCastedInt8PtrValue(Src);
 282
 283   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
 284   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 285   Module *M = BB->getParent()->getParent();
 286   Function *TheFn = Intrinsic::getDeclaration(
 287       M, Intrinsic::memmove_element_unordered_atomic, Tys);
 288
 289   CallInst *CI = createCallHelper(TheFn, Ops, this);
 290
 291   // Set the alignment of the pointer args.
 292   CI->addParamAttr(
 293       0, Attribute::getWithAlignment(CI->getContext(), Align(DstAlign)));
 294   CI->addParamAttr(
 295       1, Attribute::getWithAlignment(CI->getContext(), Align(SrcAlign)));
 296
 297   // Set the TBAA info if present.
 298   if (TBAATag)
 299     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 300
 301   // Set the TBAA Struct info if present.
 302   if (TBAAStructTag)
 303     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 304
 305   if (ScopeTag)
 306     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 307
 308   if (NoAliasTag)
 309     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 310
 311   return CI;
 312 }
 313
 314 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
 315                                     Value *Src) {
 316   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 317   Value *Ops[] = {Src};
 318   Type *Tys[] = { Src->getType() };
 319   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
 320   return createCallHelper(Decl, Ops, Builder);
 321 }
 322
 323 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
 324   Module *M = GetInsertBlock()->getParent()->getParent();
 325   Value *Ops[] = {Acc, Src};
 326   Type *Tys[] = {Acc->getType(), Src->getType()};
 327   auto Decl = Intrinsic::getDeclaration(
 328       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
 329   return createCallHelper(Decl, Ops, this);
 330 }
 331
 332 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
 333   Module *M = GetInsertBlock()->getParent()->getParent();
 334   Value *Ops[] = {Acc, Src};
 335   Type *Tys[] = {Acc->getType(), Src->getType()};
 336   auto Decl = Intrinsic::getDeclaration(
 337       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
 338   return createCallHelper(Decl, Ops, this);
 339 }
 340
 341 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
 342   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
 343                                Src);
 344 }
 345
 346 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
 347   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
 348                                Src);
 349 }
 350
 351 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
 352   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
 353                                Src);
 354 }
 355
 356 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
 357   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
 358                                Src);
 359 }
 360
 361 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
 362   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
 363                                Src);
 364 }
 365
 366 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
 367   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
 368                      : Intrinsic::experimental_vector_reduce_umax;
 369   return getReductionIntrinsic(this, ID, Src);
 370 }
 371
 372 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
 373   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
 374                      : Intrinsic::experimental_vector_reduce_umin;
 375   return getReductionIntrinsic(this, ID, Src);
 376 }
 377
 378 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
 379   auto Rdx = getReductionIntrinsic(
 380       this, Intrinsic::experimental_vector_reduce_fmax, Src);
 381   if (NoNaN) {
 382     FastMathFlags FMF;
 383     FMF.setNoNaNs();
 384     Rdx->setFastMathFlags(FMF);
 385   }
 386   return Rdx;
 387 }
 388
 389 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
 390   auto Rdx = getReductionIntrinsic(
 391       this, Intrinsic::experimental_vector_reduce_fmin, Src);
 392   if (NoNaN) {
 393     FastMathFlags FMF;
 394     FMF.setNoNaNs();
 395     Rdx->setFastMathFlags(FMF);
 396   }
 397   return Rdx;
 398 }
 399
 400 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
 401   assert(isa<PointerType>(Ptr->getType()) &&
 402          "lifetime.start only applies to pointers.");
 403   Ptr = getCastedInt8PtrValue(Ptr);
 404   if (!Size)
 405     Size = getInt64(-1);
 406   else
 407     assert(Size->getType() == getInt64Ty() &&
 408            "lifetime.start requires the size to be an i64");
 409   Value *Ops[] = { Size, Ptr };
 410   Module *M = BB->getParent()->getParent();
 411   Function *TheFn =
 412       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
 413   return createCallHelper(TheFn, Ops, this);
 414 }
 415
 416 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
 417   assert(isa<PointerType>(Ptr->getType()) &&
 418          "lifetime.end only applies to pointers.");
 419   Ptr = getCastedInt8PtrValue(Ptr);
 420   if (!Size)
 421     Size = getInt64(-1);
 422   else
 423     assert(Size->getType() == getInt64Ty() &&
 424            "lifetime.end requires the size to be an i64");
 425   Value *Ops[] = { Size, Ptr };
 426   Module *M = BB->getParent()->getParent();
 427   Function *TheFn =
 428       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
 429   return createCallHelper(TheFn, Ops, this);
 430 }
 431
 432 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
 433
 434   assert(isa<PointerType>(Ptr->getType()) &&
 435          "invariant.start only applies to pointers.");
 436   Ptr = getCastedInt8PtrValue(Ptr);
 437   if (!Size)
 438     Size = getInt64(-1);
 439   else
 440     assert(Size->getType() == getInt64Ty() &&
 441            "invariant.start requires the size to be an i64");
 442
 443   Value *Ops[] = {Size, Ptr};
 444   // Fill in the single overloaded type: memory object type.
 445   Type *ObjectPtr[1] = {Ptr->getType()};
 446   Module *M = BB->getParent()->getParent();
 447   Function *TheFn =
 448       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
 449   return createCallHelper(TheFn, Ops, this);
 450 }
 451
 452 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
 453   assert(Cond->getType() == getInt1Ty() &&
 454          "an assumption condition must be of type i1");
 455
 456   Value *Ops[] = { Cond };
 457   Module *M = BB->getParent()->getParent();
 458   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
 459   return createCallHelper(FnAssume, Ops, this);
 460 }
 461
 462 /// Create a call to a Masked Load intrinsic.
 463 /// \p Ptr      - base pointer for the load
 464 /// \p Align    - alignment of the source location
 465 /// \p Mask     - vector of booleans which indicates what vector lanes should
 466 ///               be accessed in memory
 467 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
 468 ///               of the result
 469 /// \p Name     - name of the result variable
 470 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
 471                                           Value *Mask, Value *PassThru,
 472                                           const Twine &Name) {
 473   auto *PtrTy = cast<PointerType>(Ptr->getType());
 474   Type *DataTy = PtrTy->getElementType();
 475   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
 476   assert(Mask && "Mask should not be all-ones (null)");
 477   if (!PassThru)
 478     PassThru = UndefValue::get(DataTy);
 479   Type *OverloadedTypes[] = { DataTy, PtrTy };
 480   Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
 481   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
 482                                OverloadedTypes, Name);
 483 }
 484
 485 /// Create a call to a Masked Store intrinsic.
 486 /// \p Val   - data to be stored,
 487 /// \p Ptr   - base pointer for the store
 488 /// \p Align - alignment of the destination location
 489 /// \p Mask  - vector of booleans which indicates what vector lanes should
 490 ///            be accessed in memory
 491 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
 492                                            unsigned Align, Value *Mask) {
 493   auto *PtrTy = cast<PointerType>(Ptr->getType());
 494   Type *DataTy = PtrTy->getElementType();
 495   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
 496   assert(Mask && "Mask should not be all-ones (null)");
 497   Type *OverloadedTypes[] = { DataTy, PtrTy };
 498   Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
 499   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
 500 }
 501
 502 /// Create a call to a Masked intrinsic, with given intrinsic Id,
 503 /// an array of operands - Ops, and an array of overloaded types -
 504 /// OverloadedTypes.
 505 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
 506                                                ArrayRef<Value *> Ops,
 507                                                ArrayRef<Type *> OverloadedTypes,
 508                                                const Twine &Name) {
 509   Module *M = BB->getParent()->getParent();
 510   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
 511   return createCallHelper(TheFn, Ops, this, Name);
 512 }
 513
 514 /// Create a call to a Masked Gather intrinsic.
 515 /// \p Ptrs     - vector of pointers for loading
 516 /// \p Align    - alignment for one element
 517 /// \p Mask     - vector of booleans which indicates what vector lanes should
 518 ///               be accessed in memory
 519 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
 520 ///               of the result
 521 /// \p Name     - name of the result variable
 522 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
 523                                             Value *Mask,  Value *PassThru,
 524                                             const Twine& Name) {
 525   auto PtrsTy = cast<VectorType>(Ptrs->getType());
 526   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
 527   unsigned NumElts = PtrsTy->getVectorNumElements();
 528   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
 529
 530   if (!Mask)
 531     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
 532                                      NumElts));
 533
 534   if (!PassThru)
 535     PassThru = UndefValue::get(DataTy);
 536
 537   Type *OverloadedTypes[] = {DataTy, PtrsTy};
 538   Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
 539
 540   // We specify only one type when we create this intrinsic. Types of other
 541   // arguments are derived from this type.
 542   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
 543                                Name);
 544 }
 545
 546 /// Create a call to a Masked Scatter intrinsic.
 547 /// \p Data  - data to be stored,
 548 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
 549 ///            stored
 550 /// \p Align - alignment for one element
 551 /// \p Mask  - vector of booleans which indicates what vector lanes should
 552 ///            be accessed in memory
 553 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
 554                                              unsigned Align, Value *Mask) {
 555   auto PtrsTy = cast<VectorType>(Ptrs->getType());
 556   auto DataTy = cast<VectorType>(Data->getType());
 557   unsigned NumElts = PtrsTy->getVectorNumElements();
 558
 559 #ifndef NDEBUG
 560   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
 561   assert(NumElts == DataTy->getVectorNumElements() &&
 562          PtrTy->getElementType() == DataTy->getElementType() &&
 563          "Incompatible pointer and data types");
 564 #endif
 565
 566   if (!Mask)
 567     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
 568                                      NumElts));
 569
 570   Type *OverloadedTypes[] = {DataTy, PtrsTy};
 571   Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
 572
 573   // We specify only one type when we create this intrinsic. Types of other
 574   // arguments are derived from this type.
 575   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
 576 }
 577
 578 template <typename T0, typename T1, typename T2, typename T3>
 579 static std::vector<Value *>
 580 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
 581                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
 582                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
 583                   ArrayRef<T3> GCArgs) {
 584   std::vector<Value *> Args;
 585   Args.push_back(B.getInt64(ID));
 586   Args.push_back(B.getInt32(NumPatchBytes));
 587   Args.push_back(ActualCallee);
 588   Args.push_back(B.getInt32(CallArgs.size()));
 589   Args.push_back(B.getInt32(Flags));
 590   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
 591   Args.push_back(B.getInt32(TransitionArgs.size()));
 592   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
 593   Args.push_back(B.getInt32(DeoptArgs.size()));
 594   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
 595   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
 596
 597   return Args;
 598 }
 599
 600 template <typename T0, typename T1, typename T2, typename T3>
 601 static CallInst *CreateGCStatepointCallCommon(
 602     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
 603     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
 604     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
 605     const Twine &Name) {
 606   // Extract out the type of the callee.
 607   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
 608   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
 609          "actual callee must be a callable value");
 610
 611   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 612   // Fill in the one generic type'd argument (the function is also vararg)
 613   Type *ArgTypes[] = { FuncPtrType };
 614   Function *FnStatepoint =
 615     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
 616                               ArgTypes);
 617
 618   std::vector<Value *> Args =
 619       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
 620                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
 621   return createCallHelper(FnStatepoint, Args, Builder, Name);
 622 }
 623
 624 CallInst *IRBuilderBase::CreateGCStatepointCall(
 625     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
 626     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
 627     ArrayRef<Value *> GCArgs, const Twine &Name) {
 628   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
 629       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
 630       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
 631 }
 632
 633 CallInst *IRBuilderBase::CreateGCStatepointCall(
 634     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
 635     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
 636     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 637   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
 638       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
 639       DeoptArgs, GCArgs, Name);
 640 }
 641
 642 CallInst *IRBuilderBase::CreateGCStatepointCall(
 643     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
 644     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
 645     ArrayRef<Value *> GCArgs, const Twine &Name) {
 646   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
 647       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
 648       CallArgs, None, DeoptArgs, GCArgs, Name);
 649 }
 650
 651 template <typename T0, typename T1, typename T2, typename T3>
 652 static InvokeInst *CreateGCStatepointInvokeCommon(
 653     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
 654     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
 655     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
 656     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
 657   // Extract out the type of the callee.
 658   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
 659   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
 660          "actual callee must be a callable value");
 661
 662   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 663   // Fill in the one generic type'd argument (the function is also vararg)
 664   Function *FnStatepoint = Intrinsic::getDeclaration(
 665       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
 666
 667   std::vector<Value *> Args =
 668       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
 669                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
 670   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
 671                             Name);
 672 }
 673
 674 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 675     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 676     BasicBlock *NormalDest, BasicBlock *UnwindDest,
 677     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
 678     ArrayRef<Value *> GCArgs, const Twine &Name) {
 679   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
 680       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
 681       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
 682       DeoptArgs, GCArgs, Name);
 683 }
 684
 685 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 686     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 687     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
 688     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
 689     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 690   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
 691       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
 692       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
 693 }
 694
 695 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 696     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 697     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
 698     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 699   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
 700       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
 701       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
 702       Name);
 703 }
 704
 705 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
 706                                        Type *ResultType,
 707                                        const Twine &Name) {
 708  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
 709  Module *M = BB->getParent()->getParent();
 710  Type *Types[] = {ResultType};
 711  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
 712
 713  Value *Args[] = {Statepoint};
 714  return createCallHelper(FnGCResult, Args, this, Name);
 715 }
 716
 717 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
 718                                          int BaseOffset,
 719                                          int DerivedOffset,
 720                                          Type *ResultType,
 721                                          const Twine &Name) {
 722  Module *M = BB->getParent()->getParent();
 723  Type *Types[] = {ResultType};
 724  Function *FnGCRelocate =
 725      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
 726
 727  Value *Args[] = {Statepoint,
 728                   getInt32(BaseOffset),
 729                   getInt32(DerivedOffset)};
 730  return createCallHelper(FnGCRelocate, Args, this, Name);
 731 }
 732
 733 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
 734                                               Instruction *FMFSource,
 735                                               const Twine &Name) {
 736   Module *M = BB->getModule();
 737   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
 738   return createCallHelper(Fn, {V}, this, Name, FMFSource);
 739 }
 740
 741 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
 742                                                Value *RHS,
 743                                                Instruction *FMFSource,
 744                                                const Twine &Name) {
 745   Module *M = BB->getModule();
 746   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
 747   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
 748 }
 749
 750 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
 751                                          ArrayRef<Type *> Types,
 752                                          ArrayRef<Value *> Args,
 753                                          Instruction *FMFSource,
 754                                          const Twine &Name) {
 755   Module *M = BB->getModule();
 756   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
 757   return createCallHelper(Fn, Args, this, Name, FMFSource);
 758 }