lib/IR/IRBuilder.cpp

   1 //===- IRBuilder.cpp - Builder for LLVM Instrs ----------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the IRBuilder class, which is used as a convenient way
  10 // to create LLVM instructions with a consistent and simplified interface.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/IR/IRBuilder.h"
  15 #include "llvm/ADT/ArrayRef.h"
  16 #include "llvm/ADT/None.h"
  17 #include "llvm/IR/Constant.h"
  18 #include "llvm/IR/Constants.h"
  19 #include "llvm/IR/DerivedTypes.h"
  20 #include "llvm/IR/Function.h"
  21 #include "llvm/IR/GlobalValue.h"
  22 #include "llvm/IR/GlobalVariable.h"
  23 #include "llvm/IR/IntrinsicInst.h"
  24 #include "llvm/IR/Intrinsics.h"
  25 #include "llvm/IR/LLVMContext.h"
  26 #include "llvm/IR/Operator.h"
  27 #include "llvm/IR/Statepoint.h"
  28 #include "llvm/IR/Type.h"
  29 #include "llvm/IR/Value.h"
  30 #include "llvm/Support/Casting.h"
  31 #include "llvm/Support/MathExtras.h"
  32 #include <cassert>
  33 #include <cstdint>
  34 #include <vector>
  35
  36 using namespace llvm;
  37
  38 /// CreateGlobalString - Make a new global variable with an initializer that
  39 /// has array of i8 type filled in with the nul terminated string value
  40 /// specified.  If Name is specified, it is the name of the global variable
  41 /// created.
  42 GlobalVariable *IRBuilderBase::CreateGlobalString(StringRef Str,
  43                                                   const Twine &Name,
  44                                                   unsigned AddressSpace) {
  45   Constant *StrConstant = ConstantDataArray::getString(Context, Str);
  46   Module &M = *BB->getParent()->getParent();
  47   auto *GV = new GlobalVariable(M, StrConstant->getType(), true,
  48                                 GlobalValue::PrivateLinkage, StrConstant, Name,
  49                                 nullptr, GlobalVariable::NotThreadLocal,
  50                                 AddressSpace);
  51   GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
  52   GV->setAlignment(1);
  53   return GV;
  54 }
  55
  56 Type *IRBuilderBase::getCurrentFunctionReturnType() const {
  57   assert(BB && BB->getParent() && "No current function!");
  58   return BB->getParent()->getReturnType();
  59 }
  60
  61 Value *IRBuilderBase::getCastedInt8PtrValue(Value *Ptr) {
  62   auto *PT = cast<PointerType>(Ptr->getType());
  63   if (PT->getElementType()->isIntegerTy(8))
  64     return Ptr;
  65
  66   // Otherwise, we need to insert a bitcast.
  67   PT = getInt8PtrTy(PT->getAddressSpace());
  68   BitCastInst *BCI = new BitCastInst(Ptr, PT, "");
  69   BB->getInstList().insert(InsertPt, BCI);
  70   SetInstDebugLocation(BCI);
  71   return BCI;
  72 }
  73
  74 static CallInst *createCallHelper(Function *Callee, ArrayRef<Value *> Ops,
  75                                   IRBuilderBase *Builder,
  76                                   const Twine &Name = "",
  77                                   Instruction *FMFSource = nullptr) {
  78   CallInst *CI = CallInst::Create(Callee, Ops, Name);
  79   if (FMFSource)
  80     CI->copyFastMathFlags(FMFSource);
  81   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),CI);
  82   Builder->SetInstDebugLocation(CI);
  83   return CI;
  84 }
  85
  86 static InvokeInst *createInvokeHelper(Function *Invokee, BasicBlock *NormalDest,
  87                                       BasicBlock *UnwindDest,
  88                                       ArrayRef<Value *> Ops,
  89                                       IRBuilderBase *Builder,
  90                                       const Twine &Name = "") {
  91   InvokeInst *II =
  92       InvokeInst::Create(Invokee, NormalDest, UnwindDest, Ops, Name);
  93   Builder->GetInsertBlock()->getInstList().insert(Builder->GetInsertPoint(),
  94                                                   II);
  95   Builder->SetInstDebugLocation(II);
  96   return II;
  97 }
  98
  99 CallInst *IRBuilderBase::
 100 CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
 101              bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
 102              MDNode *NoAliasTag) {
 103   Ptr = getCastedInt8PtrValue(Ptr);
 104   Value *Ops[] = {Ptr, Val, Size, getInt1(isVolatile)};
 105   Type *Tys[] = { Ptr->getType(), Size->getType() };
 106   Module *M = BB->getParent()->getParent();
 107   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memset, Tys);
 108
 109   CallInst *CI = createCallHelper(TheFn, Ops, this);
 110
 111   if (Align > 0)
 112     cast<MemSetInst>(CI)->setDestAlignment(Align);
 113
 114   // Set the TBAA info if present.
 115   if (TBAATag)
 116     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 117
 118   if (ScopeTag)
 119     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 120
 121   if (NoAliasTag)
 122     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 123
 124   return CI;
 125 }
 126
 127 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemSet(
 128     Value *Ptr, Value *Val, Value *Size, unsigned Align, uint32_t ElementSize,
 129     MDNode *TBAATag, MDNode *ScopeTag, MDNode *NoAliasTag) {
 130   assert(Align >= ElementSize &&
 131          "Pointer alignment must be at least element size.");
 132
 133   Ptr = getCastedInt8PtrValue(Ptr);
 134   Value *Ops[] = {Ptr, Val, Size, getInt32(ElementSize)};
 135   Type *Tys[] = {Ptr->getType(), Size->getType()};
 136   Module *M = BB->getParent()->getParent();
 137   Function *TheFn = Intrinsic::getDeclaration(
 138       M, Intrinsic::memset_element_unordered_atomic, Tys);
 139
 140   CallInst *CI = createCallHelper(TheFn, Ops, this);
 141
 142   cast<AtomicMemSetInst>(CI)->setDestAlignment(Align);
 143
 144   // Set the TBAA info if present.
 145   if (TBAATag)
 146     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 147
 148   if (ScopeTag)
 149     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 150
 151   if (NoAliasTag)
 152     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 153
 154   return CI;
 155 }
 156
 157 CallInst *IRBuilderBase::
 158 CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
 159              Value *Size, bool isVolatile, MDNode *TBAATag,
 160              MDNode *TBAAStructTag, MDNode *ScopeTag, MDNode *NoAliasTag) {
 161   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
 162   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
 163   Dst = getCastedInt8PtrValue(Dst);
 164   Src = getCastedInt8PtrValue(Src);
 165
 166   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
 167   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
 168   Module *M = BB->getParent()->getParent();
 169   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memcpy, Tys);
 170
 171   CallInst *CI = createCallHelper(TheFn, Ops, this);
 172
 173   auto* MCI = cast<MemCpyInst>(CI);
 174   if (DstAlign > 0)
 175     MCI->setDestAlignment(DstAlign);
 176   if (SrcAlign > 0)
 177     MCI->setSourceAlignment(SrcAlign);
 178
 179   // Set the TBAA info if present.
 180   if (TBAATag)
 181     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 182
 183   // Set the TBAA Struct info if present.
 184   if (TBAAStructTag)
 185     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 186
 187   if (ScopeTag)
 188     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 189
 190   if (NoAliasTag)
 191     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 192
 193   return CI;
 194 }
 195
 196 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemCpy(
 197     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
 198     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
 199     MDNode *ScopeTag, MDNode *NoAliasTag) {
 200   assert(DstAlign >= ElementSize &&
 201          "Pointer alignment must be at least element size");
 202   assert(SrcAlign >= ElementSize &&
 203          "Pointer alignment must be at least element size");
 204   Dst = getCastedInt8PtrValue(Dst);
 205   Src = getCastedInt8PtrValue(Src);
 206
 207   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
 208   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 209   Module *M = BB->getParent()->getParent();
 210   Function *TheFn = Intrinsic::getDeclaration(
 211       M, Intrinsic::memcpy_element_unordered_atomic, Tys);
 212
 213   CallInst *CI = createCallHelper(TheFn, Ops, this);
 214
 215   // Set the alignment of the pointer args.
 216   auto *AMCI = cast<AtomicMemCpyInst>(CI);
 217   AMCI->setDestAlignment(DstAlign);
 218   AMCI->setSourceAlignment(SrcAlign);
 219
 220   // Set the TBAA info if present.
 221   if (TBAATag)
 222     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 223
 224   // Set the TBAA Struct info if present.
 225   if (TBAAStructTag)
 226     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 227
 228   if (ScopeTag)
 229     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 230
 231   if (NoAliasTag)
 232     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 233
 234   return CI;
 235 }
 236
 237 CallInst *IRBuilderBase::
 238 CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign,
 239               Value *Size, bool isVolatile, MDNode *TBAATag, MDNode *ScopeTag,
 240               MDNode *NoAliasTag) {
 241   assert((DstAlign == 0 || isPowerOf2_32(DstAlign)) && "Must be 0 or a power of 2");
 242   assert((SrcAlign == 0 || isPowerOf2_32(SrcAlign)) && "Must be 0 or a power of 2");
 243   Dst = getCastedInt8PtrValue(Dst);
 244   Src = getCastedInt8PtrValue(Src);
 245
 246   Value *Ops[] = {Dst, Src, Size, getInt1(isVolatile)};
 247   Type *Tys[] = { Dst->getType(), Src->getType(), Size->getType() };
 248   Module *M = BB->getParent()->getParent();
 249   Function *TheFn = Intrinsic::getDeclaration(M, Intrinsic::memmove, Tys);
 250
 251   CallInst *CI = createCallHelper(TheFn, Ops, this);
 252
 253   auto *MMI = cast<MemMoveInst>(CI);
 254   if (DstAlign > 0)
 255     MMI->setDestAlignment(DstAlign);
 256   if (SrcAlign > 0)
 257     MMI->setSourceAlignment(SrcAlign);
 258
 259   // Set the TBAA info if present.
 260   if (TBAATag)
 261     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 262
 263   if (ScopeTag)
 264     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 265
 266   if (NoAliasTag)
 267     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 268
 269   return CI;
 270 }
 271
 272 CallInst *IRBuilderBase::CreateElementUnorderedAtomicMemMove(
 273     Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size,
 274     uint32_t ElementSize, MDNode *TBAATag, MDNode *TBAAStructTag,
 275     MDNode *ScopeTag, MDNode *NoAliasTag) {
 276   assert(DstAlign >= ElementSize &&
 277          "Pointer alignment must be at least element size");
 278   assert(SrcAlign >= ElementSize &&
 279          "Pointer alignment must be at least element size");
 280   Dst = getCastedInt8PtrValue(Dst);
 281   Src = getCastedInt8PtrValue(Src);
 282
 283   Value *Ops[] = {Dst, Src, Size, getInt32(ElementSize)};
 284   Type *Tys[] = {Dst->getType(), Src->getType(), Size->getType()};
 285   Module *M = BB->getParent()->getParent();
 286   Function *TheFn = Intrinsic::getDeclaration(
 287       M, Intrinsic::memmove_element_unordered_atomic, Tys);
 288
 289   CallInst *CI = createCallHelper(TheFn, Ops, this);
 290
 291   // Set the alignment of the pointer args.
 292   CI->addParamAttr(0, Attribute::getWithAlignment(CI->getContext(), DstAlign));
 293   CI->addParamAttr(1, Attribute::getWithAlignment(CI->getContext(), SrcAlign));
 294
 295   // Set the TBAA info if present.
 296   if (TBAATag)
 297     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
 298
 299   // Set the TBAA Struct info if present.
 300   if (TBAAStructTag)
 301     CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
 302
 303   if (ScopeTag)
 304     CI->setMetadata(LLVMContext::MD_alias_scope, ScopeTag);
 305
 306   if (NoAliasTag)
 307     CI->setMetadata(LLVMContext::MD_noalias, NoAliasTag);
 308
 309   return CI;
 310 }
 311
 312 static CallInst *getReductionIntrinsic(IRBuilderBase *Builder, Intrinsic::ID ID,
 313                                     Value *Src) {
 314   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 315   Value *Ops[] = {Src};
 316   Type *Tys[] = { Src->getType() };
 317   auto Decl = Intrinsic::getDeclaration(M, ID, Tys);
 318   return createCallHelper(Decl, Ops, Builder);
 319 }
 320
 321 CallInst *IRBuilderBase::CreateFAddReduce(Value *Acc, Value *Src) {
 322   Module *M = GetInsertBlock()->getParent()->getParent();
 323   Value *Ops[] = {Acc, Src};
 324   Type *Tys[] = {Acc->getType(), Src->getType()};
 325   auto Decl = Intrinsic::getDeclaration(
 326       M, Intrinsic::experimental_vector_reduce_v2_fadd, Tys);
 327   return createCallHelper(Decl, Ops, this);
 328 }
 329
 330 CallInst *IRBuilderBase::CreateFMulReduce(Value *Acc, Value *Src) {
 331   Module *M = GetInsertBlock()->getParent()->getParent();
 332   Value *Ops[] = {Acc, Src};
 333   Type *Tys[] = {Acc->getType(), Src->getType()};
 334   auto Decl = Intrinsic::getDeclaration(
 335       M, Intrinsic::experimental_vector_reduce_v2_fmul, Tys);
 336   return createCallHelper(Decl, Ops, this);
 337 }
 338
 339 CallInst *IRBuilderBase::CreateAddReduce(Value *Src) {
 340   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_add,
 341                                Src);
 342 }
 343
 344 CallInst *IRBuilderBase::CreateMulReduce(Value *Src) {
 345   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_mul,
 346                                Src);
 347 }
 348
 349 CallInst *IRBuilderBase::CreateAndReduce(Value *Src) {
 350   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_and,
 351                                Src);
 352 }
 353
 354 CallInst *IRBuilderBase::CreateOrReduce(Value *Src) {
 355   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_or,
 356                                Src);
 357 }
 358
 359 CallInst *IRBuilderBase::CreateXorReduce(Value *Src) {
 360   return getReductionIntrinsic(this, Intrinsic::experimental_vector_reduce_xor,
 361                                Src);
 362 }
 363
 364 CallInst *IRBuilderBase::CreateIntMaxReduce(Value *Src, bool IsSigned) {
 365   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smax
 366                      : Intrinsic::experimental_vector_reduce_umax;
 367   return getReductionIntrinsic(this, ID, Src);
 368 }
 369
 370 CallInst *IRBuilderBase::CreateIntMinReduce(Value *Src, bool IsSigned) {
 371   auto ID = IsSigned ? Intrinsic::experimental_vector_reduce_smin
 372                      : Intrinsic::experimental_vector_reduce_umin;
 373   return getReductionIntrinsic(this, ID, Src);
 374 }
 375
 376 CallInst *IRBuilderBase::CreateFPMaxReduce(Value *Src, bool NoNaN) {
 377   auto Rdx = getReductionIntrinsic(
 378       this, Intrinsic::experimental_vector_reduce_fmax, Src);
 379   if (NoNaN) {
 380     FastMathFlags FMF;
 381     FMF.setNoNaNs();
 382     Rdx->setFastMathFlags(FMF);
 383   }
 384   return Rdx;
 385 }
 386
 387 CallInst *IRBuilderBase::CreateFPMinReduce(Value *Src, bool NoNaN) {
 388   auto Rdx = getReductionIntrinsic(
 389       this, Intrinsic::experimental_vector_reduce_fmin, Src);
 390   if (NoNaN) {
 391     FastMathFlags FMF;
 392     FMF.setNoNaNs();
 393     Rdx->setFastMathFlags(FMF);
 394   }
 395   return Rdx;
 396 }
 397
 398 CallInst *IRBuilderBase::CreateLifetimeStart(Value *Ptr, ConstantInt *Size) {
 399   assert(isa<PointerType>(Ptr->getType()) &&
 400          "lifetime.start only applies to pointers.");
 401   Ptr = getCastedInt8PtrValue(Ptr);
 402   if (!Size)
 403     Size = getInt64(-1);
 404   else
 405     assert(Size->getType() == getInt64Ty() &&
 406            "lifetime.start requires the size to be an i64");
 407   Value *Ops[] = { Size, Ptr };
 408   Module *M = BB->getParent()->getParent();
 409   Function *TheFn =
 410       Intrinsic::getDeclaration(M, Intrinsic::lifetime_start, {Ptr->getType()});
 411   return createCallHelper(TheFn, Ops, this);
 412 }
 413
 414 CallInst *IRBuilderBase::CreateLifetimeEnd(Value *Ptr, ConstantInt *Size) {
 415   assert(isa<PointerType>(Ptr->getType()) &&
 416          "lifetime.end only applies to pointers.");
 417   Ptr = getCastedInt8PtrValue(Ptr);
 418   if (!Size)
 419     Size = getInt64(-1);
 420   else
 421     assert(Size->getType() == getInt64Ty() &&
 422            "lifetime.end requires the size to be an i64");
 423   Value *Ops[] = { Size, Ptr };
 424   Module *M = BB->getParent()->getParent();
 425   Function *TheFn =
 426       Intrinsic::getDeclaration(M, Intrinsic::lifetime_end, {Ptr->getType()});
 427   return createCallHelper(TheFn, Ops, this);
 428 }
 429
 430 CallInst *IRBuilderBase::CreateInvariantStart(Value *Ptr, ConstantInt *Size) {
 431
 432   assert(isa<PointerType>(Ptr->getType()) &&
 433          "invariant.start only applies to pointers.");
 434   Ptr = getCastedInt8PtrValue(Ptr);
 435   if (!Size)
 436     Size = getInt64(-1);
 437   else
 438     assert(Size->getType() == getInt64Ty() &&
 439            "invariant.start requires the size to be an i64");
 440
 441   Value *Ops[] = {Size, Ptr};
 442   // Fill in the single overloaded type: memory object type.
 443   Type *ObjectPtr[1] = {Ptr->getType()};
 444   Module *M = BB->getParent()->getParent();
 445   Function *TheFn =
 446       Intrinsic::getDeclaration(M, Intrinsic::invariant_start, ObjectPtr);
 447   return createCallHelper(TheFn, Ops, this);
 448 }
 449
 450 CallInst *IRBuilderBase::CreateAssumption(Value *Cond) {
 451   assert(Cond->getType() == getInt1Ty() &&
 452          "an assumption condition must be of type i1");
 453
 454   Value *Ops[] = { Cond };
 455   Module *M = BB->getParent()->getParent();
 456   Function *FnAssume = Intrinsic::getDeclaration(M, Intrinsic::assume);
 457   return createCallHelper(FnAssume, Ops, this);
 458 }
 459
 460 /// Create a call to a Masked Load intrinsic.
 461 /// \p Ptr      - base pointer for the load
 462 /// \p Align    - alignment of the source location
 463 /// \p Mask     - vector of booleans which indicates what vector lanes should
 464 ///               be accessed in memory
 465 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
 466 ///               of the result
 467 /// \p Name     - name of the result variable
 468 CallInst *IRBuilderBase::CreateMaskedLoad(Value *Ptr, unsigned Align,
 469                                           Value *Mask, Value *PassThru,
 470                                           const Twine &Name) {
 471   auto *PtrTy = cast<PointerType>(Ptr->getType());
 472   Type *DataTy = PtrTy->getElementType();
 473   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
 474   assert(Mask && "Mask should not be all-ones (null)");
 475   if (!PassThru)
 476     PassThru = UndefValue::get(DataTy);
 477   Type *OverloadedTypes[] = { DataTy, PtrTy };
 478   Value *Ops[] = { Ptr, getInt32(Align), Mask,  PassThru};
 479   return CreateMaskedIntrinsic(Intrinsic::masked_load, Ops,
 480                                OverloadedTypes, Name);
 481 }
 482
 483 /// Create a call to a Masked Store intrinsic.
 484 /// \p Val   - data to be stored,
 485 /// \p Ptr   - base pointer for the store
 486 /// \p Align - alignment of the destination location
 487 /// \p Mask  - vector of booleans which indicates what vector lanes should
 488 ///            be accessed in memory
 489 CallInst *IRBuilderBase::CreateMaskedStore(Value *Val, Value *Ptr,
 490                                            unsigned Align, Value *Mask) {
 491   auto *PtrTy = cast<PointerType>(Ptr->getType());
 492   Type *DataTy = PtrTy->getElementType();
 493   assert(DataTy->isVectorTy() && "Ptr should point to a vector");
 494   assert(Mask && "Mask should not be all-ones (null)");
 495   Type *OverloadedTypes[] = { DataTy, PtrTy };
 496   Value *Ops[] = { Val, Ptr, getInt32(Align), Mask };
 497   return CreateMaskedIntrinsic(Intrinsic::masked_store, Ops, OverloadedTypes);
 498 }
 499
 500 /// Create a call to a Masked intrinsic, with given intrinsic Id,
 501 /// an array of operands - Ops, and an array of overloaded types -
 502 /// OverloadedTypes.
 503 CallInst *IRBuilderBase::CreateMaskedIntrinsic(Intrinsic::ID Id,
 504                                                ArrayRef<Value *> Ops,
 505                                                ArrayRef<Type *> OverloadedTypes,
 506                                                const Twine &Name) {
 507   Module *M = BB->getParent()->getParent();
 508   Function *TheFn = Intrinsic::getDeclaration(M, Id, OverloadedTypes);
 509   return createCallHelper(TheFn, Ops, this, Name);
 510 }
 511
 512 /// Create a call to a Masked Gather intrinsic.
 513 /// \p Ptrs     - vector of pointers for loading
 514 /// \p Align    - alignment for one element
 515 /// \p Mask     - vector of booleans which indicates what vector lanes should
 516 ///               be accessed in memory
 517 /// \p PassThru - pass-through value that is used to fill the masked-off lanes
 518 ///               of the result
 519 /// \p Name     - name of the result variable
 520 CallInst *IRBuilderBase::CreateMaskedGather(Value *Ptrs, unsigned Align,
 521                                             Value *Mask,  Value *PassThru,
 522                                             const Twine& Name) {
 523   auto PtrsTy = cast<VectorType>(Ptrs->getType());
 524   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
 525   unsigned NumElts = PtrsTy->getVectorNumElements();
 526   Type *DataTy = VectorType::get(PtrTy->getElementType(), NumElts);
 527
 528   if (!Mask)
 529     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
 530                                      NumElts));
 531
 532   if (!PassThru)
 533     PassThru = UndefValue::get(DataTy);
 534
 535   Type *OverloadedTypes[] = {DataTy, PtrsTy};
 536   Value * Ops[] = {Ptrs, getInt32(Align), Mask, PassThru};
 537
 538   // We specify only one type when we create this intrinsic. Types of other
 539   // arguments are derived from this type.
 540   return CreateMaskedIntrinsic(Intrinsic::masked_gather, Ops, OverloadedTypes,
 541                                Name);
 542 }
 543
 544 /// Create a call to a Masked Scatter intrinsic.
 545 /// \p Data  - data to be stored,
 546 /// \p Ptrs  - the vector of pointers, where the \p Data elements should be
 547 ///            stored
 548 /// \p Align - alignment for one element
 549 /// \p Mask  - vector of booleans which indicates what vector lanes should
 550 ///            be accessed in memory
 551 CallInst *IRBuilderBase::CreateMaskedScatter(Value *Data, Value *Ptrs,
 552                                              unsigned Align, Value *Mask) {
 553   auto PtrsTy = cast<VectorType>(Ptrs->getType());
 554   auto DataTy = cast<VectorType>(Data->getType());
 555   unsigned NumElts = PtrsTy->getVectorNumElements();
 556
 557 #ifndef NDEBUG
 558   auto PtrTy = cast<PointerType>(PtrsTy->getElementType());
 559   assert(NumElts == DataTy->getVectorNumElements() &&
 560          PtrTy->getElementType() == DataTy->getElementType() &&
 561          "Incompatible pointer and data types");
 562 #endif
 563
 564   if (!Mask)
 565     Mask = Constant::getAllOnesValue(VectorType::get(Type::getInt1Ty(Context),
 566                                      NumElts));
 567
 568   Type *OverloadedTypes[] = {DataTy, PtrsTy};
 569   Value * Ops[] = {Data, Ptrs, getInt32(Align), Mask};
 570
 571   // We specify only one type when we create this intrinsic. Types of other
 572   // arguments are derived from this type.
 573   return CreateMaskedIntrinsic(Intrinsic::masked_scatter, Ops, OverloadedTypes);
 574 }
 575
 576 template <typename T0, typename T1, typename T2, typename T3>
 577 static std::vector<Value *>
 578 getStatepointArgs(IRBuilderBase &B, uint64_t ID, uint32_t NumPatchBytes,
 579                   Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
 580                   ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs,
 581                   ArrayRef<T3> GCArgs) {
 582   std::vector<Value *> Args;
 583   Args.push_back(B.getInt64(ID));
 584   Args.push_back(B.getInt32(NumPatchBytes));
 585   Args.push_back(ActualCallee);
 586   Args.push_back(B.getInt32(CallArgs.size()));
 587   Args.push_back(B.getInt32(Flags));
 588   Args.insert(Args.end(), CallArgs.begin(), CallArgs.end());
 589   Args.push_back(B.getInt32(TransitionArgs.size()));
 590   Args.insert(Args.end(), TransitionArgs.begin(), TransitionArgs.end());
 591   Args.push_back(B.getInt32(DeoptArgs.size()));
 592   Args.insert(Args.end(), DeoptArgs.begin(), DeoptArgs.end());
 593   Args.insert(Args.end(), GCArgs.begin(), GCArgs.end());
 594
 595   return Args;
 596 }
 597
 598 template <typename T0, typename T1, typename T2, typename T3>
 599 static CallInst *CreateGCStatepointCallCommon(
 600     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
 601     Value *ActualCallee, uint32_t Flags, ArrayRef<T0> CallArgs,
 602     ArrayRef<T1> TransitionArgs, ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs,
 603     const Twine &Name) {
 604   // Extract out the type of the callee.
 605   auto *FuncPtrType = cast<PointerType>(ActualCallee->getType());
 606   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
 607          "actual callee must be a callable value");
 608
 609   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 610   // Fill in the one generic type'd argument (the function is also vararg)
 611   Type *ArgTypes[] = { FuncPtrType };
 612   Function *FnStatepoint =
 613     Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_statepoint,
 614                               ArgTypes);
 615
 616   std::vector<Value *> Args =
 617       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualCallee, Flags,
 618                         CallArgs, TransitionArgs, DeoptArgs, GCArgs);
 619   return createCallHelper(FnStatepoint, Args, Builder, Name);
 620 }
 621
 622 CallInst *IRBuilderBase::CreateGCStatepointCall(
 623     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
 624     ArrayRef<Value *> CallArgs, ArrayRef<Value *> DeoptArgs,
 625     ArrayRef<Value *> GCArgs, const Twine &Name) {
 626   return CreateGCStatepointCallCommon<Value *, Value *, Value *, Value *>(
 627       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
 628       CallArgs, None /* No Transition Args */, DeoptArgs, GCArgs, Name);
 629 }
 630
 631 CallInst *IRBuilderBase::CreateGCStatepointCall(
 632     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee, uint32_t Flags,
 633     ArrayRef<Use> CallArgs, ArrayRef<Use> TransitionArgs,
 634     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 635   return CreateGCStatepointCallCommon<Use, Use, Use, Value *>(
 636       this, ID, NumPatchBytes, ActualCallee, Flags, CallArgs, TransitionArgs,
 637       DeoptArgs, GCArgs, Name);
 638 }
 639
 640 CallInst *IRBuilderBase::CreateGCStatepointCall(
 641     uint64_t ID, uint32_t NumPatchBytes, Value *ActualCallee,
 642     ArrayRef<Use> CallArgs, ArrayRef<Value *> DeoptArgs,
 643     ArrayRef<Value *> GCArgs, const Twine &Name) {
 644   return CreateGCStatepointCallCommon<Use, Value *, Value *, Value *>(
 645       this, ID, NumPatchBytes, ActualCallee, uint32_t(StatepointFlags::None),
 646       CallArgs, None, DeoptArgs, GCArgs, Name);
 647 }
 648
 649 template <typename T0, typename T1, typename T2, typename T3>
 650 static InvokeInst *CreateGCStatepointInvokeCommon(
 651     IRBuilderBase *Builder, uint64_t ID, uint32_t NumPatchBytes,
 652     Value *ActualInvokee, BasicBlock *NormalDest, BasicBlock *UnwindDest,
 653     uint32_t Flags, ArrayRef<T0> InvokeArgs, ArrayRef<T1> TransitionArgs,
 654     ArrayRef<T2> DeoptArgs, ArrayRef<T3> GCArgs, const Twine &Name) {
 655   // Extract out the type of the callee.
 656   auto *FuncPtrType = cast<PointerType>(ActualInvokee->getType());
 657   assert(isa<FunctionType>(FuncPtrType->getElementType()) &&
 658          "actual callee must be a callable value");
 659
 660   Module *M = Builder->GetInsertBlock()->getParent()->getParent();
 661   // Fill in the one generic type'd argument (the function is also vararg)
 662   Function *FnStatepoint = Intrinsic::getDeclaration(
 663       M, Intrinsic::experimental_gc_statepoint, {FuncPtrType});
 664
 665   std::vector<Value *> Args =
 666       getStatepointArgs(*Builder, ID, NumPatchBytes, ActualInvokee, Flags,
 667                         InvokeArgs, TransitionArgs, DeoptArgs, GCArgs);
 668   return createInvokeHelper(FnStatepoint, NormalDest, UnwindDest, Args, Builder,
 669                             Name);
 670 }
 671
 672 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 673     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 674     BasicBlock *NormalDest, BasicBlock *UnwindDest,
 675     ArrayRef<Value *> InvokeArgs, ArrayRef<Value *> DeoptArgs,
 676     ArrayRef<Value *> GCArgs, const Twine &Name) {
 677   return CreateGCStatepointInvokeCommon<Value *, Value *, Value *, Value *>(
 678       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
 679       uint32_t(StatepointFlags::None), InvokeArgs, None /* No Transition Args*/,
 680       DeoptArgs, GCArgs, Name);
 681 }
 682
 683 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 684     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 685     BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags,
 686     ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs,
 687     ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 688   return CreateGCStatepointInvokeCommon<Use, Use, Use, Value *>(
 689       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest, Flags,
 690       InvokeArgs, TransitionArgs, DeoptArgs, GCArgs, Name);
 691 }
 692
 693 InvokeInst *IRBuilderBase::CreateGCStatepointInvoke(
 694     uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee,
 695     BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs,
 696     ArrayRef<Value *> DeoptArgs, ArrayRef<Value *> GCArgs, const Twine &Name) {
 697   return CreateGCStatepointInvokeCommon<Use, Value *, Value *, Value *>(
 698       this, ID, NumPatchBytes, ActualInvokee, NormalDest, UnwindDest,
 699       uint32_t(StatepointFlags::None), InvokeArgs, None, DeoptArgs, GCArgs,
 700       Name);
 701 }
 702
 703 CallInst *IRBuilderBase::CreateGCResult(Instruction *Statepoint,
 704                                        Type *ResultType,
 705                                        const Twine &Name) {
 706  Intrinsic::ID ID = Intrinsic::experimental_gc_result;
 707  Module *M = BB->getParent()->getParent();
 708  Type *Types[] = {ResultType};
 709  Function *FnGCResult = Intrinsic::getDeclaration(M, ID, Types);
 710
 711  Value *Args[] = {Statepoint};
 712  return createCallHelper(FnGCResult, Args, this, Name);
 713 }
 714
 715 CallInst *IRBuilderBase::CreateGCRelocate(Instruction *Statepoint,
 716                                          int BaseOffset,
 717                                          int DerivedOffset,
 718                                          Type *ResultType,
 719                                          const Twine &Name) {
 720  Module *M = BB->getParent()->getParent();
 721  Type *Types[] = {ResultType};
 722  Function *FnGCRelocate =
 723      Intrinsic::getDeclaration(M, Intrinsic::experimental_gc_relocate, Types);
 724
 725  Value *Args[] = {Statepoint,
 726                   getInt32(BaseOffset),
 727                   getInt32(DerivedOffset)};
 728  return createCallHelper(FnGCRelocate, Args, this, Name);
 729 }
 730
 731 CallInst *IRBuilderBase::CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V,
 732                                               Instruction *FMFSource,
 733                                               const Twine &Name) {
 734   Module *M = BB->getModule();
 735   Function *Fn = Intrinsic::getDeclaration(M, ID, {V->getType()});
 736   return createCallHelper(Fn, {V}, this, Name, FMFSource);
 737 }
 738
 739 CallInst *IRBuilderBase::CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS,
 740                                                Value *RHS,
 741                                                Instruction *FMFSource,
 742                                                const Twine &Name) {
 743   Module *M = BB->getModule();
 744   Function *Fn = Intrinsic::getDeclaration(M, ID, { LHS->getType() });
 745   return createCallHelper(Fn, {LHS, RHS}, this, Name, FMFSource);
 746 }
 747
 748 CallInst *IRBuilderBase::CreateIntrinsic(Intrinsic::ID ID,
 749                                          ArrayRef<Type *> Types,
 750                                          ArrayRef<Value *> Args,
 751                                          Instruction *FMFSource,
 752                                          const Twine &Name) {
 753   Module *M = BB->getModule();
 754   Function *Fn = Intrinsic::getDeclaration(M, ID, Types);
 755   return createCallHelper(Fn, Args, this, Name, FMFSource);
 756 }