llvm/lib/Target/WebAssembly/WebAssemblyFastISel.cpp

   1 //===-- WebAssemblyFastISel.cpp - WebAssembly FastISel implementation -----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file defines the WebAssembly-specific support for the FastISel
  11 /// class. Some of the target-specific code is generated by tablegen in the file
  12 /// WebAssemblyGenFastISel.inc, which is #included here.
  13 ///
  14 /// TODO: kill flags
  15 ///
  16 //===----------------------------------------------------------------------===//
  17
  18 #include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
  19 #include "Utils/WebAssemblyTypeUtilities.h"
  20 #include "WebAssembly.h"
  21 #include "WebAssemblyMachineFunctionInfo.h"
  22 #include "WebAssemblySubtarget.h"
  23 #include "WebAssemblyTargetMachine.h"
  24 #include "WebAssemblyUtilities.h"
  25 #include "llvm/Analysis/BranchProbabilityInfo.h"
  26 #include "llvm/CodeGen/FastISel.h"
  27 #include "llvm/CodeGen/FunctionLoweringInfo.h"
  28 #include "llvm/CodeGen/MachineConstantPool.h"
  29 #include "llvm/CodeGen/MachineFrameInfo.h"
  30 #include "llvm/CodeGen/MachineInstrBuilder.h"
  31 #include "llvm/CodeGen/MachineModuleInfo.h"
  32 #include "llvm/CodeGen/MachineRegisterInfo.h"
  33 #include "llvm/IR/DataLayout.h"
  34 #include "llvm/IR/DerivedTypes.h"
  35 #include "llvm/IR/Function.h"
  36 #include "llvm/IR/GetElementPtrTypeIterator.h"
  37 #include "llvm/IR/GlobalAlias.h"
  38 #include "llvm/IR/GlobalVariable.h"
  39 #include "llvm/IR/Instructions.h"
  40 #include "llvm/IR/IntrinsicInst.h"
  41 #include "llvm/IR/Operator.h"
  42 #include "llvm/IR/PatternMatch.h"
  43
  44 using namespace llvm;
  45 using namespace PatternMatch;
  46
  47 #define DEBUG_TYPE "wasm-fastisel"
  48
  49 namespace {
  50
  51 class WebAssemblyFastISel final : public FastISel {
  52   // All possible address modes.
  53   class Address {
  54   public:
  55     using BaseKind = enum { RegBase, FrameIndexBase };
  56
  57   private:
  58     BaseKind Kind = RegBase;
  59     union {
  60       unsigned Reg;
  61       int FI;
  62     } Base;
  63
  64     // Whether the base has been determined yet
  65     bool IsBaseSet = false;
  66
  67     int64_t Offset = 0;
  68
  69     const GlobalValue *GV = nullptr;
  70
  71   public:
  72     // Innocuous defaults for our address.
  73     Address() { Base.Reg = 0; }
  74     void setKind(BaseKind K) {
  75       assert(!isSet() && "Can't change kind with non-zero base");
  76       Kind = K;
  77     }
  78     BaseKind getKind() const { return Kind; }
  79     bool isRegBase() const { return Kind == RegBase; }
  80     bool isFIBase() const { return Kind == FrameIndexBase; }
  81     void setReg(unsigned Reg) {
  82       assert(isRegBase() && "Invalid base register access!");
  83       assert(!IsBaseSet && "Base cannot be reset");
  84       Base.Reg = Reg;
  85       IsBaseSet = true;
  86     }
  87     unsigned getReg() const {
  88       assert(isRegBase() && "Invalid base register access!");
  89       return Base.Reg;
  90     }
  91     void setFI(unsigned FI) {
  92       assert(isFIBase() && "Invalid base frame index access!");
  93       assert(!IsBaseSet && "Base cannot be reset");
  94       Base.FI = FI;
  95       IsBaseSet = true;
  96     }
  97     unsigned getFI() const {
  98       assert(isFIBase() && "Invalid base frame index access!");
  99       return Base.FI;
 100     }
 101
 102     void setOffset(int64_t NewOffset) {
 103       assert(NewOffset >= 0 && "Offsets must be non-negative");
 104       Offset = NewOffset;
 105     }
 106     int64_t getOffset() const { return Offset; }
 107     void setGlobalValue(const GlobalValue *G) { GV = G; }
 108     const GlobalValue *getGlobalValue() const { return GV; }
 109     bool isSet() const { return IsBaseSet; }
 110   };
 111
 112   /// Keep a pointer to the WebAssemblySubtarget around so that we can make the
 113   /// right decision when generating code for different targets.
 114   const WebAssemblySubtarget *Subtarget;
 115   LLVMContext *Context;
 116
 117 private:
 118   // Utility helper routines
 119   MVT::SimpleValueType getSimpleType(Type *Ty) {
 120     EVT VT = TLI.getValueType(DL, Ty, /*AllowUnknown=*/true);
 121     return VT.isSimple() ? VT.getSimpleVT().SimpleTy
 122                          : MVT::INVALID_SIMPLE_VALUE_TYPE;
 123   }
 124   MVT::SimpleValueType getLegalType(MVT::SimpleValueType VT) {
 125     switch (VT) {
 126     case MVT::i1:
 127     case MVT::i8:
 128     case MVT::i16:
 129       return MVT::i32;
 130     case MVT::i32:
 131     case MVT::i64:
 132     case MVT::f32:
 133     case MVT::f64:
 134       return VT;
 135     case MVT::funcref:
 136     case MVT::externref:
 137       if (Subtarget->hasReferenceTypes())
 138         return VT;
 139       break;
 140     case MVT::exnref:
 141       if (Subtarget->hasReferenceTypes() && Subtarget->hasExceptionHandling())
 142         return VT;
 143       break;
 144     case MVT::f16:
 145       return MVT::f32;
 146     case MVT::v16i8:
 147     case MVT::v8i16:
 148     case MVT::v4i32:
 149     case MVT::v4f32:
 150     case MVT::v2i64:
 151     case MVT::v2f64:
 152       if (Subtarget->hasSIMD128())
 153         return VT;
 154       break;
 155     default:
 156       break;
 157     }
 158     return MVT::INVALID_SIMPLE_VALUE_TYPE;
 159   }
 160   bool computeAddress(const Value *Obj, Address &Addr);
 161   void materializeLoadStoreOperands(Address &Addr);
 162   void addLoadStoreOperands(const Address &Addr, const MachineInstrBuilder &MIB,
 163                             MachineMemOperand *MMO);
 164   unsigned maskI1Value(unsigned Reg, const Value *V);
 165   unsigned getRegForI1Value(const Value *V, const BasicBlock *BB, bool &Not);
 166   unsigned zeroExtendToI32(unsigned Reg, const Value *V,
 167                            MVT::SimpleValueType From);
 168   unsigned signExtendToI32(unsigned Reg, const Value *V,
 169                            MVT::SimpleValueType From);
 170   unsigned zeroExtend(unsigned Reg, const Value *V, MVT::SimpleValueType From,
 171                       MVT::SimpleValueType To);
 172   unsigned signExtend(unsigned Reg, const Value *V, MVT::SimpleValueType From,
 173                       MVT::SimpleValueType To);
 174   unsigned getRegForUnsignedValue(const Value *V);
 175   unsigned getRegForSignedValue(const Value *V);
 176   unsigned getRegForPromotedValue(const Value *V, bool IsSigned);
 177   unsigned notValue(unsigned Reg);
 178   unsigned copyValue(unsigned Reg);
 179
 180   // Backend specific FastISel code.
 181   unsigned fastMaterializeAlloca(const AllocaInst *AI) override;
 182   unsigned fastMaterializeConstant(const Constant *C) override;
 183   bool fastLowerArguments() override;
 184
 185   // Selection routines.
 186   bool selectCall(const Instruction *I);
 187   bool selectSelect(const Instruction *I);
 188   bool selectTrunc(const Instruction *I);
 189   bool selectZExt(const Instruction *I);
 190   bool selectSExt(const Instruction *I);
 191   bool selectICmp(const Instruction *I);
 192   bool selectFCmp(const Instruction *I);
 193   bool selectBitCast(const Instruction *I);
 194   bool selectLoad(const Instruction *I);
 195   bool selectStore(const Instruction *I);
 196   bool selectBr(const Instruction *I);
 197   bool selectRet(const Instruction *I);
 198   bool selectUnreachable(const Instruction *I);
 199
 200 public:
 201   // Backend specific FastISel code.
 202   WebAssemblyFastISel(FunctionLoweringInfo &FuncInfo,
 203                       const TargetLibraryInfo *LibInfo)
 204       : FastISel(FuncInfo, LibInfo, /*SkipTargetIndependentISel=*/true) {
 205     Subtarget = &FuncInfo.MF->getSubtarget<WebAssemblySubtarget>();
 206     Context = &FuncInfo.Fn->getContext();
 207   }
 208
 209   bool fastSelectInstruction(const Instruction *I) override;
 210
 211 #include "WebAssemblyGenFastISel.inc"
 212 };
 213
 214 } // end anonymous namespace
 215
 216 bool WebAssemblyFastISel::computeAddress(const Value *Obj, Address &Addr) {
 217   const User *U = nullptr;
 218   unsigned Opcode = Instruction::UserOp1;
 219   if (const auto *I = dyn_cast<Instruction>(Obj)) {
 220     // Don't walk into other basic blocks unless the object is an alloca from
 221     // another block, otherwise it may not have a virtual register assigned.
 222     if (FuncInfo.StaticAllocaMap.count(static_cast<const AllocaInst *>(Obj)) ||
 223         FuncInfo.MBBMap[I->getParent()] == FuncInfo.MBB) {
 224       Opcode = I->getOpcode();
 225       U = I;
 226     }
 227   } else if (const auto *C = dyn_cast<ConstantExpr>(Obj)) {
 228     Opcode = C->getOpcode();
 229     U = C;
 230   }
 231
 232   if (auto *Ty = dyn_cast<PointerType>(Obj->getType()))
 233     if (Ty->getAddressSpace() > 255)
 234       // Fast instruction selection doesn't support the special
 235       // address spaces.
 236       return false;
 237
 238   if (const auto *GV = dyn_cast<GlobalValue>(Obj)) {
 239     if (TLI.isPositionIndependent())
 240       return false;
 241     if (Addr.getGlobalValue())
 242       return false;
 243     if (GV->isThreadLocal())
 244       return false;
 245     Addr.setGlobalValue(GV);
 246     return true;
 247   }
 248
 249   switch (Opcode) {
 250   default:
 251     break;
 252   case Instruction::BitCast: {
 253     // Look through bitcasts.
 254     return computeAddress(U->getOperand(0), Addr);
 255   }
 256   case Instruction::IntToPtr: {
 257     // Look past no-op inttoptrs.
 258     if (TLI.getValueType(DL, U->getOperand(0)->getType()) ==
 259         TLI.getPointerTy(DL))
 260       return computeAddress(U->getOperand(0), Addr);
 261     break;
 262   }
 263   case Instruction::PtrToInt: {
 264     // Look past no-op ptrtoints.
 265     if (TLI.getValueType(DL, U->getType()) == TLI.getPointerTy(DL))
 266       return computeAddress(U->getOperand(0), Addr);
 267     break;
 268   }
 269   case Instruction::GetElementPtr: {
 270     Address SavedAddr = Addr;
 271     uint64_t TmpOffset = Addr.getOffset();
 272     // Non-inbounds geps can wrap; wasm's offsets can't.
 273     if (!cast<GEPOperator>(U)->isInBounds())
 274       goto unsupported_gep;
 275     // Iterate through the GEP folding the constants into offsets where
 276     // we can.
 277     for (gep_type_iterator GTI = gep_type_begin(U), E = gep_type_end(U);
 278          GTI != E; ++GTI) {
 279       const Value *Op = GTI.getOperand();
 280       if (StructType *STy = GTI.getStructTypeOrNull()) {
 281         const StructLayout *SL = DL.getStructLayout(STy);
 282         unsigned Idx = cast<ConstantInt>(Op)->getZExtValue();
 283         TmpOffset += SL->getElementOffset(Idx);
 284       } else {
 285         uint64_t S = GTI.getSequentialElementStride(DL);
 286         for (;;) {
 287           if (const auto *CI = dyn_cast<ConstantInt>(Op)) {
 288             // Constant-offset addressing.
 289             TmpOffset += CI->getSExtValue() * S;
 290             break;
 291           }
 292           if (S == 1 && Addr.isRegBase() && Addr.getReg() == 0) {
 293             // An unscaled add of a register. Set it as the new base.
 294             Register Reg = getRegForValue(Op);
 295             if (Reg == 0)
 296               return false;
 297             Addr.setReg(Reg);
 298             break;
 299           }
 300           if (canFoldAddIntoGEP(U, Op)) {
 301             // A compatible add with a constant operand. Fold the constant.
 302             auto *CI = cast<ConstantInt>(cast<AddOperator>(Op)->getOperand(1));
 303             TmpOffset += CI->getSExtValue() * S;
 304             // Iterate on the other operand.
 305             Op = cast<AddOperator>(Op)->getOperand(0);
 306             continue;
 307           }
 308           // Unsupported
 309           goto unsupported_gep;
 310         }
 311       }
 312     }
 313     // Don't fold in negative offsets.
 314     if (int64_t(TmpOffset) >= 0) {
 315       // Try to grab the base operand now.
 316       Addr.setOffset(TmpOffset);
 317       if (computeAddress(U->getOperand(0), Addr))
 318         return true;
 319     }
 320     // We failed, restore everything and try the other options.
 321     Addr = SavedAddr;
 322   unsupported_gep:
 323     break;
 324   }
 325   case Instruction::Alloca: {
 326     const auto *AI = cast<AllocaInst>(Obj);
 327     DenseMap<const AllocaInst *, int>::iterator SI =
 328         FuncInfo.StaticAllocaMap.find(AI);
 329     if (SI != FuncInfo.StaticAllocaMap.end()) {
 330       if (Addr.isSet()) {
 331         return false;
 332       }
 333       Addr.setKind(Address::FrameIndexBase);
 334       Addr.setFI(SI->second);
 335       return true;
 336     }
 337     break;
 338   }
 339   case Instruction::Add: {
 340     // Adds of constants are common and easy enough.
 341     const Value *LHS = U->getOperand(0);
 342     const Value *RHS = U->getOperand(1);
 343
 344     if (isa<ConstantInt>(LHS))
 345       std::swap(LHS, RHS);
 346
 347     if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
 348       uint64_t TmpOffset = Addr.getOffset() + CI->getSExtValue();
 349       if (int64_t(TmpOffset) >= 0) {
 350         Addr.setOffset(TmpOffset);
 351         return computeAddress(LHS, Addr);
 352       }
 353     }
 354
 355     Address Backup = Addr;
 356     if (computeAddress(LHS, Addr) && computeAddress(RHS, Addr))
 357       return true;
 358     Addr = Backup;
 359
 360     break;
 361   }
 362   case Instruction::Sub: {
 363     // Subs of constants are common and easy enough.
 364     const Value *LHS = U->getOperand(0);
 365     const Value *RHS = U->getOperand(1);
 366
 367     if (const auto *CI = dyn_cast<ConstantInt>(RHS)) {
 368       int64_t TmpOffset = Addr.getOffset() - CI->getSExtValue();
 369       if (TmpOffset >= 0) {
 370         Addr.setOffset(TmpOffset);
 371         return computeAddress(LHS, Addr);
 372       }
 373     }
 374     break;
 375   }
 376   }
 377   if (Addr.isSet()) {
 378     return false;
 379   }
 380   Register Reg = getRegForValue(Obj);
 381   if (Reg == 0)
 382     return false;
 383   Addr.setReg(Reg);
 384   return Addr.getReg() != 0;
 385 }
 386
 387 void WebAssemblyFastISel::materializeLoadStoreOperands(Address &Addr) {
 388   if (Addr.isRegBase()) {
 389     unsigned Reg = Addr.getReg();
 390     if (Reg == 0) {
 391       Reg = createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
 392                                                    : &WebAssembly::I32RegClass);
 393       unsigned Opc = Subtarget->hasAddr64() ? WebAssembly::CONST_I64
 394                                             : WebAssembly::CONST_I32;
 395       BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), Reg)
 396           .addImm(0);
 397       Addr.setReg(Reg);
 398     }
 399   }
 400 }
 401
 402 void WebAssemblyFastISel::addLoadStoreOperands(const Address &Addr,
 403                                                const MachineInstrBuilder &MIB,
 404                                                MachineMemOperand *MMO) {
 405   // Set the alignment operand (this is rewritten in SetP2AlignOperands).
 406   // TODO: Disable SetP2AlignOperands for FastISel and just do it here.
 407   MIB.addImm(0);
 408
 409   if (const GlobalValue *GV = Addr.getGlobalValue())
 410     MIB.addGlobalAddress(GV, Addr.getOffset());
 411   else
 412     MIB.addImm(Addr.getOffset());
 413
 414   if (Addr.isRegBase())
 415     MIB.addReg(Addr.getReg());
 416   else
 417     MIB.addFrameIndex(Addr.getFI());
 418
 419   MIB.addMemOperand(MMO);
 420 }
 421
 422 unsigned WebAssemblyFastISel::maskI1Value(unsigned Reg, const Value *V) {
 423   return zeroExtendToI32(Reg, V, MVT::i1);
 424 }
 425
 426 unsigned WebAssemblyFastISel::getRegForI1Value(const Value *V,
 427                                                const BasicBlock *BB,
 428                                                bool &Not) {
 429   if (const auto *ICmp = dyn_cast<ICmpInst>(V))
 430     if (const ConstantInt *C = dyn_cast<ConstantInt>(ICmp->getOperand(1)))
 431       if (ICmp->isEquality() && C->isZero() && C->getType()->isIntegerTy(32) &&
 432           ICmp->getParent() == BB) {
 433         Not = ICmp->isTrueWhenEqual();
 434         return getRegForValue(ICmp->getOperand(0));
 435       }
 436
 437   Not = false;
 438   Register Reg = getRegForValue(V);
 439   if (Reg == 0)
 440     return 0;
 441   return maskI1Value(Reg, V);
 442 }
 443
 444 unsigned WebAssemblyFastISel::zeroExtendToI32(unsigned Reg, const Value *V,
 445                                               MVT::SimpleValueType From) {
 446   if (Reg == 0)
 447     return 0;
 448
 449   switch (From) {
 450   case MVT::i1:
 451     // If the value is naturally an i1, we don't need to mask it. We only know
 452     // if a value is naturally an i1 if it is definitely lowered by FastISel,
 453     // not a DAG ISel fallback.
 454     if (V != nullptr && isa<Argument>(V) && cast<Argument>(V)->hasZExtAttr())
 455       return copyValue(Reg);
 456     break;
 457   case MVT::i8:
 458   case MVT::i16:
 459     break;
 460   case MVT::i32:
 461     return copyValue(Reg);
 462   default:
 463     return 0;
 464   }
 465
 466   Register Imm = createResultReg(&WebAssembly::I32RegClass);
 467   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 468           TII.get(WebAssembly::CONST_I32), Imm)
 469       .addImm(~(~uint64_t(0) << MVT(From).getSizeInBits()));
 470
 471   Register Result = createResultReg(&WebAssembly::I32RegClass);
 472   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 473           TII.get(WebAssembly::AND_I32), Result)
 474       .addReg(Reg)
 475       .addReg(Imm);
 476
 477   return Result;
 478 }
 479
 480 unsigned WebAssemblyFastISel::signExtendToI32(unsigned Reg, const Value *V,
 481                                               MVT::SimpleValueType From) {
 482   if (Reg == 0)
 483     return 0;
 484
 485   switch (From) {
 486   case MVT::i1:
 487   case MVT::i8:
 488   case MVT::i16:
 489     break;
 490   case MVT::i32:
 491     return copyValue(Reg);
 492   default:
 493     return 0;
 494   }
 495
 496   Register Imm = createResultReg(&WebAssembly::I32RegClass);
 497   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 498           TII.get(WebAssembly::CONST_I32), Imm)
 499       .addImm(32 - MVT(From).getSizeInBits());
 500
 501   Register Left = createResultReg(&WebAssembly::I32RegClass);
 502   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 503           TII.get(WebAssembly::SHL_I32), Left)
 504       .addReg(Reg)
 505       .addReg(Imm);
 506
 507   Register Right = createResultReg(&WebAssembly::I32RegClass);
 508   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 509           TII.get(WebAssembly::SHR_S_I32), Right)
 510       .addReg(Left)
 511       .addReg(Imm);
 512
 513   return Right;
 514 }
 515
 516 unsigned WebAssemblyFastISel::zeroExtend(unsigned Reg, const Value *V,
 517                                          MVT::SimpleValueType From,
 518                                          MVT::SimpleValueType To) {
 519   if (To == MVT::i64) {
 520     if (From == MVT::i64)
 521       return copyValue(Reg);
 522
 523     Reg = zeroExtendToI32(Reg, V, From);
 524
 525     Register Result = createResultReg(&WebAssembly::I64RegClass);
 526     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 527             TII.get(WebAssembly::I64_EXTEND_U_I32), Result)
 528         .addReg(Reg);
 529     return Result;
 530   }
 531
 532   if (To == MVT::i32)
 533     return zeroExtendToI32(Reg, V, From);
 534
 535   return 0;
 536 }
 537
 538 unsigned WebAssemblyFastISel::signExtend(unsigned Reg, const Value *V,
 539                                          MVT::SimpleValueType From,
 540                                          MVT::SimpleValueType To) {
 541   if (To == MVT::i64) {
 542     if (From == MVT::i64)
 543       return copyValue(Reg);
 544
 545     Reg = signExtendToI32(Reg, V, From);
 546
 547     Register Result = createResultReg(&WebAssembly::I64RegClass);
 548     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 549             TII.get(WebAssembly::I64_EXTEND_S_I32), Result)
 550         .addReg(Reg);
 551     return Result;
 552   }
 553
 554   if (To == MVT::i32)
 555     return signExtendToI32(Reg, V, From);
 556
 557   return 0;
 558 }
 559
 560 unsigned WebAssemblyFastISel::getRegForUnsignedValue(const Value *V) {
 561   MVT::SimpleValueType From = getSimpleType(V->getType());
 562   MVT::SimpleValueType To = getLegalType(From);
 563   Register VReg = getRegForValue(V);
 564   if (VReg == 0)
 565     return 0;
 566   if (From == To)
 567     return VReg;
 568   return zeroExtend(VReg, V, From, To);
 569 }
 570
 571 unsigned WebAssemblyFastISel::getRegForSignedValue(const Value *V) {
 572   MVT::SimpleValueType From = getSimpleType(V->getType());
 573   MVT::SimpleValueType To = getLegalType(From);
 574   Register VReg = getRegForValue(V);
 575   if (VReg == 0)
 576     return 0;
 577   if (From == To)
 578     return VReg;
 579   return signExtend(VReg, V, From, To);
 580 }
 581
 582 unsigned WebAssemblyFastISel::getRegForPromotedValue(const Value *V,
 583                                                      bool IsSigned) {
 584   return IsSigned ? getRegForSignedValue(V) : getRegForUnsignedValue(V);
 585 }
 586
 587 unsigned WebAssemblyFastISel::notValue(unsigned Reg) {
 588   assert(MRI.getRegClass(Reg) == &WebAssembly::I32RegClass);
 589
 590   Register NotReg = createResultReg(&WebAssembly::I32RegClass);
 591   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 592           TII.get(WebAssembly::EQZ_I32), NotReg)
 593       .addReg(Reg);
 594   return NotReg;
 595 }
 596
 597 unsigned WebAssemblyFastISel::copyValue(unsigned Reg) {
 598   Register ResultReg = createResultReg(MRI.getRegClass(Reg));
 599   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(WebAssembly::COPY),
 600           ResultReg)
 601       .addReg(Reg);
 602   return ResultReg;
 603 }
 604
 605 unsigned WebAssemblyFastISel::fastMaterializeAlloca(const AllocaInst *AI) {
 606   DenseMap<const AllocaInst *, int>::iterator SI =
 607       FuncInfo.StaticAllocaMap.find(AI);
 608
 609   if (SI != FuncInfo.StaticAllocaMap.end()) {
 610     Register ResultReg =
 611         createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
 612                                                : &WebAssembly::I32RegClass);
 613     unsigned Opc =
 614         Subtarget->hasAddr64() ? WebAssembly::COPY_I64 : WebAssembly::COPY_I32;
 615     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
 616         .addFrameIndex(SI->second);
 617     return ResultReg;
 618   }
 619
 620   return 0;
 621 }
 622
 623 unsigned WebAssemblyFastISel::fastMaterializeConstant(const Constant *C) {
 624   if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) {
 625     if (TLI.isPositionIndependent())
 626       return 0;
 627     if (GV->isThreadLocal())
 628       return 0;
 629     Register ResultReg =
 630         createResultReg(Subtarget->hasAddr64() ? &WebAssembly::I64RegClass
 631                                                : &WebAssembly::I32RegClass);
 632     unsigned Opc = Subtarget->hasAddr64() ? WebAssembly::CONST_I64
 633                                           : WebAssembly::CONST_I32;
 634     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
 635         .addGlobalAddress(GV);
 636     return ResultReg;
 637   }
 638
 639   // Let target-independent code handle it.
 640   return 0;
 641 }
 642
 643 bool WebAssemblyFastISel::fastLowerArguments() {
 644   if (!FuncInfo.CanLowerReturn)
 645     return false;
 646
 647   const Function *F = FuncInfo.Fn;
 648   if (F->isVarArg())
 649     return false;
 650
 651   if (FuncInfo.Fn->getCallingConv() == CallingConv::Swift)
 652     return false;
 653
 654   unsigned I = 0;
 655   for (auto const &Arg : F->args()) {
 656     const AttributeList &Attrs = F->getAttributes();
 657     if (Attrs.hasParamAttr(I, Attribute::ByVal) ||
 658         Attrs.hasParamAttr(I, Attribute::SwiftSelf) ||
 659         Attrs.hasParamAttr(I, Attribute::SwiftError) ||
 660         Attrs.hasParamAttr(I, Attribute::InAlloca) ||
 661         Attrs.hasParamAttr(I, Attribute::Nest))
 662       return false;
 663
 664     Type *ArgTy = Arg.getType();
 665     if (ArgTy->isStructTy() || ArgTy->isArrayTy())
 666       return false;
 667     if (!Subtarget->hasSIMD128() && ArgTy->isVectorTy())
 668       return false;
 669
 670     unsigned Opc;
 671     const TargetRegisterClass *RC;
 672     switch (getSimpleType(ArgTy)) {
 673     case MVT::i1:
 674     case MVT::i8:
 675     case MVT::i16:
 676     case MVT::i32:
 677       Opc = WebAssembly::ARGUMENT_i32;
 678       RC = &WebAssembly::I32RegClass;
 679       break;
 680     case MVT::i64:
 681       Opc = WebAssembly::ARGUMENT_i64;
 682       RC = &WebAssembly::I64RegClass;
 683       break;
 684     case MVT::f32:
 685       Opc = WebAssembly::ARGUMENT_f32;
 686       RC = &WebAssembly::F32RegClass;
 687       break;
 688     case MVT::f64:
 689       Opc = WebAssembly::ARGUMENT_f64;
 690       RC = &WebAssembly::F64RegClass;
 691       break;
 692     case MVT::v16i8:
 693       Opc = WebAssembly::ARGUMENT_v16i8;
 694       RC = &WebAssembly::V128RegClass;
 695       break;
 696     case MVT::v8i16:
 697       Opc = WebAssembly::ARGUMENT_v8i16;
 698       RC = &WebAssembly::V128RegClass;
 699       break;
 700     case MVT::v4i32:
 701       Opc = WebAssembly::ARGUMENT_v4i32;
 702       RC = &WebAssembly::V128RegClass;
 703       break;
 704     case MVT::v2i64:
 705       Opc = WebAssembly::ARGUMENT_v2i64;
 706       RC = &WebAssembly::V128RegClass;
 707       break;
 708     case MVT::v4f32:
 709       Opc = WebAssembly::ARGUMENT_v4f32;
 710       RC = &WebAssembly::V128RegClass;
 711       break;
 712     case MVT::v2f64:
 713       Opc = WebAssembly::ARGUMENT_v2f64;
 714       RC = &WebAssembly::V128RegClass;
 715       break;
 716     case MVT::funcref:
 717       Opc = WebAssembly::ARGUMENT_funcref;
 718       RC = &WebAssembly::FUNCREFRegClass;
 719       break;
 720     case MVT::externref:
 721       Opc = WebAssembly::ARGUMENT_externref;
 722       RC = &WebAssembly::EXTERNREFRegClass;
 723       break;
 724     case MVT::exnref:
 725       Opc = WebAssembly::ARGUMENT_exnref;
 726       RC = &WebAssembly::EXNREFRegClass;
 727       break;
 728     default:
 729       return false;
 730     }
 731     Register ResultReg = createResultReg(RC);
 732     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
 733         .addImm(I);
 734     updateValueMap(&Arg, ResultReg);
 735
 736     ++I;
 737   }
 738
 739   MRI.addLiveIn(WebAssembly::ARGUMENTS);
 740
 741   auto *MFI = MF->getInfo<WebAssemblyFunctionInfo>();
 742   for (auto const &Arg : F->args()) {
 743     MVT::SimpleValueType ArgTy = getLegalType(getSimpleType(Arg.getType()));
 744     if (ArgTy == MVT::INVALID_SIMPLE_VALUE_TYPE) {
 745       MFI->clearParamsAndResults();
 746       return false;
 747     }
 748     MFI->addParam(ArgTy);
 749   }
 750
 751   if (!F->getReturnType()->isVoidTy()) {
 752     MVT::SimpleValueType RetTy =
 753         getLegalType(getSimpleType(F->getReturnType()));
 754     if (RetTy == MVT::INVALID_SIMPLE_VALUE_TYPE) {
 755       MFI->clearParamsAndResults();
 756       return false;
 757     }
 758     MFI->addResult(RetTy);
 759   }
 760
 761   return true;
 762 }
 763
 764 bool WebAssemblyFastISel::selectCall(const Instruction *I) {
 765   const auto *Call = cast<CallInst>(I);
 766
 767   // TODO: Support tail calls in FastISel
 768   if (Call->isMustTailCall() || Call->isInlineAsm() ||
 769       Call->getFunctionType()->isVarArg())
 770     return false;
 771
 772   Function *Func = Call->getCalledFunction();
 773   if (Func && Func->isIntrinsic())
 774     return false;
 775
 776   if (Call->getCallingConv() == CallingConv::Swift)
 777     return false;
 778
 779   bool IsDirect = Func != nullptr;
 780   if (!IsDirect && isa<ConstantExpr>(Call->getCalledOperand()))
 781     return false;
 782
 783   FunctionType *FuncTy = Call->getFunctionType();
 784   unsigned Opc = IsDirect ? WebAssembly::CALL : WebAssembly::CALL_INDIRECT;
 785   bool IsVoid = FuncTy->getReturnType()->isVoidTy();
 786   unsigned ResultReg;
 787   if (!IsVoid) {
 788     if (!Subtarget->hasSIMD128() && Call->getType()->isVectorTy())
 789       return false;
 790
 791     MVT::SimpleValueType RetTy = getSimpleType(Call->getType());
 792     switch (RetTy) {
 793     case MVT::i1:
 794     case MVT::i8:
 795     case MVT::i16:
 796     case MVT::i32:
 797       ResultReg = createResultReg(&WebAssembly::I32RegClass);
 798       break;
 799     case MVT::i64:
 800       ResultReg = createResultReg(&WebAssembly::I64RegClass);
 801       break;
 802     case MVT::f32:
 803       ResultReg = createResultReg(&WebAssembly::F32RegClass);
 804       break;
 805     case MVT::f64:
 806       ResultReg = createResultReg(&WebAssembly::F64RegClass);
 807       break;
 808     case MVT::v16i8:
 809       ResultReg = createResultReg(&WebAssembly::V128RegClass);
 810       break;
 811     case MVT::v8i16:
 812       ResultReg = createResultReg(&WebAssembly::V128RegClass);
 813       break;
 814     case MVT::v4i32:
 815       ResultReg = createResultReg(&WebAssembly::V128RegClass);
 816       break;
 817     case MVT::v2i64:
 818       ResultReg = createResultReg(&WebAssembly::V128RegClass);
 819       break;
 820     case MVT::v4f32:
 821       ResultReg = createResultReg(&WebAssembly::V128RegClass);
 822       break;
 823     case MVT::v2f64:
 824       ResultReg = createResultReg(&WebAssembly::V128RegClass);
 825       break;
 826     case MVT::funcref:
 827       ResultReg = createResultReg(&WebAssembly::FUNCREFRegClass);
 828       break;
 829     case MVT::externref:
 830       ResultReg = createResultReg(&WebAssembly::EXTERNREFRegClass);
 831       break;
 832     case MVT::exnref:
 833       ResultReg = createResultReg(&WebAssembly::EXNREFRegClass);
 834       break;
 835     default:
 836       return false;
 837     }
 838   }
 839
 840   SmallVector<unsigned, 8> Args;
 841   for (unsigned I = 0, E = Call->arg_size(); I < E; ++I) {
 842     Value *V = Call->getArgOperand(I);
 843     MVT::SimpleValueType ArgTy = getSimpleType(V->getType());
 844     if (ArgTy == MVT::INVALID_SIMPLE_VALUE_TYPE)
 845       return false;
 846
 847     const AttributeList &Attrs = Call->getAttributes();
 848     if (Attrs.hasParamAttr(I, Attribute::ByVal) ||
 849         Attrs.hasParamAttr(I, Attribute::SwiftSelf) ||
 850         Attrs.hasParamAttr(I, Attribute::SwiftError) ||
 851         Attrs.hasParamAttr(I, Attribute::InAlloca) ||
 852         Attrs.hasParamAttr(I, Attribute::Nest))
 853       return false;
 854
 855     unsigned Reg;
 856
 857     if (Call->paramHasAttr(I, Attribute::SExt))
 858       Reg = getRegForSignedValue(V);
 859     else if (Call->paramHasAttr(I, Attribute::ZExt))
 860       Reg = getRegForUnsignedValue(V);
 861     else
 862       Reg = getRegForValue(V);
 863
 864     if (Reg == 0)
 865       return false;
 866
 867     Args.push_back(Reg);
 868   }
 869
 870   unsigned CalleeReg = 0;
 871   if (!IsDirect) {
 872     CalleeReg = getRegForValue(Call->getCalledOperand());
 873     if (!CalleeReg)
 874       return false;
 875   }
 876
 877   auto MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc));
 878
 879   if (!IsVoid)
 880     MIB.addReg(ResultReg, RegState::Define);
 881
 882   if (IsDirect) {
 883     MIB.addGlobalAddress(Func);
 884   } else {
 885     // Placeholder for the type index.
 886     MIB.addImm(0);
 887     // The table into which this call_indirect indexes.
 888     MCSymbolWasm *Table = WebAssembly::getOrCreateFunctionTableSymbol(
 889         MF->getContext(), Subtarget);
 890     if (Subtarget->hasReferenceTypes()) {
 891       MIB.addSym(Table);
 892     } else {
 893       // Otherwise for the MVP there is at most one table whose number is 0, but
 894       // we can't write a table symbol or issue relocations.  Instead we just
 895       // ensure the table is live.
 896       Table->setNoStrip();
 897       MIB.addImm(0);
 898     }
 899   }
 900
 901   for (unsigned ArgReg : Args)
 902     MIB.addReg(ArgReg);
 903
 904   if (!IsDirect)
 905     MIB.addReg(CalleeReg);
 906
 907   if (!IsVoid)
 908     updateValueMap(Call, ResultReg);
 909   return true;
 910 }
 911
 912 bool WebAssemblyFastISel::selectSelect(const Instruction *I) {
 913   const auto *Select = cast<SelectInst>(I);
 914
 915   bool Not;
 916   unsigned CondReg =
 917       getRegForI1Value(Select->getCondition(), I->getParent(), Not);
 918   if (CondReg == 0)
 919     return false;
 920
 921   Register TrueReg = getRegForValue(Select->getTrueValue());
 922   if (TrueReg == 0)
 923     return false;
 924
 925   Register FalseReg = getRegForValue(Select->getFalseValue());
 926   if (FalseReg == 0)
 927     return false;
 928
 929   if (Not)
 930     std::swap(TrueReg, FalseReg);
 931
 932   unsigned Opc;
 933   const TargetRegisterClass *RC;
 934   switch (getSimpleType(Select->getType())) {
 935   case MVT::i1:
 936   case MVT::i8:
 937   case MVT::i16:
 938   case MVT::i32:
 939     Opc = WebAssembly::SELECT_I32;
 940     RC = &WebAssembly::I32RegClass;
 941     break;
 942   case MVT::i64:
 943     Opc = WebAssembly::SELECT_I64;
 944     RC = &WebAssembly::I64RegClass;
 945     break;
 946   case MVT::f32:
 947     Opc = WebAssembly::SELECT_F32;
 948     RC = &WebAssembly::F32RegClass;
 949     break;
 950   case MVT::f64:
 951     Opc = WebAssembly::SELECT_F64;
 952     RC = &WebAssembly::F64RegClass;
 953     break;
 954   case MVT::funcref:
 955     Opc = WebAssembly::SELECT_FUNCREF;
 956     RC = &WebAssembly::FUNCREFRegClass;
 957     break;
 958   case MVT::externref:
 959     Opc = WebAssembly::SELECT_EXTERNREF;
 960     RC = &WebAssembly::EXTERNREFRegClass;
 961     break;
 962   case MVT::exnref:
 963     Opc = WebAssembly::SELECT_EXNREF;
 964     RC = &WebAssembly::EXNREFRegClass;
 965     break;
 966   default:
 967     return false;
 968   }
 969
 970   Register ResultReg = createResultReg(RC);
 971   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
 972       .addReg(TrueReg)
 973       .addReg(FalseReg)
 974       .addReg(CondReg);
 975
 976   updateValueMap(Select, ResultReg);
 977   return true;
 978 }
 979
 980 bool WebAssemblyFastISel::selectTrunc(const Instruction *I) {
 981   const auto *Trunc = cast<TruncInst>(I);
 982
 983   Register Reg = getRegForValue(Trunc->getOperand(0));
 984   if (Reg == 0)
 985     return false;
 986
 987   if (Trunc->getOperand(0)->getType()->isIntegerTy(64)) {
 988     Register Result = createResultReg(&WebAssembly::I32RegClass);
 989     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
 990             TII.get(WebAssembly::I32_WRAP_I64), Result)
 991         .addReg(Reg);
 992     Reg = Result;
 993   }
 994
 995   updateValueMap(Trunc, Reg);
 996   return true;
 997 }
 998
 999 bool WebAssemblyFastISel::selectZExt(const Instruction *I) {
1000   const auto *ZExt = cast<ZExtInst>(I);
1001
1002   const Value *Op = ZExt->getOperand(0);
1003   MVT::SimpleValueType From = getSimpleType(Op->getType());
1004   MVT::SimpleValueType To = getLegalType(getSimpleType(ZExt->getType()));
1005   Register In = getRegForValue(Op);
1006   if (In == 0)
1007     return false;
1008   unsigned Reg = zeroExtend(In, Op, From, To);
1009   if (Reg == 0)
1010     return false;
1011
1012   updateValueMap(ZExt, Reg);
1013   return true;
1014 }
1015
1016 bool WebAssemblyFastISel::selectSExt(const Instruction *I) {
1017   const auto *SExt = cast<SExtInst>(I);
1018
1019   const Value *Op = SExt->getOperand(0);
1020   MVT::SimpleValueType From = getSimpleType(Op->getType());
1021   MVT::SimpleValueType To = getLegalType(getSimpleType(SExt->getType()));
1022   Register In = getRegForValue(Op);
1023   if (In == 0)
1024     return false;
1025   unsigned Reg = signExtend(In, Op, From, To);
1026   if (Reg == 0)
1027     return false;
1028
1029   updateValueMap(SExt, Reg);
1030   return true;
1031 }
1032
1033 bool WebAssemblyFastISel::selectICmp(const Instruction *I) {
1034   const auto *ICmp = cast<ICmpInst>(I);
1035
1036   bool I32 = getSimpleType(ICmp->getOperand(0)->getType()) != MVT::i64;
1037   unsigned Opc;
1038   bool IsSigned = false;
1039   switch (ICmp->getPredicate()) {
1040   case ICmpInst::ICMP_EQ:
1041     Opc = I32 ? WebAssembly::EQ_I32 : WebAssembly::EQ_I64;
1042     break;
1043   case ICmpInst::ICMP_NE:
1044     Opc = I32 ? WebAssembly::NE_I32 : WebAssembly::NE_I64;
1045     break;
1046   case ICmpInst::ICMP_UGT:
1047     Opc = I32 ? WebAssembly::GT_U_I32 : WebAssembly::GT_U_I64;
1048     break;
1049   case ICmpInst::ICMP_UGE:
1050     Opc = I32 ? WebAssembly::GE_U_I32 : WebAssembly::GE_U_I64;
1051     break;
1052   case ICmpInst::ICMP_ULT:
1053     Opc = I32 ? WebAssembly::LT_U_I32 : WebAssembly::LT_U_I64;
1054     break;
1055   case ICmpInst::ICMP_ULE:
1056     Opc = I32 ? WebAssembly::LE_U_I32 : WebAssembly::LE_U_I64;
1057     break;
1058   case ICmpInst::ICMP_SGT:
1059     Opc = I32 ? WebAssembly::GT_S_I32 : WebAssembly::GT_S_I64;
1060     IsSigned = true;
1061     break;
1062   case ICmpInst::ICMP_SGE:
1063     Opc = I32 ? WebAssembly::GE_S_I32 : WebAssembly::GE_S_I64;
1064     IsSigned = true;
1065     break;
1066   case ICmpInst::ICMP_SLT:
1067     Opc = I32 ? WebAssembly::LT_S_I32 : WebAssembly::LT_S_I64;
1068     IsSigned = true;
1069     break;
1070   case ICmpInst::ICMP_SLE:
1071     Opc = I32 ? WebAssembly::LE_S_I32 : WebAssembly::LE_S_I64;
1072     IsSigned = true;
1073     break;
1074   default:
1075     return false;
1076   }
1077
1078   unsigned LHS = getRegForPromotedValue(ICmp->getOperand(0), IsSigned);
1079   if (LHS == 0)
1080     return false;
1081
1082   unsigned RHS = getRegForPromotedValue(ICmp->getOperand(1), IsSigned);
1083   if (RHS == 0)
1084     return false;
1085
1086   Register ResultReg = createResultReg(&WebAssembly::I32RegClass);
1087   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
1088       .addReg(LHS)
1089       .addReg(RHS);
1090   updateValueMap(ICmp, ResultReg);
1091   return true;
1092 }
1093
1094 bool WebAssemblyFastISel::selectFCmp(const Instruction *I) {
1095   const auto *FCmp = cast<FCmpInst>(I);
1096
1097   Register LHS = getRegForValue(FCmp->getOperand(0));
1098   if (LHS == 0)
1099     return false;
1100
1101   Register RHS = getRegForValue(FCmp->getOperand(1));
1102   if (RHS == 0)
1103     return false;
1104
1105   bool F32 = getSimpleType(FCmp->getOperand(0)->getType()) != MVT::f64;
1106   unsigned Opc;
1107   bool Not = false;
1108   switch (FCmp->getPredicate()) {
1109   case FCmpInst::FCMP_OEQ:
1110     Opc = F32 ? WebAssembly::EQ_F32 : WebAssembly::EQ_F64;
1111     break;
1112   case FCmpInst::FCMP_UNE:
1113     Opc = F32 ? WebAssembly::NE_F32 : WebAssembly::NE_F64;
1114     break;
1115   case FCmpInst::FCMP_OGT:
1116     Opc = F32 ? WebAssembly::GT_F32 : WebAssembly::GT_F64;
1117     break;
1118   case FCmpInst::FCMP_OGE:
1119     Opc = F32 ? WebAssembly::GE_F32 : WebAssembly::GE_F64;
1120     break;
1121   case FCmpInst::FCMP_OLT:
1122     Opc = F32 ? WebAssembly::LT_F32 : WebAssembly::LT_F64;
1123     break;
1124   case FCmpInst::FCMP_OLE:
1125     Opc = F32 ? WebAssembly::LE_F32 : WebAssembly::LE_F64;
1126     break;
1127   case FCmpInst::FCMP_UGT:
1128     Opc = F32 ? WebAssembly::LE_F32 : WebAssembly::LE_F64;
1129     Not = true;
1130     break;
1131   case FCmpInst::FCMP_UGE:
1132     Opc = F32 ? WebAssembly::LT_F32 : WebAssembly::LT_F64;
1133     Not = true;
1134     break;
1135   case FCmpInst::FCMP_ULT:
1136     Opc = F32 ? WebAssembly::GE_F32 : WebAssembly::GE_F64;
1137     Not = true;
1138     break;
1139   case FCmpInst::FCMP_ULE:
1140     Opc = F32 ? WebAssembly::GT_F32 : WebAssembly::GT_F64;
1141     Not = true;
1142     break;
1143   default:
1144     return false;
1145   }
1146
1147   Register ResultReg = createResultReg(&WebAssembly::I32RegClass);
1148   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc), ResultReg)
1149       .addReg(LHS)
1150       .addReg(RHS);
1151
1152   if (Not)
1153     ResultReg = notValue(ResultReg);
1154
1155   updateValueMap(FCmp, ResultReg);
1156   return true;
1157 }
1158
1159 bool WebAssemblyFastISel::selectBitCast(const Instruction *I) {
1160   // Target-independent code can handle this, except it doesn't set the dead
1161   // flag on the ARGUMENTS clobber, so we have to do that manually in order
1162   // to satisfy code that expects this of isBitcast() instructions.
1163   EVT VT = TLI.getValueType(DL, I->getOperand(0)->getType());
1164   EVT RetVT = TLI.getValueType(DL, I->getType());
1165   if (!VT.isSimple() || !RetVT.isSimple())
1166     return false;
1167
1168   Register In = getRegForValue(I->getOperand(0));
1169   if (In == 0)
1170     return false;
1171
1172   if (VT == RetVT) {
1173     // No-op bitcast.
1174     updateValueMap(I, In);
1175     return true;
1176   }
1177
1178   Register Reg = fastEmit_ISD_BITCAST_r(VT.getSimpleVT(), RetVT.getSimpleVT(),
1179                                         In);
1180   if (!Reg)
1181     return false;
1182   MachineBasicBlock::iterator Iter = FuncInfo.InsertPt;
1183   --Iter;
1184   assert(Iter->isBitcast());
1185   Iter->setPhysRegsDeadExcept(ArrayRef<Register>(), TRI);
1186   updateValueMap(I, Reg);
1187   return true;
1188 }
1189
1190 bool WebAssemblyFastISel::selectLoad(const Instruction *I) {
1191   const auto *Load = cast<LoadInst>(I);
1192   if (Load->isAtomic())
1193     return false;
1194   if (!WebAssembly::isDefaultAddressSpace(Load->getPointerAddressSpace()))
1195     return false;
1196   if (!Subtarget->hasSIMD128() && Load->getType()->isVectorTy())
1197     return false;
1198
1199   Address Addr;
1200   if (!computeAddress(Load->getPointerOperand(), Addr))
1201     return false;
1202
1203   // TODO: Fold a following sign-/zero-extend into the load instruction.
1204
1205   unsigned Opc;
1206   const TargetRegisterClass *RC;
1207   bool A64 = Subtarget->hasAddr64();
1208   switch (getSimpleType(Load->getType())) {
1209   case MVT::i1:
1210   case MVT::i8:
1211     Opc = A64 ? WebAssembly::LOAD8_U_I32_A64 : WebAssembly::LOAD8_U_I32_A32;
1212     RC = &WebAssembly::I32RegClass;
1213     break;
1214   case MVT::i16:
1215     Opc = A64 ? WebAssembly::LOAD16_U_I32_A64 : WebAssembly::LOAD16_U_I32_A32;
1216     RC = &WebAssembly::I32RegClass;
1217     break;
1218   case MVT::i32:
1219     Opc = A64 ? WebAssembly::LOAD_I32_A64 : WebAssembly::LOAD_I32_A32;
1220     RC = &WebAssembly::I32RegClass;
1221     break;
1222   case MVT::i64:
1223     Opc = A64 ? WebAssembly::LOAD_I64_A64 : WebAssembly::LOAD_I64_A32;
1224     RC = &WebAssembly::I64RegClass;
1225     break;
1226   case MVT::f32:
1227     Opc = A64 ? WebAssembly::LOAD_F32_A64 : WebAssembly::LOAD_F32_A32;
1228     RC = &WebAssembly::F32RegClass;
1229     break;
1230   case MVT::f64:
1231     Opc = A64 ? WebAssembly::LOAD_F64_A64 : WebAssembly::LOAD_F64_A32;
1232     RC = &WebAssembly::F64RegClass;
1233     break;
1234   default:
1235     return false;
1236   }
1237
1238   materializeLoadStoreOperands(Addr);
1239
1240   Register ResultReg = createResultReg(RC);
1241   auto MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc),
1242                      ResultReg);
1243
1244   addLoadStoreOperands(Addr, MIB, createMachineMemOperandFor(Load));
1245
1246   updateValueMap(Load, ResultReg);
1247   return true;
1248 }
1249
1250 bool WebAssemblyFastISel::selectStore(const Instruction *I) {
1251   const auto *Store = cast<StoreInst>(I);
1252   if (Store->isAtomic())
1253     return false;
1254   if (!WebAssembly::isDefaultAddressSpace(Store->getPointerAddressSpace()))
1255     return false;
1256   if (!Subtarget->hasSIMD128() &&
1257       Store->getValueOperand()->getType()->isVectorTy())
1258     return false;
1259
1260   Address Addr;
1261   if (!computeAddress(Store->getPointerOperand(), Addr))
1262     return false;
1263
1264   unsigned Opc;
1265   bool VTIsi1 = false;
1266   bool A64 = Subtarget->hasAddr64();
1267   switch (getSimpleType(Store->getValueOperand()->getType())) {
1268   case MVT::i1:
1269     VTIsi1 = true;
1270     [[fallthrough]];
1271   case MVT::i8:
1272     Opc = A64 ? WebAssembly::STORE8_I32_A64 : WebAssembly::STORE8_I32_A32;
1273     break;
1274   case MVT::i16:
1275     Opc = A64 ? WebAssembly::STORE16_I32_A64 : WebAssembly::STORE16_I32_A32;
1276     break;
1277   case MVT::i32:
1278     Opc = A64 ? WebAssembly::STORE_I32_A64 : WebAssembly::STORE_I32_A32;
1279     break;
1280   case MVT::i64:
1281     Opc = A64 ? WebAssembly::STORE_I64_A64 : WebAssembly::STORE_I64_A32;
1282     break;
1283   case MVT::f32:
1284     Opc = A64 ? WebAssembly::STORE_F32_A64 : WebAssembly::STORE_F32_A32;
1285     break;
1286   case MVT::f64:
1287     Opc = A64 ? WebAssembly::STORE_F64_A64 : WebAssembly::STORE_F64_A32;
1288     break;
1289   default:
1290     return false;
1291   }
1292
1293   materializeLoadStoreOperands(Addr);
1294
1295   Register ValueReg = getRegForValue(Store->getValueOperand());
1296   if (ValueReg == 0)
1297     return false;
1298   if (VTIsi1)
1299     ValueReg = maskI1Value(ValueReg, Store->getValueOperand());
1300
1301   auto MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc));
1302
1303   addLoadStoreOperands(Addr, MIB, createMachineMemOperandFor(Store));
1304
1305   MIB.addReg(ValueReg);
1306   return true;
1307 }
1308
1309 bool WebAssemblyFastISel::selectBr(const Instruction *I) {
1310   const auto *Br = cast<BranchInst>(I);
1311   if (Br->isUnconditional()) {
1312     MachineBasicBlock *MSucc = FuncInfo.MBBMap[Br->getSuccessor(0)];
1313     fastEmitBranch(MSucc, Br->getDebugLoc());
1314     return true;
1315   }
1316
1317   MachineBasicBlock *TBB = FuncInfo.MBBMap[Br->getSuccessor(0)];
1318   MachineBasicBlock *FBB = FuncInfo.MBBMap[Br->getSuccessor(1)];
1319
1320   bool Not;
1321   unsigned CondReg = getRegForI1Value(Br->getCondition(), Br->getParent(), Not);
1322   if (CondReg == 0)
1323     return false;
1324
1325   unsigned Opc = WebAssembly::BR_IF;
1326   if (Not)
1327     Opc = WebAssembly::BR_UNLESS;
1328
1329   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD, TII.get(Opc))
1330       .addMBB(TBB)
1331       .addReg(CondReg);
1332
1333   finishCondBranch(Br->getParent(), TBB, FBB);
1334   return true;
1335 }
1336
1337 bool WebAssemblyFastISel::selectRet(const Instruction *I) {
1338   if (!FuncInfo.CanLowerReturn)
1339     return false;
1340
1341   const auto *Ret = cast<ReturnInst>(I);
1342
1343   if (Ret->getNumOperands() == 0) {
1344     BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1345             TII.get(WebAssembly::RETURN));
1346     return true;
1347   }
1348
1349   // TODO: support multiple return in FastISel
1350   if (Ret->getNumOperands() > 1)
1351     return false;
1352
1353   Value *RV = Ret->getOperand(0);
1354   if (!Subtarget->hasSIMD128() && RV->getType()->isVectorTy())
1355     return false;
1356
1357   switch (getSimpleType(RV->getType())) {
1358   case MVT::i1:
1359   case MVT::i8:
1360   case MVT::i16:
1361   case MVT::i32:
1362   case MVT::i64:
1363   case MVT::f32:
1364   case MVT::f64:
1365   case MVT::v16i8:
1366   case MVT::v8i16:
1367   case MVT::v4i32:
1368   case MVT::v2i64:
1369   case MVT::v4f32:
1370   case MVT::v2f64:
1371   case MVT::funcref:
1372   case MVT::externref:
1373   case MVT::exnref:
1374     break;
1375   default:
1376     return false;
1377   }
1378
1379   unsigned Reg;
1380   if (FuncInfo.Fn->getAttributes().hasRetAttr(Attribute::SExt))
1381     Reg = getRegForSignedValue(RV);
1382   else if (FuncInfo.Fn->getAttributes().hasRetAttr(Attribute::ZExt))
1383     Reg = getRegForUnsignedValue(RV);
1384   else
1385     Reg = getRegForValue(RV);
1386
1387   if (Reg == 0)
1388     return false;
1389
1390   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1391           TII.get(WebAssembly::RETURN))
1392       .addReg(Reg);
1393   return true;
1394 }
1395
1396 bool WebAssemblyFastISel::selectUnreachable(const Instruction *I) {
1397   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, MIMD,
1398           TII.get(WebAssembly::UNREACHABLE));
1399   return true;
1400 }
1401
1402 bool WebAssemblyFastISel::fastSelectInstruction(const Instruction *I) {
1403   switch (I->getOpcode()) {
1404   case Instruction::Call:
1405     if (selectCall(I))
1406       return true;
1407     break;
1408   case Instruction::Select:
1409     return selectSelect(I);
1410   case Instruction::Trunc:
1411     return selectTrunc(I);
1412   case Instruction::ZExt:
1413     return selectZExt(I);
1414   case Instruction::SExt:
1415     return selectSExt(I);
1416   case Instruction::ICmp:
1417     return selectICmp(I);
1418   case Instruction::FCmp:
1419     return selectFCmp(I);
1420   case Instruction::BitCast:
1421     return selectBitCast(I);
1422   case Instruction::Load:
1423     return selectLoad(I);
1424   case Instruction::Store:
1425     return selectStore(I);
1426   case Instruction::Br:
1427     return selectBr(I);
1428   case Instruction::Ret:
1429     return selectRet(I);
1430   case Instruction::Unreachable:
1431     return selectUnreachable(I);
1432   default:
1433     break;
1434   }
1435
1436   // Fall back to target-independent instruction selection.
1437   return selectOperator(I, I->getOpcode());
1438 }
1439
1440 FastISel *WebAssembly::createFastISel(FunctionLoweringInfo &FuncInfo,
1441                                       const TargetLibraryInfo *LibInfo) {
1442   return new WebAssemblyFastISel(FuncInfo, LibInfo);
1443 }