lib/CodeGen/GlobalISel/MachineIRBuilder.cpp

   1 //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 /// This file implements the MachineIRBuidler class.
  10 //===----------------------------------------------------------------------===//
  11 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
  12 #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
  13
  14 #include "llvm/CodeGen/MachineFunction.h"
  15 #include "llvm/CodeGen/MachineInstr.h"
  16 #include "llvm/CodeGen/MachineInstrBuilder.h"
  17 #include "llvm/CodeGen/MachineRegisterInfo.h"
  18 #include "llvm/CodeGen/TargetInstrInfo.h"
  19 #include "llvm/CodeGen/TargetLowering.h"
  20 #include "llvm/CodeGen/TargetOpcodes.h"
  21 #include "llvm/CodeGen/TargetSubtargetInfo.h"
  22 #include "llvm/IR/DebugInfo.h"
  23
  24 using namespace llvm;
  25
  26 void MachineIRBuilder::setMF(MachineFunction &MF) {
  27   State.MF = &MF;
  28   State.MBB = nullptr;
  29   State.MRI = &MF.getRegInfo();
  30   State.TII = MF.getSubtarget().getInstrInfo();
  31   State.DL = DebugLoc();
  32   State.II = MachineBasicBlock::iterator();
  33   State.Observer = nullptr;
  34 }
  35
  36 void MachineIRBuilder::setMBB(MachineBasicBlock &MBB) {
  37   State.MBB = &MBB;
  38   State.II = MBB.end();
  39   assert(&getMF() == MBB.getParent() &&
  40          "Basic block is in a different function");
  41 }
  42
  43 void MachineIRBuilder::setInstr(MachineInstr &MI) {
  44   assert(MI.getParent() && "Instruction is not part of a basic block");
  45   setMBB(*MI.getParent());
  46   State.II = MI.getIterator();
  47 }
  48
  49 void MachineIRBuilder::setCSEInfo(GISelCSEInfo *Info) { State.CSEInfo = Info; }
  50
  51 void MachineIRBuilder::setInsertPt(MachineBasicBlock &MBB,
  52                                    MachineBasicBlock::iterator II) {
  53   assert(MBB.getParent() == &getMF() &&
  54          "Basic block is in a different function");
  55   State.MBB = &MBB;
  56   State.II = II;
  57 }
  58
  59 void MachineIRBuilder::recordInsertion(MachineInstr *InsertedInstr) const {
  60   if (State.Observer)
  61     State.Observer->createdInstr(*InsertedInstr);
  62 }
  63
  64 void MachineIRBuilder::setChangeObserver(GISelChangeObserver &Observer) {
  65   State.Observer = &Observer;
  66 }
  67
  68 void MachineIRBuilder::stopObservingChanges() { State.Observer = nullptr; }
  69
  70 //------------------------------------------------------------------------------
  71 // Build instruction variants.
  72 //------------------------------------------------------------------------------
  73
  74 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opcode) {
  75   return insertInstr(buildInstrNoInsert(Opcode));
  76 }
  77
  78 MachineInstrBuilder MachineIRBuilder::buildInstrNoInsert(unsigned Opcode) {
  79   MachineInstrBuilder MIB = BuildMI(getMF(), getDL(), getTII().get(Opcode));
  80   return MIB;
  81 }
  82
  83 MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) {
  84   getMBB().insert(getInsertPt(), MIB);
  85   recordInsertion(MIB);
  86   return MIB;
  87 }
  88
  89 MachineInstrBuilder
  90 MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable,
  91                                       const MDNode *Expr) {
  92   assert(isa<DILocalVariable>(Variable) && "not a variable");
  93   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
  94   assert(
  95       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
  96       "Expected inlined-at fields to agree");
  97   return insertInstr(BuildMI(getMF(), getDL(),
  98                              getTII().get(TargetOpcode::DBG_VALUE),
  99                              /*IsIndirect*/ false, Reg, Variable, Expr));
 100 }
 101
 102 MachineInstrBuilder
 103 MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable,
 104                                         const MDNode *Expr) {
 105   assert(isa<DILocalVariable>(Variable) && "not a variable");
 106   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
 107   assert(
 108       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
 109       "Expected inlined-at fields to agree");
 110   // DBG_VALUE insts now carry IR-level indirection in their DIExpression
 111   // rather than encoding it in the instruction itself.
 112   const DIExpression *DIExpr = cast<DIExpression>(Expr);
 113   DIExpr = DIExpression::append(DIExpr, {dwarf::DW_OP_deref});
 114   return insertInstr(BuildMI(getMF(), getDL(),
 115                              getTII().get(TargetOpcode::DBG_VALUE),
 116                              /*IsIndirect*/ false, Reg, Variable, DIExpr));
 117 }
 118
 119 MachineInstrBuilder MachineIRBuilder::buildFIDbgValue(int FI,
 120                                                       const MDNode *Variable,
 121                                                       const MDNode *Expr) {
 122   assert(isa<DILocalVariable>(Variable) && "not a variable");
 123   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
 124   assert(
 125       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
 126       "Expected inlined-at fields to agree");
 127   // DBG_VALUE insts now carry IR-level indirection in their DIExpression
 128   // rather than encoding it in the instruction itself.
 129   const DIExpression *DIExpr = cast<DIExpression>(Expr);
 130   DIExpr = DIExpression::append(DIExpr, {dwarf::DW_OP_deref});
 131   return buildInstr(TargetOpcode::DBG_VALUE)
 132       .addFrameIndex(FI)
 133       .addReg(0)
 134       .addMetadata(Variable)
 135       .addMetadata(DIExpr);
 136 }
 137
 138 MachineInstrBuilder MachineIRBuilder::buildConstDbgValue(const Constant &C,
 139                                                          const MDNode *Variable,
 140                                                          const MDNode *Expr) {
 141   assert(isa<DILocalVariable>(Variable) && "not a variable");
 142   assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
 143   assert(
 144       cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(getDL()) &&
 145       "Expected inlined-at fields to agree");
 146   auto MIB = buildInstr(TargetOpcode::DBG_VALUE);
 147   if (auto *CI = dyn_cast<ConstantInt>(&C)) {
 148     if (CI->getBitWidth() > 64)
 149       MIB.addCImm(CI);
 150     else
 151       MIB.addImm(CI->getZExtValue());
 152   } else if (auto *CFP = dyn_cast<ConstantFP>(&C)) {
 153     MIB.addFPImm(CFP);
 154   } else {
 155     // Insert %noreg if we didn't find a usable constant and had to drop it.
 156     MIB.addReg(0U);
 157   }
 158
 159   return MIB.addReg(0).addMetadata(Variable).addMetadata(Expr);
 160 }
 161
 162 MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) {
 163   assert(isa<DILabel>(Label) && "not a label");
 164   assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(State.DL) &&
 165          "Expected inlined-at fields to agree");
 166   auto MIB = buildInstr(TargetOpcode::DBG_LABEL);
 167
 168   return MIB.addMetadata(Label);
 169 }
 170
 171 MachineInstrBuilder MachineIRBuilder::buildDynStackAlloc(const DstOp &Res,
 172                                                          const SrcOp &Size,
 173                                                          unsigned Align) {
 174   assert(Res.getLLTTy(*getMRI()).isPointer() && "expected ptr dst type");
 175   auto MIB = buildInstr(TargetOpcode::G_DYN_STACKALLOC);
 176   Res.addDefToMIB(*getMRI(), MIB);
 177   Size.addSrcToMIB(MIB);
 178   MIB.addImm(Align);
 179   return MIB;
 180 }
 181
 182 MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res,
 183                                                       int Idx) {
 184   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
 185   auto MIB = buildInstr(TargetOpcode::G_FRAME_INDEX);
 186   Res.addDefToMIB(*getMRI(), MIB);
 187   MIB.addFrameIndex(Idx);
 188   return MIB;
 189 }
 190
 191 MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res,
 192                                                        const GlobalValue *GV) {
 193   assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
 194   assert(Res.getLLTTy(*getMRI()).getAddressSpace() ==
 195              GV->getType()->getAddressSpace() &&
 196          "address space mismatch");
 197
 198   auto MIB = buildInstr(TargetOpcode::G_GLOBAL_VALUE);
 199   Res.addDefToMIB(*getMRI(), MIB);
 200   MIB.addGlobalAddress(GV);
 201   return MIB;
 202 }
 203
 204 MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy,
 205                                                      unsigned JTI) {
 206   return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {})
 207       .addJumpTableIndex(JTI);
 208 }
 209
 210 void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0,
 211                                         const LLT &Op1) {
 212   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
 213   assert((Res == Op0 && Res == Op1) && "type mismatch");
 214 }
 215
 216 void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0,
 217                                        const LLT &Op1) {
 218   assert((Res.isScalar() || Res.isVector()) && "invalid operand type");
 219   assert((Res == Op0) && "type mismatch");
 220 }
 221
 222 MachineInstrBuilder MachineIRBuilder::buildGEP(const DstOp &Res,
 223                                                const SrcOp &Op0,
 224                                                const SrcOp &Op1) {
 225   assert(Res.getLLTTy(*getMRI()).isPointer() &&
 226          Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
 227   assert(Op1.getLLTTy(*getMRI()).isScalar() && "invalid offset type");
 228
 229   return buildInstr(TargetOpcode::G_GEP, {Res}, {Op0, Op1});
 230 }
 231
 232 Optional<MachineInstrBuilder>
 233 MachineIRBuilder::materializeGEP(Register &Res, Register Op0,
 234                                  const LLT &ValueTy, uint64_t Value) {
 235   assert(Res == 0 && "Res is a result argument");
 236   assert(ValueTy.isScalar()  && "invalid offset type");
 237
 238   if (Value == 0) {
 239     Res = Op0;
 240     return None;
 241   }
 242
 243   Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0));
 244   auto Cst = buildConstant(ValueTy, Value);
 245   return buildGEP(Res, Op0, Cst.getReg(0));
 246 }
 247
 248 MachineInstrBuilder MachineIRBuilder::buildPtrMask(const DstOp &Res,
 249                                                    const SrcOp &Op0,
 250                                                    uint32_t NumBits) {
 251   assert(Res.getLLTTy(*getMRI()).isPointer() &&
 252          Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch");
 253
 254   auto MIB = buildInstr(TargetOpcode::G_PTR_MASK);
 255   Res.addDefToMIB(*getMRI(), MIB);
 256   Op0.addSrcToMIB(MIB);
 257   MIB.addImm(NumBits);
 258   return MIB;
 259 }
 260
 261 MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) {
 262   return buildInstr(TargetOpcode::G_BR).addMBB(&Dest);
 263 }
 264
 265 MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) {
 266   assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination");
 267   return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt);
 268 }
 269
 270 MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr,
 271                                                 unsigned JTI,
 272                                                 Register IndexReg) {
 273   assert(getMRI()->getType(TablePtr).isPointer() &&
 274          "Table reg must be a pointer");
 275   return buildInstr(TargetOpcode::G_BRJT)
 276       .addUse(TablePtr)
 277       .addJumpTableIndex(JTI)
 278       .addUse(IndexReg);
 279 }
 280
 281 MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res,
 282                                                 const SrcOp &Op) {
 283   return buildInstr(TargetOpcode::COPY, Res, Op);
 284 }
 285
 286 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
 287                                                     const ConstantInt &Val) {
 288   LLT Ty = Res.getLLTTy(*getMRI());
 289   LLT EltTy = Ty.getScalarType();
 290   assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
 291          "creating constant with the wrong size");
 292
 293   if (Ty.isVector()) {
 294     auto Const = buildInstr(TargetOpcode::G_CONSTANT)
 295     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
 296     .addCImm(&Val);
 297     return buildSplatVector(Res, Const);
 298   }
 299
 300   auto Const = buildInstr(TargetOpcode::G_CONSTANT);
 301   Res.addDefToMIB(*getMRI(), Const);
 302   Const.addCImm(&Val);
 303   return Const;
 304 }
 305
 306 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
 307                                                     int64_t Val) {
 308   auto IntN = IntegerType::get(getMF().getFunction().getContext(),
 309                                Res.getLLTTy(*getMRI()).getScalarSizeInBits());
 310   ConstantInt *CI = ConstantInt::get(IntN, Val, true);
 311   return buildConstant(Res, *CI);
 312 }
 313
 314 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
 315                                                      const ConstantFP &Val) {
 316   LLT Ty = Res.getLLTTy(*getMRI());
 317   LLT EltTy = Ty.getScalarType();
 318
 319   assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics())
 320          == EltTy.getSizeInBits() &&
 321          "creating fconstant with the wrong size");
 322
 323   assert(!Ty.isPointer() && "invalid operand type");
 324
 325   if (Ty.isVector()) {
 326     auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
 327     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
 328     .addFPImm(&Val);
 329
 330     return buildSplatVector(Res, Const);
 331   }
 332
 333   auto Const = buildInstr(TargetOpcode::G_FCONSTANT);
 334   Res.addDefToMIB(*getMRI(), Const);
 335   Const.addFPImm(&Val);
 336   return Const;
 337 }
 338
 339 MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
 340                                                     const APInt &Val) {
 341   ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val);
 342   return buildConstant(Res, *CI);
 343 }
 344
 345 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
 346                                                      double Val) {
 347   LLT DstTy = Res.getLLTTy(*getMRI());
 348   auto &Ctx = getMF().getFunction().getContext();
 349   auto *CFP =
 350       ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits()));
 351   return buildFConstant(Res, *CFP);
 352 }
 353
 354 MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
 355                                                      const APFloat &Val) {
 356   auto &Ctx = getMF().getFunction().getContext();
 357   auto *CFP = ConstantFP::get(Ctx, Val);
 358   return buildFConstant(Res, *CFP);
 359 }
 360
 361 MachineInstrBuilder MachineIRBuilder::buildBrCond(Register Tst,
 362                                                   MachineBasicBlock &Dest) {
 363   assert(getMRI()->getType(Tst).isScalar() && "invalid operand type");
 364
 365   return buildInstr(TargetOpcode::G_BRCOND).addUse(Tst).addMBB(&Dest);
 366 }
 367
 368 MachineInstrBuilder MachineIRBuilder::buildLoad(const DstOp &Res,
 369                                                 const SrcOp &Addr,
 370                                                 MachineMemOperand &MMO) {
 371   return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO);
 372 }
 373
 374 MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode,
 375                                                      const DstOp &Res,
 376                                                      const SrcOp &Addr,
 377                                                      MachineMemOperand &MMO) {
 378   assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type");
 379   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
 380
 381   auto MIB = buildInstr(Opcode);
 382   Res.addDefToMIB(*getMRI(), MIB);
 383   Addr.addSrcToMIB(MIB);
 384   MIB.addMemOperand(&MMO);
 385   return MIB;
 386 }
 387
 388 MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val,
 389                                                  const SrcOp &Addr,
 390                                                  MachineMemOperand &MMO) {
 391   assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type");
 392   assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type");
 393
 394   auto MIB = buildInstr(TargetOpcode::G_STORE);
 395   Val.addSrcToMIB(MIB);
 396   Addr.addSrcToMIB(MIB);
 397   MIB.addMemOperand(&MMO);
 398   return MIB;
 399 }
 400
 401 MachineInstrBuilder MachineIRBuilder::buildUAddo(const DstOp &Res,
 402                                                  const DstOp &CarryOut,
 403                                                  const SrcOp &Op0,
 404                                                  const SrcOp &Op1) {
 405   return buildInstr(TargetOpcode::G_UADDO, {Res, CarryOut}, {Op0, Op1});
 406 }
 407
 408 MachineInstrBuilder MachineIRBuilder::buildUAdde(const DstOp &Res,
 409                                                  const DstOp &CarryOut,
 410                                                  const SrcOp &Op0,
 411                                                  const SrcOp &Op1,
 412                                                  const SrcOp &CarryIn) {
 413   return buildInstr(TargetOpcode::G_UADDE, {Res, CarryOut},
 414                     {Op0, Op1, CarryIn});
 415 }
 416
 417 MachineInstrBuilder MachineIRBuilder::buildAnyExt(const DstOp &Res,
 418                                                   const SrcOp &Op) {
 419   return buildInstr(TargetOpcode::G_ANYEXT, Res, Op);
 420 }
 421
 422 MachineInstrBuilder MachineIRBuilder::buildSExt(const DstOp &Res,
 423                                                 const SrcOp &Op) {
 424   return buildInstr(TargetOpcode::G_SEXT, Res, Op);
 425 }
 426
 427 MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res,
 428                                                 const SrcOp &Op) {
 429   return buildInstr(TargetOpcode::G_ZEXT, Res, Op);
 430 }
 431
 432 unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const {
 433   const auto *TLI = getMF().getSubtarget().getTargetLowering();
 434   switch (TLI->getBooleanContents(IsVec, IsFP)) {
 435   case TargetLoweringBase::ZeroOrNegativeOneBooleanContent:
 436     return TargetOpcode::G_SEXT;
 437   case TargetLoweringBase::ZeroOrOneBooleanContent:
 438     return TargetOpcode::G_ZEXT;
 439   default:
 440     return TargetOpcode::G_ANYEXT;
 441   }
 442 }
 443
 444 MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res,
 445                                                    const SrcOp &Op,
 446                                                    bool IsFP) {
 447   unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP);
 448   return buildInstr(ExtOp, Res, Op);
 449 }
 450
 451 MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc,
 452                                                       const DstOp &Res,
 453                                                       const SrcOp &Op) {
 454   assert((TargetOpcode::G_ANYEXT == ExtOpc || TargetOpcode::G_ZEXT == ExtOpc ||
 455           TargetOpcode::G_SEXT == ExtOpc) &&
 456          "Expecting Extending Opc");
 457   assert(Res.getLLTTy(*getMRI()).isScalar() ||
 458          Res.getLLTTy(*getMRI()).isVector());
 459   assert(Res.getLLTTy(*getMRI()).isScalar() ==
 460          Op.getLLTTy(*getMRI()).isScalar());
 461
 462   unsigned Opcode = TargetOpcode::COPY;
 463   if (Res.getLLTTy(*getMRI()).getSizeInBits() >
 464       Op.getLLTTy(*getMRI()).getSizeInBits())
 465     Opcode = ExtOpc;
 466   else if (Res.getLLTTy(*getMRI()).getSizeInBits() <
 467            Op.getLLTTy(*getMRI()).getSizeInBits())
 468     Opcode = TargetOpcode::G_TRUNC;
 469   else
 470     assert(Res.getLLTTy(*getMRI()) == Op.getLLTTy(*getMRI()));
 471
 472   return buildInstr(Opcode, Res, Op);
 473 }
 474
 475 MachineInstrBuilder MachineIRBuilder::buildSExtOrTrunc(const DstOp &Res,
 476                                                        const SrcOp &Op) {
 477   return buildExtOrTrunc(TargetOpcode::G_SEXT, Res, Op);
 478 }
 479
 480 MachineInstrBuilder MachineIRBuilder::buildZExtOrTrunc(const DstOp &Res,
 481                                                        const SrcOp &Op) {
 482   return buildExtOrTrunc(TargetOpcode::G_ZEXT, Res, Op);
 483 }
 484
 485 MachineInstrBuilder MachineIRBuilder::buildAnyExtOrTrunc(const DstOp &Res,
 486                                                          const SrcOp &Op) {
 487   return buildExtOrTrunc(TargetOpcode::G_ANYEXT, Res, Op);
 488 }
 489
 490 MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst,
 491                                                 const SrcOp &Src) {
 492   LLT SrcTy = Src.getLLTTy(*getMRI());
 493   LLT DstTy = Dst.getLLTTy(*getMRI());
 494   if (SrcTy == DstTy)
 495     return buildCopy(Dst, Src);
 496
 497   unsigned Opcode;
 498   if (SrcTy.isPointer() && DstTy.isScalar())
 499     Opcode = TargetOpcode::G_PTRTOINT;
 500   else if (DstTy.isPointer() && SrcTy.isScalar())
 501     Opcode = TargetOpcode::G_INTTOPTR;
 502   else {
 503     assert(!SrcTy.isPointer() && !DstTy.isPointer() && "n G_ADDRCAST yet");
 504     Opcode = TargetOpcode::G_BITCAST;
 505   }
 506
 507   return buildInstr(Opcode, Dst, Src);
 508 }
 509
 510 MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst,
 511                                                    const SrcOp &Src,
 512                                                    uint64_t Index) {
 513   LLT SrcTy = Src.getLLTTy(*getMRI());
 514   LLT DstTy = Dst.getLLTTy(*getMRI());
 515
 516 #ifndef NDEBUG
 517   assert(SrcTy.isValid() && "invalid operand type");
 518   assert(DstTy.isValid() && "invalid operand type");
 519   assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() &&
 520          "extracting off end of register");
 521 #endif
 522
 523   if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) {
 524     assert(Index == 0 && "insertion past the end of a register");
 525     return buildCast(Dst, Src);
 526   }
 527
 528   auto Extract = buildInstr(TargetOpcode::G_EXTRACT);
 529   Dst.addDefToMIB(*getMRI(), Extract);
 530   Src.addSrcToMIB(Extract);
 531   Extract.addImm(Index);
 532   return Extract;
 533 }
 534
 535 void MachineIRBuilder::buildSequence(Register Res, ArrayRef<Register> Ops,
 536                                      ArrayRef<uint64_t> Indices) {
 537 #ifndef NDEBUG
 538   assert(Ops.size() == Indices.size() && "incompatible args");
 539   assert(!Ops.empty() && "invalid trivial sequence");
 540   assert(std::is_sorted(Indices.begin(), Indices.end()) &&
 541          "sequence offsets must be in ascending order");
 542
 543   assert(getMRI()->getType(Res).isValid() && "invalid operand type");
 544   for (auto Op : Ops)
 545     assert(getMRI()->getType(Op).isValid() && "invalid operand type");
 546 #endif
 547
 548   LLT ResTy = getMRI()->getType(Res);
 549   LLT OpTy = getMRI()->getType(Ops[0]);
 550   unsigned OpSize = OpTy.getSizeInBits();
 551   bool MaybeMerge = true;
 552   for (unsigned i = 0; i < Ops.size(); ++i) {
 553     if (getMRI()->getType(Ops[i]) != OpTy || Indices[i] != i * OpSize) {
 554       MaybeMerge = false;
 555       break;
 556     }
 557   }
 558
 559   if (MaybeMerge && Ops.size() * OpSize == ResTy.getSizeInBits()) {
 560     buildMerge(Res, Ops);
 561     return;
 562   }
 563
 564   Register ResIn = getMRI()->createGenericVirtualRegister(ResTy);
 565   buildUndef(ResIn);
 566
 567   for (unsigned i = 0; i < Ops.size(); ++i) {
 568     Register ResOut = i + 1 == Ops.size()
 569                           ? Res
 570                           : getMRI()->createGenericVirtualRegister(ResTy);
 571     buildInsert(ResOut, ResIn, Ops[i], Indices[i]);
 572     ResIn = ResOut;
 573   }
 574 }
 575
 576 MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) {
 577   return buildInstr(TargetOpcode::G_IMPLICIT_DEF, {Res}, {});
 578 }
 579
 580 MachineInstrBuilder MachineIRBuilder::buildMerge(const DstOp &Res,
 581                                                  ArrayRef<Register> Ops) {
 582   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<SrcOp>,
 583   // we need some temporary storage for the DstOp objects. Here we use a
 584   // sufficiently large SmallVector to not go through the heap.
 585   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
 586   assert(TmpVec.size() > 1);
 587   return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec);
 588 }
 589
 590 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<LLT> Res,
 591                                                    const SrcOp &Op) {
 592   // Unfortunately to convert from ArrayRef<LLT> to ArrayRef<DstOp>,
 593   // we need some temporary storage for the DstOp objects. Here we use a
 594   // sufficiently large SmallVector to not go through the heap.
 595   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
 596   assert(TmpVec.size() > 1);
 597   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
 598 }
 599
 600 MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res,
 601                                                    const SrcOp &Op) {
 602   unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits();
 603   SmallVector<Register, 8> TmpVec;
 604   for (unsigned I = 0; I != NumReg; ++I)
 605     TmpVec.push_back(getMRI()->createGenericVirtualRegister(Res));
 606   return buildUnmerge(TmpVec, Op);
 607 }
 608
 609 MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef<Register> Res,
 610                                                    const SrcOp &Op) {
 611   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<DstOp>,
 612   // we need some temporary storage for the DstOp objects. Here we use a
 613   // sufficiently large SmallVector to not go through the heap.
 614   SmallVector<DstOp, 8> TmpVec(Res.begin(), Res.end());
 615   assert(TmpVec.size() > 1);
 616   return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op);
 617 }
 618
 619 MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res,
 620                                                        ArrayRef<Register> Ops) {
 621   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
 622   // we need some temporary storage for the DstOp objects. Here we use a
 623   // sufficiently large SmallVector to not go through the heap.
 624   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
 625   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
 626 }
 627
 628 MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res,
 629                                                        const SrcOp &Src) {
 630   SmallVector<SrcOp, 8> TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src);
 631   return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec);
 632 }
 633
 634 MachineInstrBuilder
 635 MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res,
 636                                         ArrayRef<Register> Ops) {
 637   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
 638   // we need some temporary storage for the DstOp objects. Here we use a
 639   // sufficiently large SmallVector to not go through the heap.
 640   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
 641   return buildInstr(TargetOpcode::G_BUILD_VECTOR_TRUNC, Res, TmpVec);
 642 }
 643
 644 MachineInstrBuilder
 645 MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef<Register> Ops) {
 646   // Unfortunately to convert from ArrayRef<Register> to ArrayRef<SrcOp>,
 647   // we need some temporary storage for the DstOp objects. Here we use a
 648   // sufficiently large SmallVector to not go through the heap.
 649   SmallVector<SrcOp, 8> TmpVec(Ops.begin(), Ops.end());
 650   return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec);
 651 }
 652
 653 MachineInstrBuilder MachineIRBuilder::buildInsert(Register Res, Register Src,
 654                                                   Register Op, unsigned Index) {
 655   assert(Index + getMRI()->getType(Op).getSizeInBits() <=
 656              getMRI()->getType(Res).getSizeInBits() &&
 657          "insertion past the end of a register");
 658
 659   if (getMRI()->getType(Res).getSizeInBits() ==
 660       getMRI()->getType(Op).getSizeInBits()) {
 661     return buildCast(Res, Op);
 662   }
 663
 664   return buildInstr(TargetOpcode::G_INSERT)
 665       .addDef(Res)
 666       .addUse(Src)
 667       .addUse(Op)
 668       .addImm(Index);
 669 }
 670
 671 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
 672                                                      ArrayRef<Register> ResultRegs,
 673                                                      bool HasSideEffects) {
 674   auto MIB =
 675       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
 676                                 : TargetOpcode::G_INTRINSIC);
 677   for (unsigned ResultReg : ResultRegs)
 678     MIB.addDef(ResultReg);
 679   MIB.addIntrinsicID(ID);
 680   return MIB;
 681 }
 682
 683 MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID,
 684                                                      ArrayRef<DstOp> Results,
 685                                                      bool HasSideEffects) {
 686   auto MIB =
 687       buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS
 688                                 : TargetOpcode::G_INTRINSIC);
 689   for (DstOp Result : Results)
 690     Result.addDefToMIB(*getMRI(), MIB);
 691   MIB.addIntrinsicID(ID);
 692   return MIB;
 693 }
 694
 695 MachineInstrBuilder MachineIRBuilder::buildTrunc(const DstOp &Res,
 696                                                  const SrcOp &Op) {
 697   return buildInstr(TargetOpcode::G_TRUNC, Res, Op);
 698 }
 699
 700 MachineInstrBuilder MachineIRBuilder::buildFPTrunc(const DstOp &Res,
 701                                                    const SrcOp &Op) {
 702   return buildInstr(TargetOpcode::G_FPTRUNC, Res, Op);
 703 }
 704
 705 MachineInstrBuilder MachineIRBuilder::buildICmp(CmpInst::Predicate Pred,
 706                                                 const DstOp &Res,
 707                                                 const SrcOp &Op0,
 708                                                 const SrcOp &Op1) {
 709   return buildInstr(TargetOpcode::G_ICMP, Res, {Pred, Op0, Op1});
 710 }
 711
 712 MachineInstrBuilder MachineIRBuilder::buildFCmp(CmpInst::Predicate Pred,
 713                                                 const DstOp &Res,
 714                                                 const SrcOp &Op0,
 715                                                 const SrcOp &Op1,
 716                                                 Optional<unsigned> Flags) {
 717
 718   return buildInstr(TargetOpcode::G_FCMP, Res, {Pred, Op0, Op1}, Flags);
 719 }
 720
 721 MachineInstrBuilder MachineIRBuilder::buildSelect(const DstOp &Res,
 722                                                   const SrcOp &Tst,
 723                                                   const SrcOp &Op0,
 724                                                   const SrcOp &Op1,
 725                                                   Optional<unsigned> Flags) {
 726
 727   return buildInstr(TargetOpcode::G_SELECT, {Res}, {Tst, Op0, Op1}, Flags);
 728 }
 729
 730 MachineInstrBuilder
 731 MachineIRBuilder::buildInsertVectorElement(const DstOp &Res, const SrcOp &Val,
 732                                            const SrcOp &Elt, const SrcOp &Idx) {
 733   return buildInstr(TargetOpcode::G_INSERT_VECTOR_ELT, Res, {Val, Elt, Idx});
 734 }
 735
 736 MachineInstrBuilder
 737 MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val,
 738                                             const SrcOp &Idx) {
 739   return buildInstr(TargetOpcode::G_EXTRACT_VECTOR_ELT, Res, {Val, Idx});
 740 }
 741
 742 MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess(
 743     Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal,
 744     Register NewVal, MachineMemOperand &MMO) {
 745 #ifndef NDEBUG
 746   LLT OldValResTy = getMRI()->getType(OldValRes);
 747   LLT SuccessResTy = getMRI()->getType(SuccessRes);
 748   LLT AddrTy = getMRI()->getType(Addr);
 749   LLT CmpValTy = getMRI()->getType(CmpVal);
 750   LLT NewValTy = getMRI()->getType(NewVal);
 751   assert(OldValResTy.isScalar() && "invalid operand type");
 752   assert(SuccessResTy.isScalar() && "invalid operand type");
 753   assert(AddrTy.isPointer() && "invalid operand type");
 754   assert(CmpValTy.isValid() && "invalid operand type");
 755   assert(NewValTy.isValid() && "invalid operand type");
 756   assert(OldValResTy == CmpValTy && "type mismatch");
 757   assert(OldValResTy == NewValTy && "type mismatch");
 758 #endif
 759
 760   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS)
 761       .addDef(OldValRes)
 762       .addDef(SuccessRes)
 763       .addUse(Addr)
 764       .addUse(CmpVal)
 765       .addUse(NewVal)
 766       .addMemOperand(&MMO);
 767 }
 768
 769 MachineInstrBuilder
 770 MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr,
 771                                      Register CmpVal, Register NewVal,
 772                                      MachineMemOperand &MMO) {
 773 #ifndef NDEBUG
 774   LLT OldValResTy = getMRI()->getType(OldValRes);
 775   LLT AddrTy = getMRI()->getType(Addr);
 776   LLT CmpValTy = getMRI()->getType(CmpVal);
 777   LLT NewValTy = getMRI()->getType(NewVal);
 778   assert(OldValResTy.isScalar() && "invalid operand type");
 779   assert(AddrTy.isPointer() && "invalid operand type");
 780   assert(CmpValTy.isValid() && "invalid operand type");
 781   assert(NewValTy.isValid() && "invalid operand type");
 782   assert(OldValResTy == CmpValTy && "type mismatch");
 783   assert(OldValResTy == NewValTy && "type mismatch");
 784 #endif
 785
 786   return buildInstr(TargetOpcode::G_ATOMIC_CMPXCHG)
 787       .addDef(OldValRes)
 788       .addUse(Addr)
 789       .addUse(CmpVal)
 790       .addUse(NewVal)
 791       .addMemOperand(&MMO);
 792 }
 793
 794 MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(
 795   unsigned Opcode, const DstOp &OldValRes,
 796   const SrcOp &Addr, const SrcOp &Val,
 797   MachineMemOperand &MMO) {
 798
 799 #ifndef NDEBUG
 800   LLT OldValResTy = OldValRes.getLLTTy(*getMRI());
 801   LLT AddrTy = Addr.getLLTTy(*getMRI());
 802   LLT ValTy = Val.getLLTTy(*getMRI());
 803   assert(OldValResTy.isScalar() && "invalid operand type");
 804   assert(AddrTy.isPointer() && "invalid operand type");
 805   assert(ValTy.isValid() && "invalid operand type");
 806   assert(OldValResTy == ValTy && "type mismatch");
 807   assert(MMO.isAtomic() && "not atomic mem operand");
 808 #endif
 809
 810   auto MIB = buildInstr(Opcode);
 811   OldValRes.addDefToMIB(*getMRI(), MIB);
 812   Addr.addSrcToMIB(MIB);
 813   Val.addSrcToMIB(MIB);
 814   MIB.addMemOperand(&MMO);
 815   return MIB;
 816 }
 817
 818 MachineInstrBuilder
 819 MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr,
 820                                      Register Val, MachineMemOperand &MMO) {
 821   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val,
 822                         MMO);
 823 }
 824 MachineInstrBuilder
 825 MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr,
 826                                     Register Val, MachineMemOperand &MMO) {
 827   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val,
 828                         MMO);
 829 }
 830 MachineInstrBuilder
 831 MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr,
 832                                     Register Val, MachineMemOperand &MMO) {
 833   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val,
 834                         MMO);
 835 }
 836 MachineInstrBuilder
 837 MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr,
 838                                     Register Val, MachineMemOperand &MMO) {
 839   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val,
 840                         MMO);
 841 }
 842 MachineInstrBuilder
 843 MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr,
 844                                      Register Val, MachineMemOperand &MMO) {
 845   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val,
 846                         MMO);
 847 }
 848 MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes,
 849                                                        Register Addr,
 850                                                        Register Val,
 851                                                        MachineMemOperand &MMO) {
 852   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val,
 853                         MMO);
 854 }
 855 MachineInstrBuilder
 856 MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr,
 857                                     Register Val, MachineMemOperand &MMO) {
 858   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val,
 859                         MMO);
 860 }
 861 MachineInstrBuilder
 862 MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr,
 863                                     Register Val, MachineMemOperand &MMO) {
 864   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val,
 865                         MMO);
 866 }
 867 MachineInstrBuilder
 868 MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr,
 869                                     Register Val, MachineMemOperand &MMO) {
 870   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val,
 871                         MMO);
 872 }
 873 MachineInstrBuilder
 874 MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr,
 875                                      Register Val, MachineMemOperand &MMO) {
 876   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val,
 877                         MMO);
 878 }
 879 MachineInstrBuilder
 880 MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr,
 881                                      Register Val, MachineMemOperand &MMO) {
 882   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val,
 883                         MMO);
 884 }
 885
 886 MachineInstrBuilder
 887 MachineIRBuilder::buildAtomicRMWFAdd(
 888   const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
 889   MachineMemOperand &MMO) {
 890   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FADD, OldValRes, Addr, Val,
 891                         MMO);
 892 }
 893
 894 MachineInstrBuilder
 895 MachineIRBuilder::buildAtomicRMWFSub(const DstOp &OldValRes, const SrcOp &Addr, const SrcOp &Val,
 896                                      MachineMemOperand &MMO) {
 897   return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_FSUB, OldValRes, Addr, Val,
 898                         MMO);
 899 }
 900
 901 MachineInstrBuilder
 902 MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) {
 903   return buildInstr(TargetOpcode::G_FENCE)
 904     .addImm(Ordering)
 905     .addImm(Scope);
 906 }
 907
 908 MachineInstrBuilder
 909 MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) {
 910 #ifndef NDEBUG
 911   assert(getMRI()->getType(Res).isPointer() && "invalid res type");
 912 #endif
 913
 914   return buildInstr(TargetOpcode::G_BLOCK_ADDR).addDef(Res).addBlockAddress(BA);
 915 }
 916
 917 void MachineIRBuilder::validateTruncExt(const LLT &DstTy, const LLT &SrcTy,
 918                                         bool IsExtend) {
 919 #ifndef NDEBUG
 920   if (DstTy.isVector()) {
 921     assert(SrcTy.isVector() && "mismatched cast between vector and non-vector");
 922     assert(SrcTy.getNumElements() == DstTy.getNumElements() &&
 923            "different number of elements in a trunc/ext");
 924   } else
 925     assert(DstTy.isScalar() && SrcTy.isScalar() && "invalid extend/trunc");
 926
 927   if (IsExtend)
 928     assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
 929            "invalid narrowing extend");
 930   else
 931     assert(DstTy.getSizeInBits() < SrcTy.getSizeInBits() &&
 932            "invalid widening trunc");
 933 #endif
 934 }
 935
 936 void MachineIRBuilder::validateSelectOp(const LLT &ResTy, const LLT &TstTy,
 937                                         const LLT &Op0Ty, const LLT &Op1Ty) {
 938 #ifndef NDEBUG
 939   assert((ResTy.isScalar() || ResTy.isVector() || ResTy.isPointer()) &&
 940          "invalid operand type");
 941   assert((ResTy == Op0Ty && ResTy == Op1Ty) && "type mismatch");
 942   if (ResTy.isScalar() || ResTy.isPointer())
 943     assert(TstTy.isScalar() && "type mismatch");
 944   else
 945     assert((TstTy.isScalar() ||
 946             (TstTy.isVector() &&
 947              TstTy.getNumElements() == Op0Ty.getNumElements())) &&
 948            "type mismatch");
 949 #endif
 950 }
 951
 952 MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc,
 953                                                  ArrayRef<DstOp> DstOps,
 954                                                  ArrayRef<SrcOp> SrcOps,
 955                                                  Optional<unsigned> Flags) {
 956   switch (Opc) {
 957   default:
 958     break;
 959   case TargetOpcode::G_SELECT: {
 960     assert(DstOps.size() == 1 && "Invalid select");
 961     assert(SrcOps.size() == 3 && "Invalid select");
 962     validateSelectOp(
 963         DstOps[0].getLLTTy(*getMRI()), SrcOps[0].getLLTTy(*getMRI()),
 964         SrcOps[1].getLLTTy(*getMRI()), SrcOps[2].getLLTTy(*getMRI()));
 965     break;
 966   }
 967   case TargetOpcode::G_ADD:
 968   case TargetOpcode::G_AND:
 969   case TargetOpcode::G_MUL:
 970   case TargetOpcode::G_OR:
 971   case TargetOpcode::G_SUB:
 972   case TargetOpcode::G_XOR:
 973   case TargetOpcode::G_UDIV:
 974   case TargetOpcode::G_SDIV:
 975   case TargetOpcode::G_UREM:
 976   case TargetOpcode::G_SREM:
 977   case TargetOpcode::G_SMIN:
 978   case TargetOpcode::G_SMAX:
 979   case TargetOpcode::G_UMIN:
 980   case TargetOpcode::G_UMAX: {
 981     // All these are binary ops.
 982     assert(DstOps.size() == 1 && "Invalid Dst");
 983     assert(SrcOps.size() == 2 && "Invalid Srcs");
 984     validateBinaryOp(DstOps[0].getLLTTy(*getMRI()),
 985                      SrcOps[0].getLLTTy(*getMRI()),
 986                      SrcOps[1].getLLTTy(*getMRI()));
 987     break;
 988   }
 989   case TargetOpcode::G_SHL:
 990   case TargetOpcode::G_ASHR:
 991   case TargetOpcode::G_LSHR: {
 992     assert(DstOps.size() == 1 && "Invalid Dst");
 993     assert(SrcOps.size() == 2 && "Invalid Srcs");
 994     validateShiftOp(DstOps[0].getLLTTy(*getMRI()),
 995                     SrcOps[0].getLLTTy(*getMRI()),
 996                     SrcOps[1].getLLTTy(*getMRI()));
 997     break;
 998   }
 999   case TargetOpcode::G_SEXT:
1000   case TargetOpcode::G_ZEXT:
1001   case TargetOpcode::G_ANYEXT:
1002     assert(DstOps.size() == 1 && "Invalid Dst");
1003     assert(SrcOps.size() == 1 && "Invalid Srcs");
1004     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1005                      SrcOps[0].getLLTTy(*getMRI()), true);
1006     break;
1007   case TargetOpcode::G_TRUNC:
1008   case TargetOpcode::G_FPTRUNC: {
1009     assert(DstOps.size() == 1 && "Invalid Dst");
1010     assert(SrcOps.size() == 1 && "Invalid Srcs");
1011     validateTruncExt(DstOps[0].getLLTTy(*getMRI()),
1012                      SrcOps[0].getLLTTy(*getMRI()), false);
1013     break;
1014   }
1015   case TargetOpcode::COPY:
1016     assert(DstOps.size() == 1 && "Invalid Dst");
1017     // If the caller wants to add a subreg source it has to be done separately
1018     // so we may not have any SrcOps at this point yet.
1019     break;
1020   case TargetOpcode::G_FCMP:
1021   case TargetOpcode::G_ICMP: {
1022     assert(DstOps.size() == 1 && "Invalid Dst Operands");
1023     assert(SrcOps.size() == 3 && "Invalid Src Operands");
1024     // For F/ICMP, the first src operand is the predicate, followed by
1025     // the two comparands.
1026     assert(SrcOps[0].getSrcOpKind() == SrcOp::SrcType::Ty_Predicate &&
1027            "Expecting predicate");
1028     assert([&]() -> bool {
1029       CmpInst::Predicate Pred = SrcOps[0].getPredicate();
1030       return Opc == TargetOpcode::G_ICMP ? CmpInst::isIntPredicate(Pred)
1031                                          : CmpInst::isFPPredicate(Pred);
1032     }() && "Invalid predicate");
1033     assert(SrcOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1034            "Type mismatch");
1035     assert([&]() -> bool {
1036       LLT Op0Ty = SrcOps[1].getLLTTy(*getMRI());
1037       LLT DstTy = DstOps[0].getLLTTy(*getMRI());
1038       if (Op0Ty.isScalar() || Op0Ty.isPointer())
1039         return DstTy.isScalar();
1040       else
1041         return DstTy.isVector() &&
1042                DstTy.getNumElements() == Op0Ty.getNumElements();
1043     }() && "Type Mismatch");
1044     break;
1045   }
1046   case TargetOpcode::G_UNMERGE_VALUES: {
1047     assert(!DstOps.empty() && "Invalid trivial sequence");
1048     assert(SrcOps.size() == 1 && "Invalid src for Unmerge");
1049     assert(std::all_of(DstOps.begin(), DstOps.end(),
1050                        [&, this](const DstOp &Op) {
1051                          return Op.getLLTTy(*getMRI()) ==
1052                                 DstOps[0].getLLTTy(*getMRI());
1053                        }) &&
1054            "type mismatch in output list");
1055     assert(DstOps.size() * DstOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1056                SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1057            "input operands do not cover output register");
1058     break;
1059   }
1060   case TargetOpcode::G_MERGE_VALUES: {
1061     assert(!SrcOps.empty() && "invalid trivial sequence");
1062     assert(DstOps.size() == 1 && "Invalid Dst");
1063     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1064                        [&, this](const SrcOp &Op) {
1065                          return Op.getLLTTy(*getMRI()) ==
1066                                 SrcOps[0].getLLTTy(*getMRI());
1067                        }) &&
1068            "type mismatch in input list");
1069     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1070                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1071            "input operands do not cover output register");
1072     if (SrcOps.size() == 1)
1073       return buildCast(DstOps[0], SrcOps[0]);
1074     if (DstOps[0].getLLTTy(*getMRI()).isVector()) {
1075       if (SrcOps[0].getLLTTy(*getMRI()).isVector())
1076         return buildInstr(TargetOpcode::G_CONCAT_VECTORS, DstOps, SrcOps);
1077       return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps);
1078     }
1079     break;
1080   }
1081   case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1082     assert(DstOps.size() == 1 && "Invalid Dst size");
1083     assert(SrcOps.size() == 2 && "Invalid Src size");
1084     assert(SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1085     assert((DstOps[0].getLLTTy(*getMRI()).isScalar() ||
1086             DstOps[0].getLLTTy(*getMRI()).isPointer()) &&
1087            "Invalid operand type");
1088     assert(SrcOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand type");
1089     assert(SrcOps[0].getLLTTy(*getMRI()).getElementType() ==
1090                DstOps[0].getLLTTy(*getMRI()) &&
1091            "Type mismatch");
1092     break;
1093   }
1094   case TargetOpcode::G_INSERT_VECTOR_ELT: {
1095     assert(DstOps.size() == 1 && "Invalid dst size");
1096     assert(SrcOps.size() == 3 && "Invalid src size");
1097     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1098            SrcOps[0].getLLTTy(*getMRI()).isVector() && "Invalid operand type");
1099     assert(DstOps[0].getLLTTy(*getMRI()).getElementType() ==
1100                SrcOps[1].getLLTTy(*getMRI()) &&
1101            "Type mismatch");
1102     assert(SrcOps[2].getLLTTy(*getMRI()).isScalar() && "Invalid index");
1103     assert(DstOps[0].getLLTTy(*getMRI()).getNumElements() ==
1104                SrcOps[0].getLLTTy(*getMRI()).getNumElements() &&
1105            "Type mismatch");
1106     break;
1107   }
1108   case TargetOpcode::G_BUILD_VECTOR: {
1109     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1110            "Must have at least 2 operands");
1111     assert(DstOps.size() == 1 && "Invalid DstOps");
1112     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1113            "Res type must be a vector");
1114     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1115                        [&, this](const SrcOp &Op) {
1116                          return Op.getLLTTy(*getMRI()) ==
1117                                 SrcOps[0].getLLTTy(*getMRI());
1118                        }) &&
1119            "type mismatch in input list");
1120     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1121                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1122            "input scalars do not exactly cover the output vector register");
1123     break;
1124   }
1125   case TargetOpcode::G_BUILD_VECTOR_TRUNC: {
1126     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1127            "Must have at least 2 operands");
1128     assert(DstOps.size() == 1 && "Invalid DstOps");
1129     assert(DstOps[0].getLLTTy(*getMRI()).isVector() &&
1130            "Res type must be a vector");
1131     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1132                        [&, this](const SrcOp &Op) {
1133                          return Op.getLLTTy(*getMRI()) ==
1134                                 SrcOps[0].getLLTTy(*getMRI());
1135                        }) &&
1136            "type mismatch in input list");
1137     if (SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1138         DstOps[0].getLLTTy(*getMRI()).getElementType().getSizeInBits())
1139       return buildInstr(TargetOpcode::G_BUILD_VECTOR, DstOps, SrcOps);
1140     break;
1141   }
1142   case TargetOpcode::G_CONCAT_VECTORS: {
1143     assert(DstOps.size() == 1 && "Invalid DstOps");
1144     assert((!SrcOps.empty() || SrcOps.size() < 2) &&
1145            "Must have at least 2 operands");
1146     assert(std::all_of(SrcOps.begin(), SrcOps.end(),
1147                        [&, this](const SrcOp &Op) {
1148                          return (Op.getLLTTy(*getMRI()).isVector() &&
1149                                  Op.getLLTTy(*getMRI()) ==
1150                                      SrcOps[0].getLLTTy(*getMRI()));
1151                        }) &&
1152            "type mismatch in input list");
1153     assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() ==
1154                DstOps[0].getLLTTy(*getMRI()).getSizeInBits() &&
1155            "input vectors do not exactly cover the output vector register");
1156     break;
1157   }
1158   case TargetOpcode::G_UADDE: {
1159     assert(DstOps.size() == 2 && "Invalid no of dst operands");
1160     assert(SrcOps.size() == 3 && "Invalid no of src operands");
1161     assert(DstOps[0].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1162     assert((DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())) &&
1163            (DstOps[0].getLLTTy(*getMRI()) == SrcOps[1].getLLTTy(*getMRI())) &&
1164            "Invalid operand");
1165     assert(DstOps[1].getLLTTy(*getMRI()).isScalar() && "Invalid operand");
1166     assert(DstOps[1].getLLTTy(*getMRI()) == SrcOps[2].getLLTTy(*getMRI()) &&
1167            "type mismatch");
1168     break;
1169   }
1170   }
1171
1172   auto MIB = buildInstr(Opc);
1173   for (const DstOp &Op : DstOps)
1174     Op.addDefToMIB(*getMRI(), MIB);
1175   for (const SrcOp &Op : SrcOps)
1176     Op.addSrcToMIB(MIB);
1177   if (Flags)
1178     MIB->setFlags(*Flags);
1179   return MIB;
1180 }