include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

   1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file declares the API for the instruction selector.
  10 /// This class is responsible for selecting machine instructions.
  11 /// It's implemented by the target. It's used by the InstructionSelect pass.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  17
  18 #include "llvm/ADT/SmallVector.h"
  19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  21 #include "llvm/CodeGen/GlobalISel/Utils.h"
  22 #include "llvm/CodeGen/MachineInstrBuilder.h"
  23 #include "llvm/CodeGen/MachineOperand.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetInstrInfo.h"
  26 #include "llvm/CodeGen/TargetOpcodes.h"
  27 #include "llvm/CodeGen/TargetRegisterInfo.h"
  28 #include "llvm/IR/Constants.h"
  29 #include "llvm/Support/Debug.h"
  30 #include "llvm/Support/ErrorHandling.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include <cassert>
  33 #include <cstddef>
  34 #include <cstdint>
  35
  36 namespace llvm {
  37
  38 /// GlobalISel PatFrag Predicates
  39 enum {
  40   GIPFP_I64_Invalid = 0,
  41   GIPFP_APInt_Invalid = 0,
  42   GIPFP_APFloat_Invalid = 0,
  43   GIPFP_MI_Invalid = 0,
  44 };
  45
  46 template <class TgtInstructionSelector, class PredicateBitset,
  47           class ComplexMatcherMemFn, class CustomRendererFn>
  48 bool InstructionSelector::executeMatchTable(
  49     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
  50     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
  51         &ISelInfo,
  52     const int64_t *MatchTable, const TargetInstrInfo &TII,
  53     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
  54     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
  55     CodeGenCoverage &CoverageInfo) const {
  56
  57   uint64_t CurrentIdx = 0;
  58   SmallVector<uint64_t, 4> OnFailResumeAt;
  59
  60   enum RejectAction { RejectAndGiveUp, RejectAndResume };
  61   auto handleReject = [&]() -> RejectAction {
  62     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  63                     dbgs() << CurrentIdx << ": Rejected\n");
  64     if (OnFailResumeAt.empty())
  65       return RejectAndGiveUp;
  66     CurrentIdx = OnFailResumeAt.pop_back_val();
  67     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  68                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
  69                            << OnFailResumeAt.size() << " try-blocks remain)\n");
  70     return RejectAndResume;
  71   };
  72
  73   while (true) {
  74     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
  75     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
  76     switch (MatcherOpcode) {
  77     case GIM_Try: {
  78       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  79                       dbgs() << CurrentIdx << ": Begin try-block\n");
  80       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
  81       break;
  82     }
  83
  84     case GIM_RecordInsn: {
  85       int64_t NewInsnID = MatchTable[CurrentIdx++];
  86       int64_t InsnID = MatchTable[CurrentIdx++];
  87       int64_t OpIdx = MatchTable[CurrentIdx++];
  88
  89       // As an optimisation we require that MIs[0] is always the root. Refuse
  90       // any attempt to modify it.
  91       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
  92
  93       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  94       if (!MO.isReg()) {
  95         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  96                         dbgs() << CurrentIdx << ": Not a register\n");
  97         if (handleReject() == RejectAndGiveUp)
  98           return false;
  99         break;
 100       }
 101       if (Register::isPhysicalRegister(MO.getReg())) {
 102         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 103                         dbgs() << CurrentIdx << ": Is a physical register\n");
 104         if (handleReject() == RejectAndGiveUp)
 105           return false;
 106         break;
 107       }
 108
 109       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
 110       if ((size_t)NewInsnID < State.MIs.size())
 111         State.MIs[NewInsnID] = NewMI;
 112       else {
 113         assert((size_t)NewInsnID == State.MIs.size() &&
 114                "Expected to store MIs in order");
 115         State.MIs.push_back(NewMI);
 116       }
 117       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 118                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
 119                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
 120                              << ")\n");
 121       break;
 122     }
 123
 124     case GIM_CheckFeatures: {
 125       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
 126       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 127                       dbgs() << CurrentIdx
 128                              << ": GIM_CheckFeatures(ExpectedBitsetID="
 129                              << ExpectedBitsetID << ")\n");
 130       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
 131           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
 132         if (handleReject() == RejectAndGiveUp)
 133           return false;
 134       }
 135       break;
 136     }
 137
 138     case GIM_CheckOpcode: {
 139       int64_t InsnID = MatchTable[CurrentIdx++];
 140       int64_t Expected = MatchTable[CurrentIdx++];
 141
 142       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 143       unsigned Opcode = State.MIs[InsnID]->getOpcode();
 144
 145       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 146                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
 147                              << "], ExpectedOpcode=" << Expected
 148                              << ") // Got=" << Opcode << "\n");
 149       if (Opcode != Expected) {
 150         if (handleReject() == RejectAndGiveUp)
 151           return false;
 152       }
 153       break;
 154     }
 155
 156     case GIM_SwitchOpcode: {
 157       int64_t InsnID = MatchTable[CurrentIdx++];
 158       int64_t LowerBound = MatchTable[CurrentIdx++];
 159       int64_t UpperBound = MatchTable[CurrentIdx++];
 160       int64_t Default = MatchTable[CurrentIdx++];
 161
 162       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 163       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
 164
 165       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 166         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
 167                << LowerBound << ", " << UpperBound << "), Default=" << Default
 168                << ", JumpTable...) // Got=" << Opcode << "\n";
 169       });
 170       if (Opcode < LowerBound || UpperBound <= Opcode) {
 171         CurrentIdx = Default;
 172         break;
 173       }
 174       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
 175       if (!CurrentIdx) {
 176         CurrentIdx = Default;
 177         break;
 178       }
 179       OnFailResumeAt.push_back(Default);
 180       break;
 181     }
 182
 183     case GIM_SwitchType: {
 184       int64_t InsnID = MatchTable[CurrentIdx++];
 185       int64_t OpIdx = MatchTable[CurrentIdx++];
 186       int64_t LowerBound = MatchTable[CurrentIdx++];
 187       int64_t UpperBound = MatchTable[CurrentIdx++];
 188       int64_t Default = MatchTable[CurrentIdx++];
 189
 190       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 191       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 192
 193       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 194         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
 195                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
 196                << UpperBound << "), Default=" << Default
 197                << ", JumpTable...) // Got=";
 198         if (!MO.isReg())
 199           dbgs() << "Not a VReg\n";
 200         else
 201           dbgs() << MRI.getType(MO.getReg()) << "\n";
 202       });
 203       if (!MO.isReg()) {
 204         CurrentIdx = Default;
 205         break;
 206       }
 207       const LLT Ty = MRI.getType(MO.getReg());
 208       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
 209       if (TyI == ISelInfo.TypeIDMap.end()) {
 210         CurrentIdx = Default;
 211         break;
 212       }
 213       const int64_t TypeID = TyI->second;
 214       if (TypeID < LowerBound || UpperBound <= TypeID) {
 215         CurrentIdx = Default;
 216         break;
 217       }
 218       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
 219       if (!CurrentIdx) {
 220         CurrentIdx = Default;
 221         break;
 222       }
 223       OnFailResumeAt.push_back(Default);
 224       break;
 225     }
 226
 227     case GIM_CheckNumOperands: {
 228       int64_t InsnID = MatchTable[CurrentIdx++];
 229       int64_t Expected = MatchTable[CurrentIdx++];
 230       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 231                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
 232                              << InsnID << "], Expected=" << Expected << ")\n");
 233       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 234       if (State.MIs[InsnID]->getNumOperands() != Expected) {
 235         if (handleReject() == RejectAndGiveUp)
 236           return false;
 237       }
 238       break;
 239     }
 240     case GIM_CheckI64ImmPredicate: {
 241       int64_t InsnID = MatchTable[CurrentIdx++];
 242       int64_t Predicate = MatchTable[CurrentIdx++];
 243       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 244                       dbgs()
 245                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
 246                           << InsnID << "], Predicate=" << Predicate << ")\n");
 247       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 248       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 249              "Expected G_CONSTANT");
 250       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
 251       int64_t Value = 0;
 252       if (State.MIs[InsnID]->getOperand(1).isCImm())
 253         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
 254       else if (State.MIs[InsnID]->getOperand(1).isImm())
 255         Value = State.MIs[InsnID]->getOperand(1).getImm();
 256       else
 257         llvm_unreachable("Expected Imm or CImm operand");
 258
 259       if (!testImmPredicate_I64(Predicate, Value))
 260         if (handleReject() == RejectAndGiveUp)
 261           return false;
 262       break;
 263     }
 264     case GIM_CheckAPIntImmPredicate: {
 265       int64_t InsnID = MatchTable[CurrentIdx++];
 266       int64_t Predicate = MatchTable[CurrentIdx++];
 267       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 268                       dbgs()
 269                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
 270                           << InsnID << "], Predicate=" << Predicate << ")\n");
 271       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 272       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 273              "Expected G_CONSTANT");
 274       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
 275       APInt Value;
 276       if (State.MIs[InsnID]->getOperand(1).isCImm())
 277         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
 278       else
 279         llvm_unreachable("Expected Imm or CImm operand");
 280
 281       if (!testImmPredicate_APInt(Predicate, Value))
 282         if (handleReject() == RejectAndGiveUp)
 283           return false;
 284       break;
 285     }
 286     case GIM_CheckAPFloatImmPredicate: {
 287       int64_t InsnID = MatchTable[CurrentIdx++];
 288       int64_t Predicate = MatchTable[CurrentIdx++];
 289       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 290                       dbgs()
 291                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
 292                           << InsnID << "], Predicate=" << Predicate << ")\n");
 293       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 294       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
 295              "Expected G_FCONSTANT");
 296       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
 297       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
 298       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
 299
 300       if (!testImmPredicate_APFloat(Predicate, Value))
 301         if (handleReject() == RejectAndGiveUp)
 302           return false;
 303       break;
 304     }
 305     case GIM_CheckCxxInsnPredicate: {
 306       int64_t InsnID = MatchTable[CurrentIdx++];
 307       int64_t Predicate = MatchTable[CurrentIdx++];
 308       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 309                       dbgs()
 310                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
 311                           << InsnID << "], Predicate=" << Predicate << ")\n");
 312       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 313       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
 314
 315       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
 316         if (handleReject() == RejectAndGiveUp)
 317           return false;
 318       break;
 319     }
 320     case GIM_CheckAtomicOrdering: {
 321       int64_t InsnID = MatchTable[CurrentIdx++];
 322       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 323       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 324                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
 325                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 326       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 327       if (!State.MIs[InsnID]->hasOneMemOperand())
 328         if (handleReject() == RejectAndGiveUp)
 329           return false;
 330
 331       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 332         if (MMO->getOrdering() != Ordering)
 333           if (handleReject() == RejectAndGiveUp)
 334             return false;
 335       break;
 336     }
 337     case GIM_CheckAtomicOrderingOrStrongerThan: {
 338       int64_t InsnID = MatchTable[CurrentIdx++];
 339       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 340       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 341                       dbgs() << CurrentIdx
 342                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
 343                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 344       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 345       if (!State.MIs[InsnID]->hasOneMemOperand())
 346         if (handleReject() == RejectAndGiveUp)
 347           return false;
 348
 349       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 350         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
 351           if (handleReject() == RejectAndGiveUp)
 352             return false;
 353       break;
 354     }
 355     case GIM_CheckAtomicOrderingWeakerThan: {
 356       int64_t InsnID = MatchTable[CurrentIdx++];
 357       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 358       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 359                       dbgs() << CurrentIdx
 360                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
 361                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 362       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 363       if (!State.MIs[InsnID]->hasOneMemOperand())
 364         if (handleReject() == RejectAndGiveUp)
 365           return false;
 366
 367       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 368         if (!isStrongerThan(Ordering, MMO->getOrdering()))
 369           if (handleReject() == RejectAndGiveUp)
 370             return false;
 371       break;
 372     }
 373     case GIM_CheckMemoryAddressSpace: {
 374       int64_t InsnID = MatchTable[CurrentIdx++];
 375       int64_t MMOIdx = MatchTable[CurrentIdx++];
 376       // This accepts a list of possible address spaces.
 377       const int NumAddrSpace = MatchTable[CurrentIdx++];
 378
 379       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 380         if (handleReject() == RejectAndGiveUp)
 381           return false;
 382         break;
 383       }
 384
 385       // Need to still jump to the end of the list of address spaces if we find
 386       // a match earlier.
 387       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
 388
 389       const MachineMemOperand *MMO
 390         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 391       const unsigned MMOAddrSpace = MMO->getAddrSpace();
 392
 393       bool Success = false;
 394       for (int I = 0; I != NumAddrSpace; ++I) {
 395         unsigned AddrSpace = MatchTable[CurrentIdx++];
 396         DEBUG_WITH_TYPE(
 397           TgtInstructionSelector::getName(),
 398           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
 399                  << AddrSpace << '\n');
 400
 401         if (AddrSpace == MMOAddrSpace) {
 402           Success = true;
 403           break;
 404         }
 405       }
 406
 407       CurrentIdx = LastIdx;
 408       if (!Success && handleReject() == RejectAndGiveUp)
 409         return false;
 410       break;
 411     }
 412     case GIM_CheckMemoryAlignment: {
 413       int64_t InsnID = MatchTable[CurrentIdx++];
 414       int64_t MMOIdx = MatchTable[CurrentIdx++];
 415       unsigned MinAlign = MatchTable[CurrentIdx++];
 416
 417       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 418
 419       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 420         if (handleReject() == RejectAndGiveUp)
 421           return false;
 422         break;
 423       }
 424
 425       MachineMemOperand *MMO
 426         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 427       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 428                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
 429                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 430                       << ")->getAlignment() >= " << MinAlign << ")\n");
 431       if (MMO->getAlignment() < MinAlign && handleReject() == RejectAndGiveUp)
 432         return false;
 433
 434       break;
 435     }
 436     case GIM_CheckMemorySizeEqualTo: {
 437       int64_t InsnID = MatchTable[CurrentIdx++];
 438       int64_t MMOIdx = MatchTable[CurrentIdx++];
 439       uint64_t Size = MatchTable[CurrentIdx++];
 440
 441       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 442                       dbgs() << CurrentIdx
 443                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
 444                              << "]->memoperands() + " << MMOIdx
 445                              << ", Size=" << Size << ")\n");
 446       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 447
 448       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 449         if (handleReject() == RejectAndGiveUp)
 450           return false;
 451         break;
 452       }
 453
 454       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 455
 456       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 457                       dbgs() << MMO->getSize() << " bytes vs " << Size
 458                              << " bytes\n");
 459       if (MMO->getSize() != Size)
 460         if (handleReject() == RejectAndGiveUp)
 461           return false;
 462
 463       break;
 464     }
 465     case GIM_CheckMemorySizeEqualToLLT:
 466     case GIM_CheckMemorySizeLessThanLLT:
 467     case GIM_CheckMemorySizeGreaterThanLLT: {
 468       int64_t InsnID = MatchTable[CurrentIdx++];
 469       int64_t MMOIdx = MatchTable[CurrentIdx++];
 470       int64_t OpIdx = MatchTable[CurrentIdx++];
 471
 472       DEBUG_WITH_TYPE(
 473           TgtInstructionSelector::getName(),
 474           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
 475                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
 476                          ? "EqualTo"
 477                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
 478                                ? "GreaterThan"
 479                                : "LessThan")
 480                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 481                  << ", OpIdx=" << OpIdx << ")\n");
 482       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 483
 484       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 485       if (!MO.isReg()) {
 486         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 487                         dbgs() << CurrentIdx << ": Not a register\n");
 488         if (handleReject() == RejectAndGiveUp)
 489           return false;
 490         break;
 491       }
 492
 493       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 494         if (handleReject() == RejectAndGiveUp)
 495           return false;
 496         break;
 497       }
 498
 499       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 500
 501       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
 502       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
 503           MMO->getSizeInBits() != Size) {
 504         if (handleReject() == RejectAndGiveUp)
 505           return false;
 506       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
 507                  MMO->getSizeInBits() >= Size) {
 508         if (handleReject() == RejectAndGiveUp)
 509           return false;
 510       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
 511                  MMO->getSizeInBits() <= Size)
 512         if (handleReject() == RejectAndGiveUp)
 513           return false;
 514
 515       break;
 516     }
 517     case GIM_CheckType: {
 518       int64_t InsnID = MatchTable[CurrentIdx++];
 519       int64_t OpIdx = MatchTable[CurrentIdx++];
 520       int64_t TypeID = MatchTable[CurrentIdx++];
 521       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 522                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
 523                              << "]->getOperand(" << OpIdx
 524                              << "), TypeID=" << TypeID << ")\n");
 525       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 526       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 527       if (!MO.isReg() ||
 528           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
 529         if (handleReject() == RejectAndGiveUp)
 530           return false;
 531       }
 532       break;
 533     }
 534     case GIM_CheckPointerToAny: {
 535       int64_t InsnID = MatchTable[CurrentIdx++];
 536       int64_t OpIdx = MatchTable[CurrentIdx++];
 537       int64_t SizeInBits = MatchTable[CurrentIdx++];
 538
 539       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 540                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
 541                              << InsnID << "]->getOperand(" << OpIdx
 542                              << "), SizeInBits=" << SizeInBits << ")\n");
 543       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 544       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 545       const LLT Ty = MRI.getType(MO.getReg());
 546
 547       // iPTR must be looked up in the target.
 548       if (SizeInBits == 0) {
 549         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
 550         const unsigned AddrSpace = Ty.getAddressSpace();
 551         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
 552       }
 553
 554       assert(SizeInBits != 0 && "Pointer size must be known");
 555
 556       if (MO.isReg()) {
 557         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
 558           if (handleReject() == RejectAndGiveUp)
 559             return false;
 560       } else if (handleReject() == RejectAndGiveUp)
 561         return false;
 562
 563       break;
 564     }
 565     case GIM_CheckRegBankForClass: {
 566       int64_t InsnID = MatchTable[CurrentIdx++];
 567       int64_t OpIdx = MatchTable[CurrentIdx++];
 568       int64_t RCEnum = MatchTable[CurrentIdx++];
 569       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 570                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
 571                              << InsnID << "]->getOperand(" << OpIdx
 572                              << "), RCEnum=" << RCEnum << ")\n");
 573       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 574       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 575       if (!MO.isReg() ||
 576           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
 577               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
 578         if (handleReject() == RejectAndGiveUp)
 579           return false;
 580       }
 581       break;
 582     }
 583
 584     case GIM_CheckComplexPattern: {
 585       int64_t InsnID = MatchTable[CurrentIdx++];
 586       int64_t OpIdx = MatchTable[CurrentIdx++];
 587       int64_t RendererID = MatchTable[CurrentIdx++];
 588       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
 589       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 590                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
 591                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
 592                              << "]->getOperand(" << OpIdx
 593                              << "), ComplexPredicateID=" << ComplexPredicateID
 594                              << ")\n");
 595       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 596       // FIXME: Use std::invoke() when it's available.
 597       ComplexRendererFns Renderer =
 598           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
 599               State.MIs[InsnID]->getOperand(OpIdx));
 600       if (Renderer.hasValue())
 601         State.Renderers[RendererID] = Renderer.getValue();
 602       else
 603         if (handleReject() == RejectAndGiveUp)
 604           return false;
 605       break;
 606     }
 607
 608     case GIM_CheckConstantInt: {
 609       int64_t InsnID = MatchTable[CurrentIdx++];
 610       int64_t OpIdx = MatchTable[CurrentIdx++];
 611       int64_t Value = MatchTable[CurrentIdx++];
 612       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 613                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
 614                              << InsnID << "]->getOperand(" << OpIdx
 615                              << "), Value=" << Value << ")\n");
 616       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 617       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 618       if (MO.isReg()) {
 619         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
 620         LLT Ty = MRI.getType(MO.getReg());
 621         Value = SignExtend64(Value, Ty.getSizeInBits());
 622
 623         if (!isOperandImmEqual(MO, Value, MRI)) {
 624           if (handleReject() == RejectAndGiveUp)
 625             return false;
 626         }
 627       } else if (handleReject() == RejectAndGiveUp)
 628         return false;
 629
 630       break;
 631     }
 632
 633     case GIM_CheckLiteralInt: {
 634       int64_t InsnID = MatchTable[CurrentIdx++];
 635       int64_t OpIdx = MatchTable[CurrentIdx++];
 636       int64_t Value = MatchTable[CurrentIdx++];
 637       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 638                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
 639                              << InsnID << "]->getOperand(" << OpIdx
 640                              << "), Value=" << Value << ")\n");
 641       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 642       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 643       if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
 644         if (handleReject() == RejectAndGiveUp)
 645           return false;
 646       }
 647       break;
 648     }
 649
 650     case GIM_CheckIntrinsicID: {
 651       int64_t InsnID = MatchTable[CurrentIdx++];
 652       int64_t OpIdx = MatchTable[CurrentIdx++];
 653       int64_t Value = MatchTable[CurrentIdx++];
 654       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 655                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
 656                              << InsnID << "]->getOperand(" << OpIdx
 657                              << "), Value=" << Value << ")\n");
 658       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 659       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 660       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
 661         if (handleReject() == RejectAndGiveUp)
 662           return false;
 663       break;
 664     }
 665     case GIM_CheckCmpPredicate: {
 666       int64_t InsnID = MatchTable[CurrentIdx++];
 667       int64_t OpIdx = MatchTable[CurrentIdx++];
 668       int64_t Value = MatchTable[CurrentIdx++];
 669       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 670                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
 671                              << InsnID << "]->getOperand(" << OpIdx
 672                              << "), Value=" << Value << ")\n");
 673       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 674       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 675       if (!MO.isPredicate() || MO.getPredicate() != Value)
 676         if (handleReject() == RejectAndGiveUp)
 677           return false;
 678       break;
 679     }
 680     case GIM_CheckIsMBB: {
 681       int64_t InsnID = MatchTable[CurrentIdx++];
 682       int64_t OpIdx = MatchTable[CurrentIdx++];
 683       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 684                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
 685                              << "]->getOperand(" << OpIdx << "))\n");
 686       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 687       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
 688         if (handleReject() == RejectAndGiveUp)
 689           return false;
 690       }
 691       break;
 692     }
 693     case GIM_CheckIsImm: {
 694       int64_t InsnID = MatchTable[CurrentIdx++];
 695       int64_t OpIdx = MatchTable[CurrentIdx++];
 696       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 697                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
 698                              << "]->getOperand(" << OpIdx << "))\n");
 699       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 700       if (!State.MIs[InsnID]->getOperand(OpIdx).isImm()) {
 701         if (handleReject() == RejectAndGiveUp)
 702           return false;
 703       }
 704       break;
 705     }
 706     case GIM_CheckIsSafeToFold: {
 707       int64_t InsnID = MatchTable[CurrentIdx++];
 708       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 709                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
 710                              << InsnID << "])\n");
 711       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 712       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
 713         if (handleReject() == RejectAndGiveUp)
 714           return false;
 715       }
 716       break;
 717     }
 718     case GIM_CheckIsSameOperand: {
 719       int64_t InsnID = MatchTable[CurrentIdx++];
 720       int64_t OpIdx = MatchTable[CurrentIdx++];
 721       int64_t OtherInsnID = MatchTable[CurrentIdx++];
 722       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
 723       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 724                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
 725                              << InsnID << "][" << OpIdx << "], MIs["
 726                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
 727       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 728       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
 729       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
 730               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
 731         if (handleReject() == RejectAndGiveUp)
 732           return false;
 733       }
 734       break;
 735     }
 736     case GIM_Reject:
 737       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 738                       dbgs() << CurrentIdx << ": GIM_Reject\n");
 739       if (handleReject() == RejectAndGiveUp)
 740         return false;
 741       break;
 742
 743     case GIR_MutateOpcode: {
 744       int64_t OldInsnID = MatchTable[CurrentIdx++];
 745       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 746       int64_t NewOpcode = MatchTable[CurrentIdx++];
 747       if (NewInsnID >= OutMIs.size())
 748         OutMIs.resize(NewInsnID + 1);
 749
 750       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
 751                                               State.MIs[OldInsnID]);
 752       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
 753       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 754                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
 755                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 756                              << NewOpcode << ")\n");
 757       break;
 758     }
 759
 760     case GIR_BuildMI: {
 761       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 762       int64_t Opcode = MatchTable[CurrentIdx++];
 763       if (NewInsnID >= OutMIs.size())
 764         OutMIs.resize(NewInsnID + 1);
 765
 766       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
 767                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
 768       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 769                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
 770                              << NewInsnID << "], " << Opcode << ")\n");
 771       break;
 772     }
 773
 774     case GIR_Copy: {
 775       int64_t NewInsnID = MatchTable[CurrentIdx++];
 776       int64_t OldInsnID = MatchTable[CurrentIdx++];
 777       int64_t OpIdx = MatchTable[CurrentIdx++];
 778       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 779       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
 780       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 781                       dbgs()
 782                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
 783                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
 784       break;
 785     }
 786
 787     case GIR_CopyOrAddZeroReg: {
 788       int64_t NewInsnID = MatchTable[CurrentIdx++];
 789       int64_t OldInsnID = MatchTable[CurrentIdx++];
 790       int64_t OpIdx = MatchTable[CurrentIdx++];
 791       int64_t ZeroReg = MatchTable[CurrentIdx++];
 792       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 793       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
 794       if (isOperandImmEqual(MO, 0, MRI))
 795         OutMIs[NewInsnID].addReg(ZeroReg);
 796       else
 797         OutMIs[NewInsnID].add(MO);
 798       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 799                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
 800                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 801                              << OpIdx << ", " << ZeroReg << ")\n");
 802       break;
 803     }
 804
 805     case GIR_CopySubReg: {
 806       int64_t NewInsnID = MatchTable[CurrentIdx++];
 807       int64_t OldInsnID = MatchTable[CurrentIdx++];
 808       int64_t OpIdx = MatchTable[CurrentIdx++];
 809       int64_t SubRegIdx = MatchTable[CurrentIdx++];
 810       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 811       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
 812                                0, SubRegIdx);
 813       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 814                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
 815                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 816                              << OpIdx << ", " << SubRegIdx << ")\n");
 817       break;
 818     }
 819
 820     case GIR_AddImplicitDef: {
 821       int64_t InsnID = MatchTable[CurrentIdx++];
 822       int64_t RegNum = MatchTable[CurrentIdx++];
 823       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 824       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
 825       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 826                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
 827                              << InsnID << "], " << RegNum << ")\n");
 828       break;
 829     }
 830
 831     case GIR_AddImplicitUse: {
 832       int64_t InsnID = MatchTable[CurrentIdx++];
 833       int64_t RegNum = MatchTable[CurrentIdx++];
 834       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 835       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
 836       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 837                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
 838                              << InsnID << "], " << RegNum << ")\n");
 839       break;
 840     }
 841
 842     case GIR_AddRegister: {
 843       int64_t InsnID = MatchTable[CurrentIdx++];
 844       int64_t RegNum = MatchTable[CurrentIdx++];
 845       uint64_t RegFlags = MatchTable[CurrentIdx++];
 846       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 847       OutMIs[InsnID].addReg(RegNum, RegFlags);
 848       DEBUG_WITH_TYPE(
 849         TgtInstructionSelector::getName(),
 850         dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
 851         << InsnID << "], " << RegNum << ", " << RegFlags << ")\n");
 852       break;
 853     }
 854
 855     case GIR_AddTempRegister: {
 856       int64_t InsnID = MatchTable[CurrentIdx++];
 857       int64_t TempRegID = MatchTable[CurrentIdx++];
 858       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
 859       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 860       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
 861       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 862                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
 863                              << InsnID << "], TempRegisters[" << TempRegID
 864                              << "], " << TempRegFlags << ")\n");
 865       break;
 866     }
 867
 868     case GIR_AddImm: {
 869       int64_t InsnID = MatchTable[CurrentIdx++];
 870       int64_t Imm = MatchTable[CurrentIdx++];
 871       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 872       OutMIs[InsnID].addImm(Imm);
 873       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 874                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
 875                              << "], " << Imm << ")\n");
 876       break;
 877     }
 878
 879     case GIR_ComplexRenderer: {
 880       int64_t InsnID = MatchTable[CurrentIdx++];
 881       int64_t RendererID = MatchTable[CurrentIdx++];
 882       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 883       for (const auto &RenderOpFn : State.Renderers[RendererID])
 884         RenderOpFn(OutMIs[InsnID]);
 885       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 886                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
 887                              << InsnID << "], " << RendererID << ")\n");
 888       break;
 889     }
 890     case GIR_ComplexSubOperandRenderer: {
 891       int64_t InsnID = MatchTable[CurrentIdx++];
 892       int64_t RendererID = MatchTable[CurrentIdx++];
 893       int64_t RenderOpID = MatchTable[CurrentIdx++];
 894       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 895       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
 896       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 897                       dbgs() << CurrentIdx
 898                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
 899                              << InsnID << "], " << RendererID << ", "
 900                              << RenderOpID << ")\n");
 901       break;
 902     }
 903
 904     case GIR_CopyConstantAsSImm: {
 905       int64_t NewInsnID = MatchTable[CurrentIdx++];
 906       int64_t OldInsnID = MatchTable[CurrentIdx++];
 907       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 908       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
 909       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
 910         OutMIs[NewInsnID].addImm(
 911             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
 912       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
 913         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
 914       else
 915         llvm_unreachable("Expected Imm or CImm operand");
 916       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 917                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
 918                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 919       break;
 920     }
 921
 922     // TODO: Needs a test case once we have a pattern that uses this.
 923     case GIR_CopyFConstantAsFPImm: {
 924       int64_t NewInsnID = MatchTable[CurrentIdx++];
 925       int64_t OldInsnID = MatchTable[CurrentIdx++];
 926       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 927       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
 928       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
 929         OutMIs[NewInsnID].addFPImm(
 930             State.MIs[OldInsnID]->getOperand(1).getFPImm());
 931       else
 932         llvm_unreachable("Expected FPImm operand");
 933       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 934                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
 935                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 936       break;
 937     }
 938
 939     case GIR_CustomRenderer: {
 940       int64_t InsnID = MatchTable[CurrentIdx++];
 941       int64_t OldInsnID = MatchTable[CurrentIdx++];
 942       int64_t RendererFnID = MatchTable[CurrentIdx++];
 943       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 944       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 945                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
 946                              << InsnID << "], MIs[" << OldInsnID << "], "
 947                              << RendererFnID << ")\n");
 948       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
 949                                                      *State.MIs[OldInsnID]);
 950       break;
 951     }
 952     case GIR_ConstrainOperandRC: {
 953       int64_t InsnID = MatchTable[CurrentIdx++];
 954       int64_t OpIdx = MatchTable[CurrentIdx++];
 955       int64_t RCEnum = MatchTable[CurrentIdx++];
 956       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 957       constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
 958                                     *TRI.getRegClass(RCEnum), TII, TRI, RBI);
 959       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 960                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
 961                              << InsnID << "], " << OpIdx << ", " << RCEnum
 962                              << ")\n");
 963       break;
 964     }
 965
 966     case GIR_ConstrainSelectedInstOperands: {
 967       int64_t InsnID = MatchTable[CurrentIdx++];
 968       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 969       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
 970                                        RBI);
 971       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 972                       dbgs() << CurrentIdx
 973                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
 974                              << InsnID << "])\n");
 975       break;
 976     }
 977
 978     case GIR_MergeMemOperands: {
 979       int64_t InsnID = MatchTable[CurrentIdx++];
 980       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 981
 982       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 983                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
 984                              << InsnID << "]");
 985       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
 986       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
 987              GIU_MergeMemOperands_EndOfList) {
 988         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 989                         dbgs() << ", MIs[" << MergeInsnID << "]");
 990         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
 991           OutMIs[InsnID].addMemOperand(MMO);
 992       }
 993       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
 994       break;
 995     }
 996
 997     case GIR_EraseFromParent: {
 998       int64_t InsnID = MatchTable[CurrentIdx++];
 999       assert(State.MIs[InsnID] &&
1000              "Attempted to erase an undefined instruction");
1001       State.MIs[InsnID]->eraseFromParent();
1002       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1003                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
1004                              << InsnID << "])\n");
1005       break;
1006     }
1007
1008     case GIR_MakeTempReg: {
1009       int64_t TempRegID = MatchTable[CurrentIdx++];
1010       int64_t TypeID = MatchTable[CurrentIdx++];
1011
1012       State.TempRegisters[TempRegID] =
1013           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1014       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1015                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1016                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1017       break;
1018     }
1019
1020     case GIR_Coverage: {
1021       int64_t RuleID = MatchTable[CurrentIdx++];
1022       CoverageInfo.setCovered(RuleID);
1023
1024       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1025                       dbgs()
1026                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1027       break;
1028     }
1029
1030     case GIR_Done:
1031       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1032                       dbgs() << CurrentIdx << ": GIR_Done\n");
1033       return true;
1034
1035     default:
1036       llvm_unreachable("Unexpected command");
1037     }
1038   }
1039 }
1040
1041 } // end namespace llvm
1042
1043 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H