include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h

   1 //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file declares the API for the instruction selector.
  10 /// This class is responsible for selecting machine instructions.
  11 /// It's implemented by the target. It's used by the InstructionSelect pass.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #ifndef LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  16 #define LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H
  17
  18 #include "llvm/ADT/SmallVector.h"
  19 #include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
  20 #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
  21 #include "llvm/CodeGen/GlobalISel/Utils.h"
  22 #include "llvm/CodeGen/MachineInstrBuilder.h"
  23 #include "llvm/CodeGen/MachineOperand.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetInstrInfo.h"
  26 #include "llvm/CodeGen/TargetOpcodes.h"
  27 #include "llvm/CodeGen/TargetRegisterInfo.h"
  28 #include "llvm/IR/Constants.h"
  29 #include "llvm/Support/Debug.h"
  30 #include "llvm/Support/ErrorHandling.h"
  31 #include "llvm/Support/raw_ostream.h"
  32 #include <cassert>
  33 #include <cstddef>
  34 #include <cstdint>
  35
  36 namespace llvm {
  37
  38 /// GlobalISel PatFrag Predicates
  39 enum {
  40   GIPFP_I64_Invalid = 0,
  41   GIPFP_APInt_Invalid = 0,
  42   GIPFP_APFloat_Invalid = 0,
  43   GIPFP_MI_Invalid = 0,
  44 };
  45
  46 template <class TgtInstructionSelector, class PredicateBitset,
  47           class ComplexMatcherMemFn, class CustomRendererFn>
  48 bool InstructionSelector::executeMatchTable(
  49     TgtInstructionSelector &ISel, NewMIVector &OutMIs, MatcherState &State,
  50     const ISelInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
  51         &ISelInfo,
  52     const int64_t *MatchTable, const TargetInstrInfo &TII,
  53     MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI,
  54     const RegisterBankInfo &RBI, const PredicateBitset &AvailableFeatures,
  55     CodeGenCoverage &CoverageInfo) const {
  56
  57   uint64_t CurrentIdx = 0;
  58   SmallVector<uint64_t, 4> OnFailResumeAt;
  59
  60   enum RejectAction { RejectAndGiveUp, RejectAndResume };
  61   auto handleReject = [&]() -> RejectAction {
  62     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  63                     dbgs() << CurrentIdx << ": Rejected\n");
  64     if (OnFailResumeAt.empty())
  65       return RejectAndGiveUp;
  66     CurrentIdx = OnFailResumeAt.pop_back_val();
  67     DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  68                     dbgs() << CurrentIdx << ": Resume at " << CurrentIdx << " ("
  69                            << OnFailResumeAt.size() << " try-blocks remain)\n");
  70     return RejectAndResume;
  71   };
  72
  73   while (true) {
  74     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
  75     int64_t MatcherOpcode = MatchTable[CurrentIdx++];
  76     switch (MatcherOpcode) {
  77     case GIM_Try: {
  78       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  79                       dbgs() << CurrentIdx << ": Begin try-block\n");
  80       OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
  81       break;
  82     }
  83
  84     case GIM_RecordInsn: {
  85       int64_t NewInsnID = MatchTable[CurrentIdx++];
  86       int64_t InsnID = MatchTable[CurrentIdx++];
  87       int64_t OpIdx = MatchTable[CurrentIdx++];
  88
  89       // As an optimisation we require that MIs[0] is always the root. Refuse
  90       // any attempt to modify it.
  91       assert(NewInsnID != 0 && "Refusing to modify MIs[0]");
  92
  93       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
  94       if (!MO.isReg()) {
  95         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
  96                         dbgs() << CurrentIdx << ": Not a register\n");
  97         if (handleReject() == RejectAndGiveUp)
  98           return false;
  99         break;
 100       }
 101       if (Register::isPhysicalRegister(MO.getReg())) {
 102         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 103                         dbgs() << CurrentIdx << ": Is a physical register\n");
 104         if (handleReject() == RejectAndGiveUp)
 105           return false;
 106         break;
 107       }
 108
 109       MachineInstr *NewMI = MRI.getVRegDef(MO.getReg());
 110       if ((size_t)NewInsnID < State.MIs.size())
 111         State.MIs[NewInsnID] = NewMI;
 112       else {
 113         assert((size_t)NewInsnID == State.MIs.size() &&
 114                "Expected to store MIs in order");
 115         State.MIs.push_back(NewMI);
 116       }
 117       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 118                       dbgs() << CurrentIdx << ": MIs[" << NewInsnID
 119                              << "] = GIM_RecordInsn(" << InsnID << ", " << OpIdx
 120                              << ")\n");
 121       break;
 122     }
 123
 124     case GIM_CheckFeatures: {
 125       int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
 126       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 127                       dbgs() << CurrentIdx
 128                              << ": GIM_CheckFeatures(ExpectedBitsetID="
 129                              << ExpectedBitsetID << ")\n");
 130       if ((AvailableFeatures & ISelInfo.FeatureBitsets[ExpectedBitsetID]) !=
 131           ISelInfo.FeatureBitsets[ExpectedBitsetID]) {
 132         if (handleReject() == RejectAndGiveUp)
 133           return false;
 134       }
 135       break;
 136     }
 137
 138     case GIM_CheckOpcode: {
 139       int64_t InsnID = MatchTable[CurrentIdx++];
 140       int64_t Expected = MatchTable[CurrentIdx++];
 141
 142       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 143       unsigned Opcode = State.MIs[InsnID]->getOpcode();
 144
 145       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 146                       dbgs() << CurrentIdx << ": GIM_CheckOpcode(MIs[" << InsnID
 147                              << "], ExpectedOpcode=" << Expected
 148                              << ") // Got=" << Opcode << "\n");
 149       if (Opcode != Expected) {
 150         if (handleReject() == RejectAndGiveUp)
 151           return false;
 152       }
 153       break;
 154     }
 155
 156     case GIM_SwitchOpcode: {
 157       int64_t InsnID = MatchTable[CurrentIdx++];
 158       int64_t LowerBound = MatchTable[CurrentIdx++];
 159       int64_t UpperBound = MatchTable[CurrentIdx++];
 160       int64_t Default = MatchTable[CurrentIdx++];
 161
 162       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 163       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
 164
 165       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 166         dbgs() << CurrentIdx << ": GIM_SwitchOpcode(MIs[" << InsnID << "], ["
 167                << LowerBound << ", " << UpperBound << "), Default=" << Default
 168                << ", JumpTable...) // Got=" << Opcode << "\n";
 169       });
 170       if (Opcode < LowerBound || UpperBound <= Opcode) {
 171         CurrentIdx = Default;
 172         break;
 173       }
 174       CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
 175       if (!CurrentIdx) {
 176         CurrentIdx = Default;
 177         break;
 178       }
 179       OnFailResumeAt.push_back(Default);
 180       break;
 181     }
 182
 183     case GIM_SwitchType: {
 184       int64_t InsnID = MatchTable[CurrentIdx++];
 185       int64_t OpIdx = MatchTable[CurrentIdx++];
 186       int64_t LowerBound = MatchTable[CurrentIdx++];
 187       int64_t UpperBound = MatchTable[CurrentIdx++];
 188       int64_t Default = MatchTable[CurrentIdx++];
 189
 190       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 191       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 192
 193       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), {
 194         dbgs() << CurrentIdx << ": GIM_SwitchType(MIs[" << InsnID
 195                << "]->getOperand(" << OpIdx << "), [" << LowerBound << ", "
 196                << UpperBound << "), Default=" << Default
 197                << ", JumpTable...) // Got=";
 198         if (!MO.isReg())
 199           dbgs() << "Not a VReg\n";
 200         else
 201           dbgs() << MRI.getType(MO.getReg()) << "\n";
 202       });
 203       if (!MO.isReg()) {
 204         CurrentIdx = Default;
 205         break;
 206       }
 207       const LLT Ty = MRI.getType(MO.getReg());
 208       const auto TyI = ISelInfo.TypeIDMap.find(Ty);
 209       if (TyI == ISelInfo.TypeIDMap.end()) {
 210         CurrentIdx = Default;
 211         break;
 212       }
 213       const int64_t TypeID = TyI->second;
 214       if (TypeID < LowerBound || UpperBound <= TypeID) {
 215         CurrentIdx = Default;
 216         break;
 217       }
 218       CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
 219       if (!CurrentIdx) {
 220         CurrentIdx = Default;
 221         break;
 222       }
 223       OnFailResumeAt.push_back(Default);
 224       break;
 225     }
 226
 227     case GIM_CheckNumOperands: {
 228       int64_t InsnID = MatchTable[CurrentIdx++];
 229       int64_t Expected = MatchTable[CurrentIdx++];
 230       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 231                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
 232                              << InsnID << "], Expected=" << Expected << ")\n");
 233       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 234       if (State.MIs[InsnID]->getNumOperands() != Expected) {
 235         if (handleReject() == RejectAndGiveUp)
 236           return false;
 237       }
 238       break;
 239     }
 240     case GIM_CheckI64ImmPredicate: {
 241       int64_t InsnID = MatchTable[CurrentIdx++];
 242       int64_t Predicate = MatchTable[CurrentIdx++];
 243       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 244                       dbgs()
 245                           << CurrentIdx << ": GIM_CheckI64ImmPredicate(MIs["
 246                           << InsnID << "], Predicate=" << Predicate << ")\n");
 247       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 248       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 249              "Expected G_CONSTANT");
 250       assert(Predicate > GIPFP_I64_Invalid && "Expected a valid predicate");
 251       int64_t Value = 0;
 252       if (State.MIs[InsnID]->getOperand(1).isCImm())
 253         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getSExtValue();
 254       else if (State.MIs[InsnID]->getOperand(1).isImm())
 255         Value = State.MIs[InsnID]->getOperand(1).getImm();
 256       else
 257         llvm_unreachable("Expected Imm or CImm operand");
 258
 259       if (!testImmPredicate_I64(Predicate, Value))
 260         if (handleReject() == RejectAndGiveUp)
 261           return false;
 262       break;
 263     }
 264     case GIM_CheckAPIntImmPredicate: {
 265       int64_t InsnID = MatchTable[CurrentIdx++];
 266       int64_t Predicate = MatchTable[CurrentIdx++];
 267       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 268                       dbgs()
 269                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
 270                           << InsnID << "], Predicate=" << Predicate << ")\n");
 271       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 272       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
 273              "Expected G_CONSTANT");
 274       assert(Predicate > GIPFP_APInt_Invalid && "Expected a valid predicate");
 275       APInt Value;
 276       if (State.MIs[InsnID]->getOperand(1).isCImm())
 277         Value = State.MIs[InsnID]->getOperand(1).getCImm()->getValue();
 278       else
 279         llvm_unreachable("Expected Imm or CImm operand");
 280
 281       if (!testImmPredicate_APInt(Predicate, Value))
 282         if (handleReject() == RejectAndGiveUp)
 283           return false;
 284       break;
 285     }
 286     case GIM_CheckAPFloatImmPredicate: {
 287       int64_t InsnID = MatchTable[CurrentIdx++];
 288       int64_t Predicate = MatchTable[CurrentIdx++];
 289       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 290                       dbgs()
 291                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
 292                           << InsnID << "], Predicate=" << Predicate << ")\n");
 293       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 294       assert(State.MIs[InsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
 295              "Expected G_FCONSTANT");
 296       assert(State.MIs[InsnID]->getOperand(1).isFPImm() && "Expected FPImm operand");
 297       assert(Predicate > GIPFP_APFloat_Invalid && "Expected a valid predicate");
 298       APFloat Value = State.MIs[InsnID]->getOperand(1).getFPImm()->getValueAPF();
 299
 300       if (!testImmPredicate_APFloat(Predicate, Value))
 301         if (handleReject() == RejectAndGiveUp)
 302           return false;
 303       break;
 304     }
 305     case GIM_CheckCxxInsnPredicate: {
 306       int64_t InsnID = MatchTable[CurrentIdx++];
 307       int64_t Predicate = MatchTable[CurrentIdx++];
 308       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 309                       dbgs()
 310                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
 311                           << InsnID << "], Predicate=" << Predicate << ")\n");
 312       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 313       assert(Predicate > GIPFP_MI_Invalid && "Expected a valid predicate");
 314
 315       if (!testMIPredicate_MI(Predicate, *State.MIs[InsnID]))
 316         if (handleReject() == RejectAndGiveUp)
 317           return false;
 318       break;
 319     }
 320     case GIM_CheckAtomicOrdering: {
 321       int64_t InsnID = MatchTable[CurrentIdx++];
 322       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 323       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 324                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
 325                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 326       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 327       if (!State.MIs[InsnID]->hasOneMemOperand())
 328         if (handleReject() == RejectAndGiveUp)
 329           return false;
 330
 331       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 332         if (MMO->getOrdering() != Ordering)
 333           if (handleReject() == RejectAndGiveUp)
 334             return false;
 335       break;
 336     }
 337     case GIM_CheckAtomicOrderingOrStrongerThan: {
 338       int64_t InsnID = MatchTable[CurrentIdx++];
 339       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 340       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 341                       dbgs() << CurrentIdx
 342                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
 343                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 344       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 345       if (!State.MIs[InsnID]->hasOneMemOperand())
 346         if (handleReject() == RejectAndGiveUp)
 347           return false;
 348
 349       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 350         if (!isAtLeastOrStrongerThan(MMO->getOrdering(), Ordering))
 351           if (handleReject() == RejectAndGiveUp)
 352             return false;
 353       break;
 354     }
 355     case GIM_CheckAtomicOrderingWeakerThan: {
 356       int64_t InsnID = MatchTable[CurrentIdx++];
 357       AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
 358       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 359                       dbgs() << CurrentIdx
 360                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
 361                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
 362       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 363       if (!State.MIs[InsnID]->hasOneMemOperand())
 364         if (handleReject() == RejectAndGiveUp)
 365           return false;
 366
 367       for (const auto &MMO : State.MIs[InsnID]->memoperands())
 368         if (!isStrongerThan(Ordering, MMO->getOrdering()))
 369           if (handleReject() == RejectAndGiveUp)
 370             return false;
 371       break;
 372     }
 373     case GIM_CheckMemoryAddressSpace: {
 374       int64_t InsnID = MatchTable[CurrentIdx++];
 375       int64_t MMOIdx = MatchTable[CurrentIdx++];
 376       // This accepts a list of possible address spaces.
 377       const int NumAddrSpace = MatchTable[CurrentIdx++];
 378
 379       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 380         if (handleReject() == RejectAndGiveUp)
 381           return false;
 382         break;
 383       }
 384
 385       // Need to still jump to the end of the list of address spaces if we find
 386       // a match earlier.
 387       const uint64_t LastIdx = CurrentIdx + NumAddrSpace;
 388
 389       const MachineMemOperand *MMO
 390         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 391       const unsigned MMOAddrSpace = MMO->getAddrSpace();
 392
 393       bool Success = false;
 394       for (int I = 0; I != NumAddrSpace; ++I) {
 395         unsigned AddrSpace = MatchTable[CurrentIdx++];
 396         DEBUG_WITH_TYPE(
 397           TgtInstructionSelector::getName(),
 398           dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
 399                  << AddrSpace << '\n');
 400
 401         if (AddrSpace == MMOAddrSpace) {
 402           Success = true;
 403           break;
 404         }
 405       }
 406
 407       CurrentIdx = LastIdx;
 408       if (!Success && handleReject() == RejectAndGiveUp)
 409         return false;
 410       break;
 411     }
 412     case GIM_CheckMemoryAlignment: {
 413       int64_t InsnID = MatchTable[CurrentIdx++];
 414       int64_t MMOIdx = MatchTable[CurrentIdx++];
 415       unsigned MinAlign = MatchTable[CurrentIdx++];
 416
 417       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 418
 419       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 420         if (handleReject() == RejectAndGiveUp)
 421           return false;
 422         break;
 423       }
 424
 425       MachineMemOperand *MMO
 426         = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 427       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 428                       dbgs() << CurrentIdx << ": GIM_CheckMemoryAlignment"
 429                       << "(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 430                       << ")->getAlignment() >= " << MinAlign << ")\n");
 431       if (MMO->getAlignment() < MinAlign && handleReject() == RejectAndGiveUp)
 432         return false;
 433
 434       break;
 435     }
 436     case GIM_CheckMemorySizeEqualTo: {
 437       int64_t InsnID = MatchTable[CurrentIdx++];
 438       int64_t MMOIdx = MatchTable[CurrentIdx++];
 439       uint64_t Size = MatchTable[CurrentIdx++];
 440
 441       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 442                       dbgs() << CurrentIdx
 443                              << ": GIM_CheckMemorySizeEqual(MIs[" << InsnID
 444                              << "]->memoperands() + " << MMOIdx
 445                              << ", Size=" << Size << ")\n");
 446       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 447
 448       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 449         if (handleReject() == RejectAndGiveUp)
 450           return false;
 451         break;
 452       }
 453
 454       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 455
 456       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 457                       dbgs() << MMO->getSize() << " bytes vs " << Size
 458                              << " bytes\n");
 459       if (MMO->getSize() != Size)
 460         if (handleReject() == RejectAndGiveUp)
 461           return false;
 462
 463       break;
 464     }
 465     case GIM_CheckMemorySizeEqualToLLT:
 466     case GIM_CheckMemorySizeLessThanLLT:
 467     case GIM_CheckMemorySizeGreaterThanLLT: {
 468       int64_t InsnID = MatchTable[CurrentIdx++];
 469       int64_t MMOIdx = MatchTable[CurrentIdx++];
 470       int64_t OpIdx = MatchTable[CurrentIdx++];
 471
 472       DEBUG_WITH_TYPE(
 473           TgtInstructionSelector::getName(),
 474           dbgs() << CurrentIdx << ": GIM_CheckMemorySize"
 475                  << (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT
 476                          ? "EqualTo"
 477                          : MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT
 478                                ? "GreaterThan"
 479                                : "LessThan")
 480                  << "LLT(MIs[" << InsnID << "]->memoperands() + " << MMOIdx
 481                  << ", OpIdx=" << OpIdx << ")\n");
 482       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 483
 484       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 485       if (!MO.isReg()) {
 486         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 487                         dbgs() << CurrentIdx << ": Not a register\n");
 488         if (handleReject() == RejectAndGiveUp)
 489           return false;
 490         break;
 491       }
 492
 493       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
 494         if (handleReject() == RejectAndGiveUp)
 495           return false;
 496         break;
 497       }
 498
 499       MachineMemOperand *MMO = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx);
 500
 501       unsigned Size = MRI.getType(MO.getReg()).getSizeInBits();
 502       if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT &&
 503           MMO->getSizeInBits() != Size) {
 504         if (handleReject() == RejectAndGiveUp)
 505           return false;
 506       } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT &&
 507                  MMO->getSizeInBits() >= Size) {
 508         if (handleReject() == RejectAndGiveUp)
 509           return false;
 510       } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT &&
 511                  MMO->getSizeInBits() <= Size)
 512         if (handleReject() == RejectAndGiveUp)
 513           return false;
 514
 515       break;
 516     }
 517     case GIM_CheckType: {
 518       int64_t InsnID = MatchTable[CurrentIdx++];
 519       int64_t OpIdx = MatchTable[CurrentIdx++];
 520       int64_t TypeID = MatchTable[CurrentIdx++];
 521       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 522                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
 523                              << "]->getOperand(" << OpIdx
 524                              << "), TypeID=" << TypeID << ")\n");
 525       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 526       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 527       if (!MO.isReg() ||
 528           MRI.getType(MO.getReg()) != ISelInfo.TypeObjects[TypeID]) {
 529         if (handleReject() == RejectAndGiveUp)
 530           return false;
 531       }
 532       break;
 533     }
 534     case GIM_CheckPointerToAny: {
 535       int64_t InsnID = MatchTable[CurrentIdx++];
 536       int64_t OpIdx = MatchTable[CurrentIdx++];
 537       int64_t SizeInBits = MatchTable[CurrentIdx++];
 538
 539       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 540                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
 541                              << InsnID << "]->getOperand(" << OpIdx
 542                              << "), SizeInBits=" << SizeInBits << ")\n");
 543       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 544       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 545       const LLT Ty = MRI.getType(MO.getReg());
 546
 547       // iPTR must be looked up in the target.
 548       if (SizeInBits == 0) {
 549         MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent();
 550         const unsigned AddrSpace = Ty.getAddressSpace();
 551         SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace);
 552       }
 553
 554       assert(SizeInBits != 0 && "Pointer size must be known");
 555
 556       if (MO.isReg()) {
 557         if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits)
 558           if (handleReject() == RejectAndGiveUp)
 559             return false;
 560       } else if (handleReject() == RejectAndGiveUp)
 561         return false;
 562
 563       break;
 564     }
 565     case GIM_CheckRegBankForClass: {
 566       int64_t InsnID = MatchTable[CurrentIdx++];
 567       int64_t OpIdx = MatchTable[CurrentIdx++];
 568       int64_t RCEnum = MatchTable[CurrentIdx++];
 569       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 570                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
 571                              << InsnID << "]->getOperand(" << OpIdx
 572                              << "), RCEnum=" << RCEnum << ")\n");
 573       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 574       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 575       if (!MO.isReg() ||
 576           &RBI.getRegBankFromRegClass(*TRI.getRegClass(RCEnum)) !=
 577               RBI.getRegBank(MO.getReg(), MRI, TRI)) {
 578         if (handleReject() == RejectAndGiveUp)
 579           return false;
 580       }
 581       break;
 582     }
 583
 584     case GIM_CheckComplexPattern: {
 585       int64_t InsnID = MatchTable[CurrentIdx++];
 586       int64_t OpIdx = MatchTable[CurrentIdx++];
 587       int64_t RendererID = MatchTable[CurrentIdx++];
 588       int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
 589       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 590                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
 591                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
 592                              << "]->getOperand(" << OpIdx
 593                              << "), ComplexPredicateID=" << ComplexPredicateID
 594                              << ")\n");
 595       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 596       // FIXME: Use std::invoke() when it's available.
 597       ComplexRendererFns Renderer =
 598           (ISel.*ISelInfo.ComplexPredicates[ComplexPredicateID])(
 599               State.MIs[InsnID]->getOperand(OpIdx));
 600       if (Renderer.hasValue())
 601         State.Renderers[RendererID] = Renderer.getValue();
 602       else
 603         if (handleReject() == RejectAndGiveUp)
 604           return false;
 605       break;
 606     }
 607
 608     case GIM_CheckConstantInt: {
 609       int64_t InsnID = MatchTable[CurrentIdx++];
 610       int64_t OpIdx = MatchTable[CurrentIdx++];
 611       int64_t Value = MatchTable[CurrentIdx++];
 612       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 613                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
 614                              << InsnID << "]->getOperand(" << OpIdx
 615                              << "), Value=" << Value << ")\n");
 616       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 617       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 618       if (MO.isReg()) {
 619         // isOperandImmEqual() will sign-extend to 64-bits, so should we.
 620         LLT Ty = MRI.getType(MO.getReg());
 621         Value = SignExtend64(Value, Ty.getSizeInBits());
 622
 623         if (!isOperandImmEqual(MO, Value, MRI)) {
 624           if (handleReject() == RejectAndGiveUp)
 625             return false;
 626         }
 627       } else if (handleReject() == RejectAndGiveUp)
 628         return false;
 629
 630       break;
 631     }
 632
 633     case GIM_CheckLiteralInt: {
 634       int64_t InsnID = MatchTable[CurrentIdx++];
 635       int64_t OpIdx = MatchTable[CurrentIdx++];
 636       int64_t Value = MatchTable[CurrentIdx++];
 637       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 638                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
 639                              << InsnID << "]->getOperand(" << OpIdx
 640                              << "), Value=" << Value << ")\n");
 641       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 642       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 643       if (!MO.isCImm() || !MO.getCImm()->equalsInt(Value)) {
 644         if (handleReject() == RejectAndGiveUp)
 645           return false;
 646       }
 647       break;
 648     }
 649
 650     case GIM_CheckIntrinsicID: {
 651       int64_t InsnID = MatchTable[CurrentIdx++];
 652       int64_t OpIdx = MatchTable[CurrentIdx++];
 653       int64_t Value = MatchTable[CurrentIdx++];
 654       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 655                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
 656                              << InsnID << "]->getOperand(" << OpIdx
 657                              << "), Value=" << Value << ")\n");
 658       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 659       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 660       if (!MO.isIntrinsicID() || MO.getIntrinsicID() != Value)
 661         if (handleReject() == RejectAndGiveUp)
 662           return false;
 663       break;
 664     }
 665     case GIM_CheckCmpPredicate: {
 666       int64_t InsnID = MatchTable[CurrentIdx++];
 667       int64_t OpIdx = MatchTable[CurrentIdx++];
 668       int64_t Value = MatchTable[CurrentIdx++];
 669       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 670                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
 671                              << InsnID << "]->getOperand(" << OpIdx
 672                              << "), Value=" << Value << ")\n");
 673       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 674       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
 675       if (!MO.isPredicate() || MO.getPredicate() != Value)
 676         if (handleReject() == RejectAndGiveUp)
 677           return false;
 678       break;
 679     }
 680     case GIM_CheckIsMBB: {
 681       int64_t InsnID = MatchTable[CurrentIdx++];
 682       int64_t OpIdx = MatchTable[CurrentIdx++];
 683       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 684                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
 685                              << "]->getOperand(" << OpIdx << "))\n");
 686       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 687       if (!State.MIs[InsnID]->getOperand(OpIdx).isMBB()) {
 688         if (handleReject() == RejectAndGiveUp)
 689           return false;
 690       }
 691       break;
 692     }
 693
 694     case GIM_CheckIsSafeToFold: {
 695       int64_t InsnID = MatchTable[CurrentIdx++];
 696       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 697                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
 698                              << InsnID << "])\n");
 699       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 700       if (!isObviouslySafeToFold(*State.MIs[InsnID], *State.MIs[0])) {
 701         if (handleReject() == RejectAndGiveUp)
 702           return false;
 703       }
 704       break;
 705     }
 706     case GIM_CheckIsSameOperand: {
 707       int64_t InsnID = MatchTable[CurrentIdx++];
 708       int64_t OpIdx = MatchTable[CurrentIdx++];
 709       int64_t OtherInsnID = MatchTable[CurrentIdx++];
 710       int64_t OtherOpIdx = MatchTable[CurrentIdx++];
 711       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 712                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
 713                              << InsnID << "][" << OpIdx << "], MIs["
 714                              << OtherInsnID << "][" << OtherOpIdx << "])\n");
 715       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 716       assert(State.MIs[OtherInsnID] != nullptr && "Used insn before defined");
 717       if (!State.MIs[InsnID]->getOperand(OpIdx).isIdenticalTo(
 718               State.MIs[OtherInsnID]->getOperand(OtherOpIdx))) {
 719         if (handleReject() == RejectAndGiveUp)
 720           return false;
 721       }
 722       break;
 723     }
 724     case GIM_Reject:
 725       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 726                       dbgs() << CurrentIdx << ": GIM_Reject\n");
 727       if (handleReject() == RejectAndGiveUp)
 728         return false;
 729       break;
 730
 731     case GIR_MutateOpcode: {
 732       int64_t OldInsnID = MatchTable[CurrentIdx++];
 733       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 734       int64_t NewOpcode = MatchTable[CurrentIdx++];
 735       if (NewInsnID >= OutMIs.size())
 736         OutMIs.resize(NewInsnID + 1);
 737
 738       OutMIs[NewInsnID] = MachineInstrBuilder(*State.MIs[OldInsnID]->getMF(),
 739                                               State.MIs[OldInsnID]);
 740       OutMIs[NewInsnID]->setDesc(TII.get(NewOpcode));
 741       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 742                       dbgs() << CurrentIdx << ": GIR_MutateOpcode(OutMIs["
 743                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 744                              << NewOpcode << ")\n");
 745       break;
 746     }
 747
 748     case GIR_BuildMI: {
 749       uint64_t NewInsnID = MatchTable[CurrentIdx++];
 750       int64_t Opcode = MatchTable[CurrentIdx++];
 751       if (NewInsnID >= OutMIs.size())
 752         OutMIs.resize(NewInsnID + 1);
 753
 754       OutMIs[NewInsnID] = BuildMI(*State.MIs[0]->getParent(), State.MIs[0],
 755                                   State.MIs[0]->getDebugLoc(), TII.get(Opcode));
 756       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 757                       dbgs() << CurrentIdx << ": GIR_BuildMI(OutMIs["
 758                              << NewInsnID << "], " << Opcode << ")\n");
 759       break;
 760     }
 761
 762     case GIR_Copy: {
 763       int64_t NewInsnID = MatchTable[CurrentIdx++];
 764       int64_t OldInsnID = MatchTable[CurrentIdx++];
 765       int64_t OpIdx = MatchTable[CurrentIdx++];
 766       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 767       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
 768       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 769                       dbgs()
 770                           << CurrentIdx << ": GIR_Copy(OutMIs[" << NewInsnID
 771                           << "], MIs[" << OldInsnID << "], " << OpIdx << ")\n");
 772       break;
 773     }
 774
 775     case GIR_CopyOrAddZeroReg: {
 776       int64_t NewInsnID = MatchTable[CurrentIdx++];
 777       int64_t OldInsnID = MatchTable[CurrentIdx++];
 778       int64_t OpIdx = MatchTable[CurrentIdx++];
 779       int64_t ZeroReg = MatchTable[CurrentIdx++];
 780       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 781       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
 782       if (isOperandImmEqual(MO, 0, MRI))
 783         OutMIs[NewInsnID].addReg(ZeroReg);
 784       else
 785         OutMIs[NewInsnID].add(MO);
 786       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 787                       dbgs() << CurrentIdx << ": GIR_CopyOrAddZeroReg(OutMIs["
 788                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 789                              << OpIdx << ", " << ZeroReg << ")\n");
 790       break;
 791     }
 792
 793     case GIR_CopySubReg: {
 794       int64_t NewInsnID = MatchTable[CurrentIdx++];
 795       int64_t OldInsnID = MatchTable[CurrentIdx++];
 796       int64_t OpIdx = MatchTable[CurrentIdx++];
 797       int64_t SubRegIdx = MatchTable[CurrentIdx++];
 798       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 799       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
 800                                0, SubRegIdx);
 801       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 802                       dbgs() << CurrentIdx << ": GIR_CopySubReg(OutMIs["
 803                              << NewInsnID << "], MIs[" << OldInsnID << "], "
 804                              << OpIdx << ", " << SubRegIdx << ")\n");
 805       break;
 806     }
 807
 808     case GIR_AddImplicitDef: {
 809       int64_t InsnID = MatchTable[CurrentIdx++];
 810       int64_t RegNum = MatchTable[CurrentIdx++];
 811       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 812       OutMIs[InsnID].addDef(RegNum, RegState::Implicit);
 813       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 814                       dbgs() << CurrentIdx << ": GIR_AddImplicitDef(OutMIs["
 815                              << InsnID << "], " << RegNum << ")\n");
 816       break;
 817     }
 818
 819     case GIR_AddImplicitUse: {
 820       int64_t InsnID = MatchTable[CurrentIdx++];
 821       int64_t RegNum = MatchTable[CurrentIdx++];
 822       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 823       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
 824       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 825                       dbgs() << CurrentIdx << ": GIR_AddImplicitUse(OutMIs["
 826                              << InsnID << "], " << RegNum << ")\n");
 827       break;
 828     }
 829
 830     case GIR_AddRegister: {
 831       int64_t InsnID = MatchTable[CurrentIdx++];
 832       int64_t RegNum = MatchTable[CurrentIdx++];
 833       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 834       OutMIs[InsnID].addReg(RegNum);
 835       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 836                       dbgs() << CurrentIdx << ": GIR_AddRegister(OutMIs["
 837                              << InsnID << "], " << RegNum << ")\n");
 838       break;
 839     }
 840
 841     case GIR_AddTempRegister: {
 842       int64_t InsnID = MatchTable[CurrentIdx++];
 843       int64_t TempRegID = MatchTable[CurrentIdx++];
 844       uint64_t TempRegFlags = MatchTable[CurrentIdx++];
 845       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 846       OutMIs[InsnID].addReg(State.TempRegisters[TempRegID], TempRegFlags);
 847       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 848                       dbgs() << CurrentIdx << ": GIR_AddTempRegister(OutMIs["
 849                              << InsnID << "], TempRegisters[" << TempRegID
 850                              << "], " << TempRegFlags << ")\n");
 851       break;
 852     }
 853
 854     case GIR_AddImm: {
 855       int64_t InsnID = MatchTable[CurrentIdx++];
 856       int64_t Imm = MatchTable[CurrentIdx++];
 857       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 858       OutMIs[InsnID].addImm(Imm);
 859       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 860                       dbgs() << CurrentIdx << ": GIR_AddImm(OutMIs[" << InsnID
 861                              << "], " << Imm << ")\n");
 862       break;
 863     }
 864
 865     case GIR_ComplexRenderer: {
 866       int64_t InsnID = MatchTable[CurrentIdx++];
 867       int64_t RendererID = MatchTable[CurrentIdx++];
 868       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 869       for (const auto &RenderOpFn : State.Renderers[RendererID])
 870         RenderOpFn(OutMIs[InsnID]);
 871       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 872                       dbgs() << CurrentIdx << ": GIR_ComplexRenderer(OutMIs["
 873                              << InsnID << "], " << RendererID << ")\n");
 874       break;
 875     }
 876     case GIR_ComplexSubOperandRenderer: {
 877       int64_t InsnID = MatchTable[CurrentIdx++];
 878       int64_t RendererID = MatchTable[CurrentIdx++];
 879       int64_t RenderOpID = MatchTable[CurrentIdx++];
 880       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 881       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
 882       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 883                       dbgs() << CurrentIdx
 884                              << ": GIR_ComplexSubOperandRenderer(OutMIs["
 885                              << InsnID << "], " << RendererID << ", "
 886                              << RenderOpID << ")\n");
 887       break;
 888     }
 889
 890     case GIR_CopyConstantAsSImm: {
 891       int64_t NewInsnID = MatchTable[CurrentIdx++];
 892       int64_t OldInsnID = MatchTable[CurrentIdx++];
 893       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 894       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT");
 895       if (State.MIs[OldInsnID]->getOperand(1).isCImm()) {
 896         OutMIs[NewInsnID].addImm(
 897             State.MIs[OldInsnID]->getOperand(1).getCImm()->getSExtValue());
 898       } else if (State.MIs[OldInsnID]->getOperand(1).isImm())
 899         OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(1));
 900       else
 901         llvm_unreachable("Expected Imm or CImm operand");
 902       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 903                       dbgs() << CurrentIdx << ": GIR_CopyConstantAsSImm(OutMIs["
 904                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 905       break;
 906     }
 907
 908     // TODO: Needs a test case once we have a pattern that uses this.
 909     case GIR_CopyFConstantAsFPImm: {
 910       int64_t NewInsnID = MatchTable[CurrentIdx++];
 911       int64_t OldInsnID = MatchTable[CurrentIdx++];
 912       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
 913       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT && "Expected G_FCONSTANT");
 914       if (State.MIs[OldInsnID]->getOperand(1).isFPImm())
 915         OutMIs[NewInsnID].addFPImm(
 916             State.MIs[OldInsnID]->getOperand(1).getFPImm());
 917       else
 918         llvm_unreachable("Expected FPImm operand");
 919       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 920                       dbgs() << CurrentIdx << ": GIR_CopyFPConstantAsFPImm(OutMIs["
 921                              << NewInsnID << "], MIs[" << OldInsnID << "])\n");
 922       break;
 923     }
 924
 925     case GIR_CustomRenderer: {
 926       int64_t InsnID = MatchTable[CurrentIdx++];
 927       int64_t OldInsnID = MatchTable[CurrentIdx++];
 928       int64_t RendererFnID = MatchTable[CurrentIdx++];
 929       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 930       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 931                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
 932                              << InsnID << "], MIs[" << OldInsnID << "], "
 933                              << RendererFnID << ")\n");
 934       (ISel.*ISelInfo.CustomRenderers[RendererFnID])(OutMIs[InsnID],
 935                                                      *State.MIs[OldInsnID]);
 936       break;
 937     }
 938     case GIR_ConstrainOperandRC: {
 939       int64_t InsnID = MatchTable[CurrentIdx++];
 940       int64_t OpIdx = MatchTable[CurrentIdx++];
 941       int64_t RCEnum = MatchTable[CurrentIdx++];
 942       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 943       constrainOperandRegToRegClass(*OutMIs[InsnID].getInstr(), OpIdx,
 944                                     *TRI.getRegClass(RCEnum), TII, TRI, RBI);
 945       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 946                       dbgs() << CurrentIdx << ": GIR_ConstrainOperandRC(OutMIs["
 947                              << InsnID << "], " << OpIdx << ", " << RCEnum
 948                              << ")\n");
 949       break;
 950     }
 951
 952     case GIR_ConstrainSelectedInstOperands: {
 953       int64_t InsnID = MatchTable[CurrentIdx++];
 954       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 955       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
 956                                        RBI);
 957       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 958                       dbgs() << CurrentIdx
 959                              << ": GIR_ConstrainSelectedInstOperands(OutMIs["
 960                              << InsnID << "])\n");
 961       break;
 962     }
 963
 964     case GIR_MergeMemOperands: {
 965       int64_t InsnID = MatchTable[CurrentIdx++];
 966       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 967
 968       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 969                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
 970                              << InsnID << "]");
 971       int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
 972       while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
 973              GIU_MergeMemOperands_EndOfList) {
 974         DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 975                         dbgs() << ", MIs[" << MergeInsnID << "]");
 976         for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
 977           OutMIs[InsnID].addMemOperand(MMO);
 978       }
 979       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(), dbgs() << ")\n");
 980       break;
 981     }
 982
 983     case GIR_EraseFromParent: {
 984       int64_t InsnID = MatchTable[CurrentIdx++];
 985       assert(State.MIs[InsnID] &&
 986              "Attempted to erase an undefined instruction");
 987       State.MIs[InsnID]->eraseFromParent();
 988       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
 989                       dbgs() << CurrentIdx << ": GIR_EraseFromParent(MIs["
 990                              << InsnID << "])\n");
 991       break;
 992     }
 993
 994     case GIR_MakeTempReg: {
 995       int64_t TempRegID = MatchTable[CurrentIdx++];
 996       int64_t TypeID = MatchTable[CurrentIdx++];
 997
 998       State.TempRegisters[TempRegID] =
 999           MRI.createGenericVirtualRegister(ISelInfo.TypeObjects[TypeID]);
1000       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1001                       dbgs() << CurrentIdx << ": TempRegs[" << TempRegID
1002                              << "] = GIR_MakeTempReg(" << TypeID << ")\n");
1003       break;
1004     }
1005
1006     case GIR_Coverage: {
1007       int64_t RuleID = MatchTable[CurrentIdx++];
1008       CoverageInfo.setCovered(RuleID);
1009
1010       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1011                       dbgs()
1012                           << CurrentIdx << ": GIR_Coverage(" << RuleID << ")");
1013       break;
1014     }
1015
1016     case GIR_Done:
1017       DEBUG_WITH_TYPE(TgtInstructionSelector::getName(),
1018                       dbgs() << CurrentIdx << ": GIR_Done\n");
1019       return true;
1020
1021     default:
1022       llvm_unreachable("Unexpected command");
1023     }
1024   }
1025 }
1026
1027 } // end namespace llvm
1028
1029 #endif // LLVM_CODEGEN_GLOBALISEL_INSTRUCTIONSELECTORIMPL_H