llvm/utils/TableGen/GlobalISelMatchTable.cpp

   1 //===- GlobalISelMatchTable.cpp -------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "GlobalISelMatchTable.h"
  10 #include "CodeGenInstruction.h"
  11 #include "CodeGenRegisters.h"
  12 #include "llvm/ADT/Statistic.h"
  13 #include "llvm/Support/Debug.h"
  14 #include "llvm/Support/ScopedPrinter.h"
  15 #include "llvm/Support/raw_ostream.h"
  16 #include "llvm/TableGen/Error.h"
  17
  18 #define DEBUG_TYPE "gi-match-table"
  19
  20 STATISTIC(NumPatternEmitted, "Number of patterns emitted");
  21
  22 namespace llvm {
  23 namespace gi {
  24
  25 namespace {
  26
  27 Error failUnsupported(const Twine &Reason) {
  28   return make_error<StringError>(Reason, inconvertibleErrorCode());
  29 }
  30
  31 /// Get the name of the enum value used to number the predicate function.
  32 std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
  33   if (Predicate.hasGISelPredicateCode())
  34     return "GICXXPred_MI_" + Predicate.getFnName();
  35   return "GICXXPred_" + Predicate.getImmTypeIdentifier().str() + "_" +
  36          Predicate.getFnName();
  37 }
  38
  39 std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
  40   return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
  41 }
  42 } // namespace
  43
  44 //===- Helpers ------------------------------------------------------------===//
  45
  46 std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset,
  47                                     int HwModeIdx) {
  48   std::string Name = "GIFBS";
  49   for (const auto &Feature : FeatureBitset)
  50     Name += ("_" + Feature->getName()).str();
  51   if (HwModeIdx >= 0)
  52     Name += ("_HwMode" + std::to_string(HwModeIdx));
  53   return Name;
  54 }
  55
  56 template <class GroupT>
  57 std::vector<Matcher *>
  58 optimizeRules(ArrayRef<Matcher *> Rules,
  59               std::vector<std::unique_ptr<Matcher>> &MatcherStorage) {
  60
  61   std::vector<Matcher *> OptRules;
  62   std::unique_ptr<GroupT> CurrentGroup = std::make_unique<GroupT>();
  63   assert(CurrentGroup->empty() && "Newly created group isn't empty!");
  64   unsigned NumGroups = 0;
  65
  66   auto ProcessCurrentGroup = [&]() {
  67     if (CurrentGroup->empty())
  68       // An empty group is good to be reused:
  69       return;
  70
  71     // If the group isn't large enough to provide any benefit, move all the
  72     // added rules out of it and make sure to re-create the group to properly
  73     // re-initialize it:
  74     if (CurrentGroup->size() < 2)
  75       append_range(OptRules, CurrentGroup->matchers());
  76     else {
  77       CurrentGroup->finalize();
  78       OptRules.push_back(CurrentGroup.get());
  79       MatcherStorage.emplace_back(std::move(CurrentGroup));
  80       ++NumGroups;
  81     }
  82     CurrentGroup = std::make_unique<GroupT>();
  83   };
  84   for (Matcher *Rule : Rules) {
  85     // Greedily add as many matchers as possible to the current group:
  86     if (CurrentGroup->addMatcher(*Rule))
  87       continue;
  88
  89     ProcessCurrentGroup();
  90     assert(CurrentGroup->empty() && "A group wasn't properly re-initialized");
  91
  92     // Try to add the pending matcher to a newly created empty group:
  93     if (!CurrentGroup->addMatcher(*Rule))
  94       // If we couldn't add the matcher to an empty group, that group type
  95       // doesn't support that kind of matchers at all, so just skip it:
  96       OptRules.push_back(Rule);
  97   }
  98   ProcessCurrentGroup();
  99
 100   LLVM_DEBUG(dbgs() << "NumGroups: " << NumGroups << "\n");
 101   (void)NumGroups;
 102   assert(CurrentGroup->empty() && "The last group wasn't properly processed");
 103   return OptRules;
 104 }
 105
 106 template std::vector<Matcher *> optimizeRules<GroupMatcher>(
 107     ArrayRef<Matcher *> Rules,
 108     std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
 109
 110 template std::vector<Matcher *> optimizeRules<SwitchMatcher>(
 111     ArrayRef<Matcher *> Rules,
 112     std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
 113
 114 //===- Global Data --------------------------------------------------------===//
 115
 116 std::set<LLTCodeGen> KnownTypes;
 117
 118 //===- MatchTableRecord ---------------------------------------------------===//
 119
 120 void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
 121                             const MatchTable &Table) const {
 122   bool UseLineComment =
 123       LineBreakIsNextAfterThis || (Flags & MTRF_LineBreakFollows);
 124   if (Flags & (MTRF_JumpTarget | MTRF_CommaFollows))
 125     UseLineComment = false;
 126
 127   if (Flags & MTRF_Comment)
 128     OS << (UseLineComment ? "// " : "/*");
 129
 130   OS << EmitStr;
 131   if (Flags & MTRF_Label)
 132     OS << ": @" << Table.getLabelIndex(LabelID);
 133
 134   if ((Flags & MTRF_Comment) && !UseLineComment)
 135     OS << "*/";
 136
 137   if (Flags & MTRF_JumpTarget) {
 138     if (Flags & MTRF_Comment)
 139       OS << " ";
 140     OS << Table.getLabelIndex(LabelID);
 141   }
 142
 143   if (Flags & MTRF_CommaFollows) {
 144     OS << ",";
 145     if (!LineBreakIsNextAfterThis && !(Flags & MTRF_LineBreakFollows))
 146       OS << " ";
 147   }
 148
 149   if (Flags & MTRF_LineBreakFollows)
 150     OS << "\n";
 151 }
 152
 153 //===- MatchTable ---------------------------------------------------------===//
 154
 155 MatchTableRecord MatchTable::LineBreak = {
 156     std::nullopt, "" /* Emit String */, 0 /* Elements */,
 157     MatchTableRecord::MTRF_LineBreakFollows};
 158
 159 MatchTableRecord MatchTable::Comment(StringRef Comment) {
 160   return MatchTableRecord(std::nullopt, Comment, 0,
 161                           MatchTableRecord::MTRF_Comment);
 162 }
 163
 164 MatchTableRecord MatchTable::Opcode(StringRef Opcode, int IndentAdjust) {
 165   unsigned ExtraFlags = 0;
 166   if (IndentAdjust > 0)
 167     ExtraFlags |= MatchTableRecord::MTRF_Indent;
 168   if (IndentAdjust < 0)
 169     ExtraFlags |= MatchTableRecord::MTRF_Outdent;
 170
 171   return MatchTableRecord(std::nullopt, Opcode, 1,
 172                           MatchTableRecord::MTRF_CommaFollows | ExtraFlags);
 173 }
 174
 175 MatchTableRecord MatchTable::NamedValue(StringRef NamedValue) {
 176   return MatchTableRecord(std::nullopt, NamedValue, 1,
 177                           MatchTableRecord::MTRF_CommaFollows);
 178 }
 179
 180 MatchTableRecord MatchTable::NamedValue(StringRef NamedValue,
 181                                         int64_t RawValue) {
 182   return MatchTableRecord(std::nullopt, NamedValue, 1,
 183                           MatchTableRecord::MTRF_CommaFollows, RawValue);
 184 }
 185
 186 MatchTableRecord MatchTable::NamedValue(StringRef Namespace,
 187                                         StringRef NamedValue) {
 188   return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
 189                           1, MatchTableRecord::MTRF_CommaFollows);
 190 }
 191
 192 MatchTableRecord MatchTable::NamedValue(StringRef Namespace,
 193                                         StringRef NamedValue,
 194                                         int64_t RawValue) {
 195   return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
 196                           1, MatchTableRecord::MTRF_CommaFollows, RawValue);
 197 }
 198
 199 MatchTableRecord MatchTable::IntValue(int64_t IntValue) {
 200   return MatchTableRecord(std::nullopt, llvm::to_string(IntValue), 1,
 201                           MatchTableRecord::MTRF_CommaFollows);
 202 }
 203
 204 MatchTableRecord MatchTable::Label(unsigned LabelID) {
 205   return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 0,
 206                           MatchTableRecord::MTRF_Label |
 207                               MatchTableRecord::MTRF_Comment |
 208                               MatchTableRecord::MTRF_LineBreakFollows);
 209 }
 210
 211 MatchTableRecord MatchTable::JumpTarget(unsigned LabelID) {
 212   return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 1,
 213                           MatchTableRecord::MTRF_JumpTarget |
 214                               MatchTableRecord::MTRF_Comment |
 215                               MatchTableRecord::MTRF_CommaFollows);
 216 }
 217
 218 void MatchTable::emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; }
 219
 220 void MatchTable::emitDeclaration(raw_ostream &OS) const {
 221   unsigned Indentation = 4;
 222   OS << "  constexpr static int64_t MatchTable" << ID << "[] = {";
 223   LineBreak.emit(OS, true, *this);
 224   OS << std::string(Indentation, ' ');
 225
 226   for (auto I = Contents.begin(), E = Contents.end(); I != E; ++I) {
 227     bool LineBreakIsNext = false;
 228     const auto &NextI = std::next(I);
 229
 230     if (NextI != E) {
 231       if (NextI->EmitStr == "" &&
 232           NextI->Flags == MatchTableRecord::MTRF_LineBreakFollows)
 233         LineBreakIsNext = true;
 234     }
 235
 236     if (I->Flags & MatchTableRecord::MTRF_Indent)
 237       Indentation += 2;
 238
 239     I->emit(OS, LineBreakIsNext, *this);
 240     if (I->Flags & MatchTableRecord::MTRF_LineBreakFollows)
 241       OS << std::string(Indentation, ' ');
 242
 243     if (I->Flags & MatchTableRecord::MTRF_Outdent)
 244       Indentation -= 2;
 245   }
 246   OS << "};\n";
 247 }
 248
 249 MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage,
 250                                   bool IsCombiner) {
 251   MatchTable Table(WithCoverage, IsCombiner);
 252   for (Matcher *Rule : Rules)
 253     Rule->emit(Table);
 254
 255   return Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
 256 }
 257
 258 //===- LLTCodeGen ---------------------------------------------------------===//
 259
 260 std::string LLTCodeGen::getCxxEnumValue() const {
 261   std::string Str;
 262   raw_string_ostream OS(Str);
 263
 264   emitCxxEnumValue(OS);
 265   return Str;
 266 }
 267
 268 void LLTCodeGen::emitCxxEnumValue(raw_ostream &OS) const {
 269   if (Ty.isScalar()) {
 270     OS << "GILLT_s" << Ty.getSizeInBits();
 271     return;
 272   }
 273   if (Ty.isVector()) {
 274     OS << (Ty.isScalable() ? "GILLT_nxv" : "GILLT_v")
 275        << Ty.getElementCount().getKnownMinValue() << "s"
 276        << Ty.getScalarSizeInBits();
 277     return;
 278   }
 279   if (Ty.isPointer()) {
 280     OS << "GILLT_p" << Ty.getAddressSpace();
 281     if (Ty.getSizeInBits() > 0)
 282       OS << "s" << Ty.getSizeInBits();
 283     return;
 284   }
 285   llvm_unreachable("Unhandled LLT");
 286 }
 287
 288 void LLTCodeGen::emitCxxConstructorCall(raw_ostream &OS) const {
 289   if (Ty.isScalar()) {
 290     OS << "LLT::scalar(" << Ty.getSizeInBits() << ")";
 291     return;
 292   }
 293   if (Ty.isVector()) {
 294     OS << "LLT::vector("
 295        << (Ty.isScalable() ? "ElementCount::getScalable("
 296                            : "ElementCount::getFixed(")
 297        << Ty.getElementCount().getKnownMinValue() << "), "
 298        << Ty.getScalarSizeInBits() << ")";
 299     return;
 300   }
 301   if (Ty.isPointer() && Ty.getSizeInBits() > 0) {
 302     OS << "LLT::pointer(" << Ty.getAddressSpace() << ", " << Ty.getSizeInBits()
 303        << ")";
 304     return;
 305   }
 306   llvm_unreachable("Unhandled LLT");
 307 }
 308
 309 /// This ordering is used for std::unique() and llvm::sort(). There's no
 310 /// particular logic behind the order but either A < B or B < A must be
 311 /// true if A != B.
 312 bool LLTCodeGen::operator<(const LLTCodeGen &Other) const {
 313   if (Ty.isValid() != Other.Ty.isValid())
 314     return Ty.isValid() < Other.Ty.isValid();
 315   if (!Ty.isValid())
 316     return false;
 317
 318   if (Ty.isVector() != Other.Ty.isVector())
 319     return Ty.isVector() < Other.Ty.isVector();
 320   if (Ty.isScalar() != Other.Ty.isScalar())
 321     return Ty.isScalar() < Other.Ty.isScalar();
 322   if (Ty.isPointer() != Other.Ty.isPointer())
 323     return Ty.isPointer() < Other.Ty.isPointer();
 324
 325   if (Ty.isPointer() && Ty.getAddressSpace() != Other.Ty.getAddressSpace())
 326     return Ty.getAddressSpace() < Other.Ty.getAddressSpace();
 327
 328   if (Ty.isVector() && Ty.getElementCount() != Other.Ty.getElementCount())
 329     return std::make_tuple(Ty.isScalable(),
 330                            Ty.getElementCount().getKnownMinValue()) <
 331            std::make_tuple(Other.Ty.isScalable(),
 332                            Other.Ty.getElementCount().getKnownMinValue());
 333
 334   assert((!Ty.isVector() || Ty.isScalable() == Other.Ty.isScalable()) &&
 335          "Unexpected mismatch of scalable property");
 336   return Ty.isVector()
 337              ? std::make_tuple(Ty.isScalable(),
 338                                Ty.getSizeInBits().getKnownMinValue()) <
 339                    std::make_tuple(Other.Ty.isScalable(),
 340                                    Other.Ty.getSizeInBits().getKnownMinValue())
 341              : Ty.getSizeInBits().getFixedValue() <
 342                    Other.Ty.getSizeInBits().getFixedValue();
 343 }
 344
 345 //===- LLTCodeGen Helpers -------------------------------------------------===//
 346
 347 std::optional<LLTCodeGen> MVTToLLT(MVT::SimpleValueType SVT) {
 348   MVT VT(SVT);
 349
 350   if (VT.isVector() && !VT.getVectorElementCount().isScalar())
 351     return LLTCodeGen(
 352         LLT::vector(VT.getVectorElementCount(), VT.getScalarSizeInBits()));
 353
 354   if (VT.isInteger() || VT.isFloatingPoint())
 355     return LLTCodeGen(LLT::scalar(VT.getSizeInBits()));
 356
 357   return std::nullopt;
 358 }
 359
 360 //===- Matcher ------------------------------------------------------------===//
 361
 362 void Matcher::optimize() {}
 363
 364 Matcher::~Matcher() {}
 365
 366 //===- GroupMatcher -------------------------------------------------------===//
 367
 368 bool GroupMatcher::candidateConditionMatches(
 369     const PredicateMatcher &Predicate) const {
 370
 371   if (empty()) {
 372     // Sharing predicates for nested instructions is not supported yet as we
 373     // currently don't hoist the GIM_RecordInsn's properly, therefore we can
 374     // only work on the original root instruction (InsnVarID == 0):
 375     if (Predicate.getInsnVarID() != 0)
 376       return false;
 377     // ... otherwise an empty group can handle any predicate with no specific
 378     // requirements:
 379     return true;
 380   }
 381
 382   const Matcher &Representative = **Matchers.begin();
 383   const auto &RepresentativeCondition = Representative.getFirstCondition();
 384   // ... if not empty, the group can only accomodate matchers with the exact
 385   // same first condition:
 386   return Predicate.isIdentical(RepresentativeCondition);
 387 }
 388
 389 bool GroupMatcher::addMatcher(Matcher &Candidate) {
 390   if (!Candidate.hasFirstCondition())
 391     return false;
 392
 393   const PredicateMatcher &Predicate = Candidate.getFirstCondition();
 394   if (!candidateConditionMatches(Predicate))
 395     return false;
 396
 397   Matchers.push_back(&Candidate);
 398   return true;
 399 }
 400
 401 void GroupMatcher::finalize() {
 402   assert(Conditions.empty() && "Already finalized?");
 403   if (empty())
 404     return;
 405
 406   Matcher &FirstRule = **Matchers.begin();
 407   for (;;) {
 408     // All the checks are expected to succeed during the first iteration:
 409     for (const auto &Rule : Matchers)
 410       if (!Rule->hasFirstCondition())
 411         return;
 412     const auto &FirstCondition = FirstRule.getFirstCondition();
 413     for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
 414       if (!Matchers[I]->getFirstCondition().isIdentical(FirstCondition))
 415         return;
 416
 417     Conditions.push_back(FirstRule.popFirstCondition());
 418     for (unsigned I = 1, E = Matchers.size(); I < E; ++I)
 419       Matchers[I]->popFirstCondition();
 420   }
 421 }
 422
 423 void GroupMatcher::emit(MatchTable &Table) {
 424   unsigned LabelID = ~0U;
 425   if (!Conditions.empty()) {
 426     LabelID = Table.allocateLabelID();
 427     Table << MatchTable::Opcode("GIM_Try", +1)
 428           << MatchTable::Comment("On fail goto")
 429           << MatchTable::JumpTarget(LabelID) << MatchTable::LineBreak;
 430   }
 431   for (auto &Condition : Conditions)
 432     Condition->emitPredicateOpcodes(
 433         Table, *static_cast<RuleMatcher *>(*Matchers.begin()));
 434
 435   for (const auto &M : Matchers)
 436     M->emit(Table);
 437
 438   // Exit the group
 439   if (!Conditions.empty())
 440     Table << MatchTable::Opcode("GIM_Reject", -1) << MatchTable::LineBreak
 441           << MatchTable::Label(LabelID);
 442 }
 443
 444 void GroupMatcher::optimize() {
 445   // Make sure we only sort by a specific predicate within a range of rules that
 446   // all have that predicate checked against a specific value (not a wildcard):
 447   auto F = Matchers.begin();
 448   auto T = F;
 449   auto E = Matchers.end();
 450   while (T != E) {
 451     while (T != E) {
 452       auto *R = static_cast<RuleMatcher *>(*T);
 453       if (!R->getFirstConditionAsRootType().get().isValid())
 454         break;
 455       ++T;
 456     }
 457     std::stable_sort(F, T, [](Matcher *A, Matcher *B) {
 458       auto *L = static_cast<RuleMatcher *>(A);
 459       auto *R = static_cast<RuleMatcher *>(B);
 460       return L->getFirstConditionAsRootType() <
 461              R->getFirstConditionAsRootType();
 462     });
 463     if (T != E)
 464       F = ++T;
 465   }
 466   optimizeRules<GroupMatcher>(Matchers, MatcherStorage).swap(Matchers);
 467   optimizeRules<SwitchMatcher>(Matchers, MatcherStorage).swap(Matchers);
 468 }
 469
 470 //===- SwitchMatcher ------------------------------------------------------===//
 471
 472 bool SwitchMatcher::isSupportedPredicateType(const PredicateMatcher &P) {
 473   return isa<InstructionOpcodeMatcher>(P) || isa<LLTOperandMatcher>(P);
 474 }
 475
 476 bool SwitchMatcher::candidateConditionMatches(
 477     const PredicateMatcher &Predicate) const {
 478
 479   if (empty()) {
 480     // Sharing predicates for nested instructions is not supported yet as we
 481     // currently don't hoist the GIM_RecordInsn's properly, therefore we can
 482     // only work on the original root instruction (InsnVarID == 0):
 483     if (Predicate.getInsnVarID() != 0)
 484       return false;
 485     // ... while an attempt to add even a root matcher to an empty SwitchMatcher
 486     // could fail as not all the types of conditions are supported:
 487     if (!isSupportedPredicateType(Predicate))
 488       return false;
 489     // ... or the condition might not have a proper implementation of
 490     // getValue() / isIdenticalDownToValue() yet:
 491     if (!Predicate.hasValue())
 492       return false;
 493     // ... otherwise an empty Switch can accomodate the condition with no
 494     // further requirements:
 495     return true;
 496   }
 497
 498   const Matcher &CaseRepresentative = **Matchers.begin();
 499   const auto &RepresentativeCondition = CaseRepresentative.getFirstCondition();
 500   // Switch-cases must share the same kind of condition and path to the value it
 501   // checks:
 502   if (!Predicate.isIdenticalDownToValue(RepresentativeCondition))
 503     return false;
 504
 505   const auto Value = Predicate.getValue();
 506   // ... but be unique with respect to the actual value they check:
 507   return Values.count(Value) == 0;
 508 }
 509
 510 bool SwitchMatcher::addMatcher(Matcher &Candidate) {
 511   if (!Candidate.hasFirstCondition())
 512     return false;
 513
 514   const PredicateMatcher &Predicate = Candidate.getFirstCondition();
 515   if (!candidateConditionMatches(Predicate))
 516     return false;
 517   const auto Value = Predicate.getValue();
 518   Values.insert(Value);
 519
 520   Matchers.push_back(&Candidate);
 521   return true;
 522 }
 523
 524 void SwitchMatcher::finalize() {
 525   assert(Condition == nullptr && "Already finalized");
 526   assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
 527   if (empty())
 528     return;
 529
 530   llvm::stable_sort(Matchers, [](const Matcher *L, const Matcher *R) {
 531     return L->getFirstCondition().getValue() <
 532            R->getFirstCondition().getValue();
 533   });
 534   Condition = Matchers[0]->popFirstCondition();
 535   for (unsigned I = 1, E = Values.size(); I < E; ++I)
 536     Matchers[I]->popFirstCondition();
 537 }
 538
 539 void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
 540                                                  MatchTable &Table) {
 541   assert(isSupportedPredicateType(P) && "Predicate type is not supported");
 542
 543   if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(&P)) {
 544     Table << MatchTable::Opcode("GIM_SwitchOpcode") << MatchTable::Comment("MI")
 545           << MatchTable::IntValue(Condition->getInsnVarID());
 546     return;
 547   }
 548   if (const auto *Condition = dyn_cast<LLTOperandMatcher>(&P)) {
 549     Table << MatchTable::Opcode("GIM_SwitchType") << MatchTable::Comment("MI")
 550           << MatchTable::IntValue(Condition->getInsnVarID())
 551           << MatchTable::Comment("Op")
 552           << MatchTable::IntValue(Condition->getOpIdx());
 553     return;
 554   }
 555
 556   llvm_unreachable("emitPredicateSpecificOpcodes is broken: can not handle a "
 557                    "predicate type that is claimed to be supported");
 558 }
 559
 560 void SwitchMatcher::emit(MatchTable &Table) {
 561   assert(Values.size() == Matchers.size() && "Broken SwitchMatcher");
 562   if (empty())
 563     return;
 564   assert(Condition != nullptr &&
 565          "Broken SwitchMatcher, hasn't been finalized?");
 566
 567   std::vector<unsigned> LabelIDs(Values.size());
 568   std::generate(LabelIDs.begin(), LabelIDs.end(),
 569                 [&Table]() { return Table.allocateLabelID(); });
 570   const unsigned Default = Table.allocateLabelID();
 571
 572   const int64_t LowerBound = Values.begin()->getRawValue();
 573   const int64_t UpperBound = Values.rbegin()->getRawValue() + 1;
 574
 575   emitPredicateSpecificOpcodes(*Condition, Table);
 576
 577   Table << MatchTable::Comment("[") << MatchTable::IntValue(LowerBound)
 578         << MatchTable::IntValue(UpperBound) << MatchTable::Comment(")")
 579         << MatchTable::Comment("default:") << MatchTable::JumpTarget(Default);
 580
 581   int64_t J = LowerBound;
 582   auto VI = Values.begin();
 583   for (unsigned I = 0, E = Values.size(); I < E; ++I) {
 584     auto V = *VI++;
 585     while (J++ < V.getRawValue())
 586       Table << MatchTable::IntValue(0);
 587     V.turnIntoComment();
 588     Table << MatchTable::LineBreak << V << MatchTable::JumpTarget(LabelIDs[I]);
 589   }
 590   Table << MatchTable::LineBreak;
 591
 592   for (unsigned I = 0, E = Values.size(); I < E; ++I) {
 593     Table << MatchTable::Label(LabelIDs[I]);
 594     Matchers[I]->emit(Table);
 595     Table << MatchTable::Opcode("GIM_Reject") << MatchTable::LineBreak;
 596   }
 597   Table << MatchTable::Label(Default);
 598 }
 599
 600 //===- RuleMatcher --------------------------------------------------------===//
 601
 602 uint64_t RuleMatcher::NextRuleID = 0;
 603
 604 StringRef RuleMatcher::getOpcode() const {
 605   return Matchers.front()->getOpcode();
 606 }
 607
 608 unsigned RuleMatcher::getNumOperands() const {
 609   return Matchers.front()->getNumOperands();
 610 }
 611
 612 LLTCodeGen RuleMatcher::getFirstConditionAsRootType() {
 613   InstructionMatcher &InsnMatcher = *Matchers.front();
 614   if (!InsnMatcher.predicates_empty())
 615     if (const auto *TM =
 616             dyn_cast<LLTOperandMatcher>(&**InsnMatcher.predicates_begin()))
 617       if (TM->getInsnVarID() == 0 && TM->getOpIdx() == 0)
 618         return TM->getTy();
 619   return {};
 620 }
 621
 622 void RuleMatcher::optimize() {
 623   for (auto &Item : InsnVariableIDs) {
 624     InstructionMatcher &InsnMatcher = *Item.first;
 625     for (auto &OM : InsnMatcher.operands()) {
 626       // Complex Patterns are usually expensive and they relatively rarely fail
 627       // on their own: more often we end up throwing away all the work done by a
 628       // matching part of a complex pattern because some other part of the
 629       // enclosing pattern didn't match. All of this makes it beneficial to
 630       // delay complex patterns until the very end of the rule matching,
 631       // especially for targets having lots of complex patterns.
 632       for (auto &OP : OM->predicates())
 633         if (isa<ComplexPatternOperandMatcher>(OP))
 634           EpilogueMatchers.emplace_back(std::move(OP));
 635       OM->eraseNullPredicates();
 636     }
 637     InsnMatcher.optimize();
 638   }
 639   llvm::sort(EpilogueMatchers, [](const std::unique_ptr<PredicateMatcher> &L,
 640                                   const std::unique_ptr<PredicateMatcher> &R) {
 641     return std::make_tuple(L->getKind(), L->getInsnVarID(), L->getOpIdx()) <
 642            std::make_tuple(R->getKind(), R->getInsnVarID(), R->getOpIdx());
 643   });
 644 }
 645
 646 bool RuleMatcher::hasFirstCondition() const {
 647   if (insnmatchers_empty())
 648     return false;
 649   InstructionMatcher &Matcher = insnmatchers_front();
 650   if (!Matcher.predicates_empty())
 651     return true;
 652   for (auto &OM : Matcher.operands())
 653     for (auto &OP : OM->predicates())
 654       if (!isa<InstructionOperandMatcher>(OP))
 655         return true;
 656   return false;
 657 }
 658
 659 const PredicateMatcher &RuleMatcher::getFirstCondition() const {
 660   assert(!insnmatchers_empty() &&
 661          "Trying to get a condition from an empty RuleMatcher");
 662
 663   InstructionMatcher &Matcher = insnmatchers_front();
 664   if (!Matcher.predicates_empty())
 665     return **Matcher.predicates_begin();
 666   // If there is no more predicate on the instruction itself, look at its
 667   // operands.
 668   for (auto &OM : Matcher.operands())
 669     for (auto &OP : OM->predicates())
 670       if (!isa<InstructionOperandMatcher>(OP))
 671         return *OP;
 672
 673   llvm_unreachable("Trying to get a condition from an InstructionMatcher with "
 674                    "no conditions");
 675 }
 676
 677 std::unique_ptr<PredicateMatcher> RuleMatcher::popFirstCondition() {
 678   assert(!insnmatchers_empty() &&
 679          "Trying to pop a condition from an empty RuleMatcher");
 680
 681   InstructionMatcher &Matcher = insnmatchers_front();
 682   if (!Matcher.predicates_empty())
 683     return Matcher.predicates_pop_front();
 684   // If there is no more predicate on the instruction itself, look at its
 685   // operands.
 686   for (auto &OM : Matcher.operands())
 687     for (auto &OP : OM->predicates())
 688       if (!isa<InstructionOperandMatcher>(OP)) {
 689         std::unique_ptr<PredicateMatcher> Result = std::move(OP);
 690         OM->eraseNullPredicates();
 691         return Result;
 692       }
 693
 694   llvm_unreachable("Trying to pop a condition from an InstructionMatcher with "
 695                    "no conditions");
 696 }
 697
 698 GISelFlags RuleMatcher::updateGISelFlag(GISelFlags CurFlags, const Record *R,
 699                                         StringRef FlagName,
 700                                         GISelFlags FlagBit) {
 701   // If the value of a flag is unset, ignore it.
 702   // If it's set, it always takes precedence over the existing value so
 703   // clear/set the corresponding bit.
 704   bool Unset = false;
 705   bool Value = R->getValueAsBitOrUnset("GIIgnoreCopies", Unset);
 706   if (!Unset)
 707     return Value ? (CurFlags | FlagBit) : (CurFlags & ~FlagBit);
 708   return CurFlags;
 709 }
 710
 711 SaveAndRestore<GISelFlags> RuleMatcher::setGISelFlags(const Record *R) {
 712   if (!R || !R->isSubClassOf("GISelFlags"))
 713     return {Flags, Flags};
 714
 715   assert((R->isSubClassOf("PatFrags") || R->isSubClassOf("Pattern")) &&
 716          "GISelFlags is only expected on Pattern/PatFrags!");
 717
 718   GISelFlags NewFlags =
 719       updateGISelFlag(Flags, R, "GIIgnoreCopies", GISF_IgnoreCopies);
 720   return {Flags, NewFlags};
 721 }
 722
 723 Error RuleMatcher::defineComplexSubOperand(StringRef SymbolicName,
 724                                            Record *ComplexPattern,
 725                                            unsigned RendererID,
 726                                            unsigned SubOperandID,
 727                                            StringRef ParentSymbolicName) {
 728   std::string ParentName(ParentSymbolicName);
 729   if (ComplexSubOperands.count(SymbolicName)) {
 730     const std::string &RecordedParentName =
 731         ComplexSubOperandsParentName[SymbolicName];
 732     if (RecordedParentName != ParentName)
 733       return failUnsupported("Error: Complex suboperand " + SymbolicName +
 734                              " referenced by different operands: " +
 735                              RecordedParentName + " and " + ParentName + ".");
 736     // Complex suboperand referenced more than once from same the operand is
 737     // used to generate 'same operand check'. Emitting of
 738     // GIR_ComplexSubOperandRenderer for them is already handled.
 739     return Error::success();
 740   }
 741
 742   ComplexSubOperands[SymbolicName] =
 743       std::make_tuple(ComplexPattern, RendererID, SubOperandID);
 744   ComplexSubOperandsParentName[SymbolicName] = ParentName;
 745
 746   return Error::success();
 747 }
 748
 749 InstructionMatcher &RuleMatcher::addInstructionMatcher(StringRef SymbolicName) {
 750   Matchers.emplace_back(new InstructionMatcher(*this, SymbolicName));
 751   MutatableInsns.insert(Matchers.back().get());
 752   return *Matchers.back();
 753 }
 754
 755 void RuleMatcher::addRequiredSimplePredicate(StringRef PredName) {
 756   RequiredSimplePredicates.push_back(PredName.str());
 757 }
 758
 759 const std::vector<std::string> &RuleMatcher::getRequiredSimplePredicates() {
 760   return RequiredSimplePredicates;
 761 }
 762
 763 void RuleMatcher::addRequiredFeature(Record *Feature) {
 764   RequiredFeatures.push_back(Feature);
 765 }
 766
 767 const std::vector<Record *> &RuleMatcher::getRequiredFeatures() const {
 768   return RequiredFeatures;
 769 }
 770
 771 unsigned RuleMatcher::implicitlyDefineInsnVar(InstructionMatcher &Matcher) {
 772   unsigned NewInsnVarID = NextInsnVarID++;
 773   InsnVariableIDs[&Matcher] = NewInsnVarID;
 774   return NewInsnVarID;
 775 }
 776
 777 unsigned RuleMatcher::getInsnVarID(InstructionMatcher &InsnMatcher) const {
 778   const auto &I = InsnVariableIDs.find(&InsnMatcher);
 779   if (I != InsnVariableIDs.end())
 780     return I->second;
 781   llvm_unreachable("Matched Insn was not captured in a local variable");
 782 }
 783
 784 void RuleMatcher::defineOperand(StringRef SymbolicName, OperandMatcher &OM) {
 785   if (!DefinedOperands.contains(SymbolicName)) {
 786     DefinedOperands[SymbolicName] = &OM;
 787     return;
 788   }
 789
 790   // If the operand is already defined, then we must ensure both references in
 791   // the matcher have the exact same node.
 792   RuleMatcher &RM = OM.getInstructionMatcher().getRuleMatcher();
 793   OM.addPredicate<SameOperandMatcher>(
 794       OM.getSymbolicName(), getOperandMatcher(OM.getSymbolicName()).getOpIdx(),
 795       RM.getGISelFlags());
 796 }
 797
 798 void RuleMatcher::definePhysRegOperand(Record *Reg, OperandMatcher &OM) {
 799   if (!PhysRegOperands.contains(Reg)) {
 800     PhysRegOperands[Reg] = &OM;
 801     return;
 802   }
 803 }
 804
 805 InstructionMatcher &
 806 RuleMatcher::getInstructionMatcher(StringRef SymbolicName) const {
 807   for (const auto &I : InsnVariableIDs)
 808     if (I.first->getSymbolicName() == SymbolicName)
 809       return *I.first;
 810   llvm_unreachable(
 811       ("Failed to lookup instruction " + SymbolicName).str().c_str());
 812 }
 813
 814 const OperandMatcher &RuleMatcher::getPhysRegOperandMatcher(Record *Reg) const {
 815   const auto &I = PhysRegOperands.find(Reg);
 816
 817   if (I == PhysRegOperands.end()) {
 818     PrintFatalError(SrcLoc, "Register " + Reg->getName() +
 819                                 " was not declared in matcher");
 820   }
 821
 822   return *I->second;
 823 }
 824
 825 OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) {
 826   const auto &I = DefinedOperands.find(Name);
 827
 828   if (I == DefinedOperands.end())
 829     PrintFatalError(SrcLoc, "Operand " + Name + " was not declared in matcher");
 830
 831   return *I->second;
 832 }
 833
 834 const OperandMatcher &RuleMatcher::getOperandMatcher(StringRef Name) const {
 835   const auto &I = DefinedOperands.find(Name);
 836
 837   if (I == DefinedOperands.end())
 838     PrintFatalError(SrcLoc, "Operand " + Name + " was not declared in matcher");
 839
 840   return *I->second;
 841 }
 842
 843 void RuleMatcher::emit(MatchTable &Table) {
 844   if (Matchers.empty())
 845     llvm_unreachable("Unexpected empty matcher!");
 846
 847   // The representation supports rules that require multiple roots such as:
 848   //    %ptr(p0) = ...
 849   //    %elt0(s32) = G_LOAD %ptr
 850   //    %1(p0) = G_ADD %ptr, 4
 851   //    %elt1(s32) = G_LOAD p0 %1
 852   // which could be usefully folded into:
 853   //    %ptr(p0) = ...
 854   //    %elt0(s32), %elt1(s32) = TGT_LOAD_PAIR %ptr
 855   // on some targets but we don't need to make use of that yet.
 856   assert(Matchers.size() == 1 && "Cannot handle multi-root matchers yet");
 857
 858   unsigned LabelID = Table.allocateLabelID();
 859   Table << MatchTable::Opcode("GIM_Try", +1)
 860         << MatchTable::Comment("On fail goto")
 861         << MatchTable::JumpTarget(LabelID)
 862         << MatchTable::Comment(("Rule ID " + Twine(RuleID) + " //").str())
 863         << MatchTable::LineBreak;
 864
 865   if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
 866     Table << MatchTable::Opcode("GIM_CheckFeatures")
 867           << MatchTable::NamedValue(
 868                  getNameForFeatureBitset(RequiredFeatures, HwModeIdx))
 869           << MatchTable::LineBreak;
 870   }
 871
 872   if (!RequiredSimplePredicates.empty()) {
 873     for (const auto &Pred : RequiredSimplePredicates) {
 874       Table << MatchTable::Opcode("GIM_CheckSimplePredicate")
 875             << MatchTable::NamedValue(Pred) << MatchTable::LineBreak;
 876     }
 877   }
 878
 879   Matchers.front()->emitPredicateOpcodes(Table, *this);
 880
 881   // Check if it's safe to replace registers.
 882   for (const auto &MA : Actions)
 883     MA->emitAdditionalPredicates(Table, *this);
 884
 885   // We must also check if it's safe to fold the matched instructions.
 886   if (InsnVariableIDs.size() >= 2) {
 887     // Invert the map to create stable ordering (by var names)
 888     SmallVector<unsigned, 2> InsnIDs;
 889     for (const auto &Pair : InsnVariableIDs) {
 890       // Skip the root node since it isn't moving anywhere. Everything else is
 891       // sinking to meet it.
 892       if (Pair.first == Matchers.front().get())
 893         continue;
 894
 895       InsnIDs.push_back(Pair.second);
 896     }
 897     llvm::sort(InsnIDs);
 898
 899     for (const auto &InsnID : InsnIDs) {
 900       // Reject the difficult cases until we have a more accurate check.
 901       Table << MatchTable::Opcode("GIM_CheckIsSafeToFold")
 902             << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
 903             << MatchTable::LineBreak;
 904
 905       // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
 906       //        account for unsafe cases.
 907       //
 908       //        Example:
 909       //          MI1--> %0 = ...
 910       //                 %1 = ... %0
 911       //          MI0--> %2 = ... %0
 912       //          It's not safe to erase MI1. We currently handle this by not
 913       //          erasing %0 (even when it's dead).
 914       //
 915       //        Example:
 916       //          MI1--> %0 = load volatile @a
 917       //                 %1 = load volatile @a
 918       //          MI0--> %2 = ... %0
 919       //          It's not safe to sink %0's def past %1. We currently handle
 920       //          this by rejecting all loads.
 921       //
 922       //        Example:
 923       //          MI1--> %0 = load @a
 924       //                 %1 = store @a
 925       //          MI0--> %2 = ... %0
 926       //          It's not safe to sink %0's def past %1. We currently handle
 927       //          this by rejecting all loads.
 928       //
 929       //        Example:
 930       //                   G_CONDBR %cond, @BB1
 931       //                 BB0:
 932       //          MI1-->   %0 = load @a
 933       //                   G_BR @BB1
 934       //                 BB1:
 935       //          MI0-->   %2 = ... %0
 936       //          It's not always safe to sink %0 across control flow. In this
 937       //          case it may introduce a memory fault. We currentl handle
 938       //          this by rejecting all loads.
 939     }
 940   }
 941
 942   for (const auto &PM : EpilogueMatchers)
 943     PM->emitPredicateOpcodes(Table, *this);
 944
 945   for (const auto &MA : Actions)
 946     MA->emitActionOpcodes(Table, *this);
 947
 948   assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
 949          "Combiner tables don't support coverage!");
 950   if (Table.isWithCoverage())
 951     Table << MatchTable::Opcode("GIR_Coverage") << MatchTable::IntValue(RuleID)
 952           << MatchTable::LineBreak;
 953   else if (!Table.isCombiner())
 954     Table << MatchTable::Comment(("GIR_Coverage, " + Twine(RuleID) + ",").str())
 955           << MatchTable::LineBreak;
 956
 957   Table << MatchTable::Opcode("GIR_Done", -1) << MatchTable::LineBreak
 958         << MatchTable::Label(LabelID);
 959   ++NumPatternEmitted;
 960 }
 961
 962 bool RuleMatcher::isHigherPriorityThan(const RuleMatcher &B) const {
 963   // Rules involving more match roots have higher priority.
 964   if (Matchers.size() > B.Matchers.size())
 965     return true;
 966   if (Matchers.size() < B.Matchers.size())
 967     return false;
 968
 969   for (auto Matcher : zip(Matchers, B.Matchers)) {
 970     if (std::get<0>(Matcher)->isHigherPriorityThan(*std::get<1>(Matcher)))
 971       return true;
 972     if (std::get<1>(Matcher)->isHigherPriorityThan(*std::get<0>(Matcher)))
 973       return false;
 974   }
 975
 976   return false;
 977 }
 978
 979 unsigned RuleMatcher::countRendererFns() const {
 980   return std::accumulate(
 981       Matchers.begin(), Matchers.end(), 0,
 982       [](unsigned A, const std::unique_ptr<InstructionMatcher> &Matcher) {
 983         return A + Matcher->countRendererFns();
 984       });
 985 }
 986
 987 //===- PredicateMatcher ---------------------------------------------------===//
 988
 989 PredicateMatcher::~PredicateMatcher() {}
 990
 991 //===- OperandPredicateMatcher --------------------------------------------===//
 992
 993 OperandPredicateMatcher::~OperandPredicateMatcher() {}
 994
 995 bool OperandPredicateMatcher::isHigherPriorityThan(
 996     const OperandPredicateMatcher &B) const {
 997   // Generally speaking, an instruction is more important than an Int or a
 998   // LiteralInt because it can cover more nodes but theres an exception to
 999   // this. G_CONSTANT's are less important than either of those two because they
1000   // are more permissive.
1001
1002   const InstructionOperandMatcher *AOM =
1003       dyn_cast<InstructionOperandMatcher>(this);
1004   const InstructionOperandMatcher *BOM =
1005       dyn_cast<InstructionOperandMatcher>(&B);
1006   bool AIsConstantInsn = AOM && AOM->getInsnMatcher().isConstantInstruction();
1007   bool BIsConstantInsn = BOM && BOM->getInsnMatcher().isConstantInstruction();
1008
1009   if (AOM && BOM) {
1010     // The relative priorities between a G_CONSTANT and any other instruction
1011     // don't actually matter but this code is needed to ensure a strict weak
1012     // ordering. This is particularly important on Windows where the rules will
1013     // be incorrectly sorted without it.
1014     if (AIsConstantInsn != BIsConstantInsn)
1015       return AIsConstantInsn < BIsConstantInsn;
1016     return false;
1017   }
1018
1019   if (AOM && AIsConstantInsn && (B.Kind == OPM_Int || B.Kind == OPM_LiteralInt))
1020     return false;
1021   if (BOM && BIsConstantInsn && (Kind == OPM_Int || Kind == OPM_LiteralInt))
1022     return true;
1023
1024   return Kind < B.Kind;
1025 }
1026
1027 //===- SameOperandMatcher -------------------------------------------------===//
1028
1029 void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1030                                               RuleMatcher &Rule) const {
1031   const OperandMatcher &OtherOM = Rule.getOperandMatcher(MatchingName);
1032   unsigned OtherInsnVarID = Rule.getInsnVarID(OtherOM.getInstructionMatcher());
1033   assert(OtherInsnVarID == OtherOM.getInstructionMatcher().getInsnVarID());
1034   const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1035   Table << MatchTable::Opcode(IgnoreCopies
1036                                   ? "GIM_CheckIsSameOperandIgnoreCopies"
1037                                   : "GIM_CheckIsSameOperand")
1038         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1039         << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
1040         << MatchTable::Comment("OtherMI")
1041         << MatchTable::IntValue(OtherInsnVarID)
1042         << MatchTable::Comment("OtherOpIdx")
1043         << MatchTable::IntValue(OtherOM.getOpIdx()) << MatchTable::LineBreak;
1044 }
1045
1046 //===- LLTOperandMatcher --------------------------------------------------===//
1047
1048 std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
1049
1050 MatchTableRecord LLTOperandMatcher::getValue() const {
1051   const auto VI = TypeIDValues.find(Ty);
1052   if (VI == TypeIDValues.end())
1053     return MatchTable::NamedValue(getTy().getCxxEnumValue());
1054   return MatchTable::NamedValue(getTy().getCxxEnumValue(), VI->second);
1055 }
1056
1057 bool LLTOperandMatcher::hasValue() const {
1058   if (TypeIDValues.size() != KnownTypes.size())
1059     initTypeIDValuesMap();
1060   return TypeIDValues.count(Ty);
1061 }
1062
1063 void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1064                                              RuleMatcher &Rule) const {
1065   Table << MatchTable::Opcode("GIM_CheckType") << MatchTable::Comment("MI")
1066         << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
1067         << MatchTable::IntValue(OpIdx) << MatchTable::Comment("Type")
1068         << getValue() << MatchTable::LineBreak;
1069 }
1070
1071 //===- PointerToAnyOperandMatcher -----------------------------------------===//
1072
1073 void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1074                                                       RuleMatcher &Rule) const {
1075   Table << MatchTable::Opcode("GIM_CheckPointerToAny")
1076         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1077         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1078         << MatchTable::Comment("SizeInBits") << MatchTable::IntValue(SizeInBits)
1079         << MatchTable::LineBreak;
1080 }
1081
1082 //===- RecordNamedOperandMatcher ------------------------------------------===//
1083
1084 void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1085                                                      RuleMatcher &Rule) const {
1086   Table << MatchTable::Opcode("GIM_RecordNamedOperand")
1087         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1088         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1089         << MatchTable::Comment("StoreIdx") << MatchTable::IntValue(StoreIdx)
1090         << MatchTable::Comment("Name : " + Name) << MatchTable::LineBreak;
1091 }
1092
1093 //===- RecordRegisterType ------------------------------------------===//
1094
1095 void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table,
1096                                               RuleMatcher &Rule) const {
1097   assert(Idx < 0 && "Temp types always have negative indexes!");
1098   Table << MatchTable::Opcode("GIM_RecordRegType") << MatchTable::Comment("MI")
1099         << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
1100         << MatchTable::IntValue(OpIdx) << MatchTable::Comment("TempTypeIdx")
1101         << MatchTable::IntValue(Idx) << MatchTable::LineBreak;
1102 }
1103
1104 //===- ComplexPatternOperandMatcher ---------------------------------------===//
1105
1106 void ComplexPatternOperandMatcher::emitPredicateOpcodes(
1107     MatchTable &Table, RuleMatcher &Rule) const {
1108   unsigned ID = getAllocatedTemporariesBaseID();
1109   Table << MatchTable::Opcode("GIM_CheckComplexPattern")
1110         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1111         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1112         << MatchTable::Comment("Renderer") << MatchTable::IntValue(ID)
1113         << MatchTable::NamedValue(("GICP_" + TheDef.getName()).str())
1114         << MatchTable::LineBreak;
1115 }
1116
1117 unsigned ComplexPatternOperandMatcher::getAllocatedTemporariesBaseID() const {
1118   return Operand.getAllocatedTemporariesBaseID();
1119 }
1120
1121 //===- RegisterBankOperandMatcher -----------------------------------------===//
1122
1123 bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
1124   return OperandPredicateMatcher::isIdentical(B) &&
1125          RC.getDef() == cast<RegisterBankOperandMatcher>(&B)->RC.getDef();
1126 }
1127
1128 void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1129                                                       RuleMatcher &Rule) const {
1130   Table << MatchTable::Opcode("GIM_CheckRegBankForClass")
1131         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1132         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1133         << MatchTable::Comment("RC")
1134         << MatchTable::NamedValue(RC.getQualifiedIdName())
1135         << MatchTable::LineBreak;
1136 }
1137
1138 //===- MBBOperandMatcher --------------------------------------------------===//
1139
1140 void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1141                                              RuleMatcher &Rule) const {
1142   Table << MatchTable::Opcode("GIM_CheckIsMBB") << MatchTable::Comment("MI")
1143         << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
1144         << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
1145 }
1146
1147 //===- ImmOperandMatcher --------------------------------------------------===//
1148
1149 void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1150                                              RuleMatcher &Rule) const {
1151   Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI")
1152         << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
1153         << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
1154 }
1155
1156 //===- ConstantIntOperandMatcher ------------------------------------------===//
1157
1158 void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1159                                                      RuleMatcher &Rule) const {
1160   Table << MatchTable::Opcode("GIM_CheckConstantInt")
1161         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1162         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1163         << MatchTable::IntValue(Value) << MatchTable::LineBreak;
1164 }
1165
1166 //===- LiteralIntOperandMatcher -------------------------------------------===//
1167
1168 void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1169                                                     RuleMatcher &Rule) const {
1170   Table << MatchTable::Opcode("GIM_CheckLiteralInt")
1171         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1172         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1173         << MatchTable::IntValue(Value) << MatchTable::LineBreak;
1174 }
1175
1176 //===- CmpPredicateOperandMatcher -----------------------------------------===//
1177
1178 void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1179                                                       RuleMatcher &Rule) const {
1180   Table << MatchTable::Opcode("GIM_CheckCmpPredicate")
1181         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1182         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1183         << MatchTable::Comment("Predicate")
1184         << MatchTable::NamedValue("CmpInst", PredName) << MatchTable::LineBreak;
1185 }
1186
1187 //===- IntrinsicIDOperandMatcher ------------------------------------------===//
1188
1189 void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1190                                                      RuleMatcher &Rule) const {
1191   Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
1192         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1193         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
1194         << MatchTable::NamedValue("Intrinsic::" + II->EnumName)
1195         << MatchTable::LineBreak;
1196 }
1197
1198 //===- OperandImmPredicateMatcher -----------------------------------------===//
1199
1200 void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
1201                                                       RuleMatcher &Rule) const {
1202   Table << MatchTable::Opcode("GIM_CheckImmOperandPredicate")
1203         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1204         << MatchTable::Comment("MO") << MatchTable::IntValue(OpIdx)
1205         << MatchTable::Comment("Predicate")
1206         << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
1207         << MatchTable::LineBreak;
1208 }
1209
1210 //===- OperandMatcher -----------------------------------------------------===//
1211
1212 std::string OperandMatcher::getOperandExpr(unsigned InsnVarID) const {
1213   return "State.MIs[" + llvm::to_string(InsnVarID) + "]->getOperand(" +
1214          llvm::to_string(OpIdx) + ")";
1215 }
1216
1217 unsigned OperandMatcher::getInsnVarID() const { return Insn.getInsnVarID(); }
1218
1219 TempTypeIdx OperandMatcher::getTempTypeIdx(RuleMatcher &Rule) {
1220   if (TTIdx >= 0) {
1221     // Temp type index not assigned yet, so assign one and add the necessary
1222     // predicate.
1223     TTIdx = Rule.getNextTempTypeIdx();
1224     assert(TTIdx < 0);
1225     addPredicate<RecordRegisterType>(TTIdx);
1226     return TTIdx;
1227   }
1228   return TTIdx;
1229 }
1230
1231 void OperandMatcher::emitPredicateOpcodes(MatchTable &Table,
1232                                           RuleMatcher &Rule) {
1233   if (!Optimized) {
1234     std::string Comment;
1235     raw_string_ostream CommentOS(Comment);
1236     CommentOS << "MIs[" << getInsnVarID() << "] ";
1237     if (SymbolicName.empty())
1238       CommentOS << "Operand " << OpIdx;
1239     else
1240       CommentOS << SymbolicName;
1241     Table << MatchTable::Comment(Comment) << MatchTable::LineBreak;
1242   }
1243
1244   emitPredicateListOpcodes(Table, Rule);
1245 }
1246
1247 bool OperandMatcher::isHigherPriorityThan(OperandMatcher &B) {
1248   // Operand matchers involving more predicates have higher priority.
1249   if (predicates_size() > B.predicates_size())
1250     return true;
1251   if (predicates_size() < B.predicates_size())
1252     return false;
1253
1254   // This assumes that predicates are added in a consistent order.
1255   for (auto &&Predicate : zip(predicates(), B.predicates())) {
1256     if (std::get<0>(Predicate)->isHigherPriorityThan(*std::get<1>(Predicate)))
1257       return true;
1258     if (std::get<1>(Predicate)->isHigherPriorityThan(*std::get<0>(Predicate)))
1259       return false;
1260   }
1261
1262   return false;
1263 }
1264
1265 unsigned OperandMatcher::countRendererFns() {
1266   return std::accumulate(
1267       predicates().begin(), predicates().end(), 0,
1268       [](unsigned A,
1269          const std::unique_ptr<OperandPredicateMatcher> &Predicate) {
1270         return A + Predicate->countRendererFns();
1271       });
1272 }
1273
1274 Error OperandMatcher::addTypeCheckPredicate(const TypeSetByHwMode &VTy,
1275                                             bool OperandIsAPointer) {
1276   if (!VTy.isMachineValueType())
1277     return failUnsupported("unsupported typeset");
1278
1279   if (VTy.getMachineValueType() == MVT::iPTR && OperandIsAPointer) {
1280     addPredicate<PointerToAnyOperandMatcher>(0);
1281     return Error::success();
1282   }
1283
1284   auto OpTyOrNone = MVTToLLT(VTy.getMachineValueType().SimpleTy);
1285   if (!OpTyOrNone)
1286     return failUnsupported("unsupported type");
1287
1288   if (OperandIsAPointer)
1289     addPredicate<PointerToAnyOperandMatcher>(OpTyOrNone->get().getSizeInBits());
1290   else if (VTy.isPointer())
1291     addPredicate<LLTOperandMatcher>(
1292         LLT::pointer(VTy.getPtrAddrSpace(), OpTyOrNone->get().getSizeInBits()));
1293   else
1294     addPredicate<LLTOperandMatcher>(*OpTyOrNone);
1295   return Error::success();
1296 }
1297
1298 //===- InstructionOpcodeMatcher -------------------------------------------===//
1299
1300 DenseMap<const CodeGenInstruction *, unsigned>
1301     InstructionOpcodeMatcher::OpcodeValues;
1302
1303 MatchTableRecord
1304 InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
1305   const auto VI = OpcodeValues.find(I);
1306   if (VI != OpcodeValues.end())
1307     return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
1308                                   VI->second);
1309   return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
1310 }
1311
1312 void InstructionOpcodeMatcher::initOpcodeValuesMap(
1313     const CodeGenTarget &Target) {
1314   OpcodeValues.clear();
1315
1316   unsigned OpcodeValue = 0;
1317   for (const CodeGenInstruction *I : Target.getInstructionsByEnumValue())
1318     OpcodeValues[I] = OpcodeValue++;
1319 }
1320
1321 MatchTableRecord InstructionOpcodeMatcher::getValue() const {
1322   assert(Insts.size() == 1);
1323
1324   const CodeGenInstruction *I = Insts[0];
1325   const auto VI = OpcodeValues.find(I);
1326   if (VI != OpcodeValues.end())
1327     return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
1328                                   VI->second);
1329   return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
1330 }
1331
1332 void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table,
1333                                                     RuleMatcher &Rule) const {
1334   StringRef CheckType =
1335       Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
1336   Table << MatchTable::Opcode(CheckType) << MatchTable::Comment("MI")
1337         << MatchTable::IntValue(InsnVarID);
1338
1339   for (const CodeGenInstruction *I : Insts)
1340     Table << getInstValue(I);
1341   Table << MatchTable::LineBreak;
1342 }
1343
1344 bool InstructionOpcodeMatcher::isHigherPriorityThan(
1345     const InstructionPredicateMatcher &B) const {
1346   if (InstructionPredicateMatcher::isHigherPriorityThan(B))
1347     return true;
1348   if (B.InstructionPredicateMatcher::isHigherPriorityThan(*this))
1349     return false;
1350
1351   // Prioritize opcodes for cosmetic reasons in the generated source. Although
1352   // this is cosmetic at the moment, we may want to drive a similar ordering
1353   // using instruction frequency information to improve compile time.
1354   if (const InstructionOpcodeMatcher *BO =
1355           dyn_cast<InstructionOpcodeMatcher>(&B))
1356     return Insts[0]->TheDef->getName() < BO->Insts[0]->TheDef->getName();
1357
1358   return false;
1359 }
1360
1361 bool InstructionOpcodeMatcher::isConstantInstruction() const {
1362   return Insts.size() == 1 && Insts[0]->TheDef->getName() == "G_CONSTANT";
1363 }
1364
1365 StringRef InstructionOpcodeMatcher::getOpcode() const {
1366   return Insts[0]->TheDef->getName();
1367 }
1368
1369 bool InstructionOpcodeMatcher::isVariadicNumOperands() const {
1370   // If one is variadic, they all should be.
1371   return Insts[0]->Operands.isVariadic;
1372 }
1373
1374 StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
1375   // Types expected to be uniform for all alternatives.
1376   return Insts[0]->Operands[OpIdx].OperandType;
1377 }
1378
1379 //===- InstructionNumOperandsMatcher --------------------------------------===//
1380
1381 void InstructionNumOperandsMatcher::emitPredicateOpcodes(
1382     MatchTable &Table, RuleMatcher &Rule) const {
1383   Table << MatchTable::Opcode("GIM_CheckNumOperands")
1384         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1385         << MatchTable::Comment("Expected") << MatchTable::IntValue(NumOperands)
1386         << MatchTable::LineBreak;
1387 }
1388
1389 //===- InstructionImmPredicateMatcher -------------------------------------===//
1390
1391 bool InstructionImmPredicateMatcher::isIdentical(
1392     const PredicateMatcher &B) const {
1393   return InstructionPredicateMatcher::isIdentical(B) &&
1394          Predicate.getOrigPatFragRecord() ==
1395              cast<InstructionImmPredicateMatcher>(&B)
1396                  ->Predicate.getOrigPatFragRecord();
1397 }
1398
1399 void InstructionImmPredicateMatcher::emitPredicateOpcodes(
1400     MatchTable &Table, RuleMatcher &Rule) const {
1401   Table << MatchTable::Opcode(getMatchOpcodeForImmPredicate(Predicate))
1402         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1403         << MatchTable::Comment("Predicate")
1404         << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
1405         << MatchTable::LineBreak;
1406 }
1407
1408 //===- AtomicOrderingMMOPredicateMatcher ----------------------------------===//
1409
1410 bool AtomicOrderingMMOPredicateMatcher::isIdentical(
1411     const PredicateMatcher &B) const {
1412   if (!InstructionPredicateMatcher::isIdentical(B))
1413     return false;
1414   const auto &R = *cast<AtomicOrderingMMOPredicateMatcher>(&B);
1415   return Order == R.Order && Comparator == R.Comparator;
1416 }
1417
1418 void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
1419     MatchTable &Table, RuleMatcher &Rule) const {
1420   StringRef Opcode = "GIM_CheckAtomicOrdering";
1421
1422   if (Comparator == AO_OrStronger)
1423     Opcode = "GIM_CheckAtomicOrderingOrStrongerThan";
1424   if (Comparator == AO_WeakerThan)
1425     Opcode = "GIM_CheckAtomicOrderingWeakerThan";
1426
1427   Table << MatchTable::Opcode(Opcode) << MatchTable::Comment("MI")
1428         << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Order")
1429         << MatchTable::NamedValue(("(int64_t)AtomicOrdering::" + Order).str())
1430         << MatchTable::LineBreak;
1431 }
1432
1433 //===- MemorySizePredicateMatcher -----------------------------------------===//
1434
1435 void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
1436                                                       RuleMatcher &Rule) const {
1437   Table << MatchTable::Opcode("GIM_CheckMemorySizeEqualTo")
1438         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1439         << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
1440         << MatchTable::Comment("Size") << MatchTable::IntValue(Size)
1441         << MatchTable::LineBreak;
1442 }
1443
1444 //===- MemoryAddressSpacePredicateMatcher ---------------------------------===//
1445
1446 bool MemoryAddressSpacePredicateMatcher::isIdentical(
1447     const PredicateMatcher &B) const {
1448   if (!InstructionPredicateMatcher::isIdentical(B))
1449     return false;
1450   auto *Other = cast<MemoryAddressSpacePredicateMatcher>(&B);
1451   return MMOIdx == Other->MMOIdx && AddrSpaces == Other->AddrSpaces;
1452 }
1453
1454 void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
1455     MatchTable &Table, RuleMatcher &Rule) const {
1456   Table << MatchTable::Opcode("GIM_CheckMemoryAddressSpace")
1457         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1458         << MatchTable::Comment("MMO")
1459         << MatchTable::IntValue(MMOIdx)
1460         // Encode number of address spaces to expect.
1461         << MatchTable::Comment("NumAddrSpace")
1462         << MatchTable::IntValue(AddrSpaces.size());
1463   for (unsigned AS : AddrSpaces)
1464     Table << MatchTable::Comment("AddrSpace") << MatchTable::IntValue(AS);
1465
1466   Table << MatchTable::LineBreak;
1467 }
1468
1469 //===- MemoryAlignmentPredicateMatcher ------------------------------------===//
1470
1471 bool MemoryAlignmentPredicateMatcher::isIdentical(
1472     const PredicateMatcher &B) const {
1473   if (!InstructionPredicateMatcher::isIdentical(B))
1474     return false;
1475   auto *Other = cast<MemoryAlignmentPredicateMatcher>(&B);
1476   return MMOIdx == Other->MMOIdx && MinAlign == Other->MinAlign;
1477 }
1478
1479 void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
1480     MatchTable &Table, RuleMatcher &Rule) const {
1481   Table << MatchTable::Opcode("GIM_CheckMemoryAlignment")
1482         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1483         << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
1484         << MatchTable::Comment("MinAlign") << MatchTable::IntValue(MinAlign)
1485         << MatchTable::LineBreak;
1486 }
1487
1488 //===- MemoryVsLLTSizePredicateMatcher ------------------------------------===//
1489
1490 bool MemoryVsLLTSizePredicateMatcher::isIdentical(
1491     const PredicateMatcher &B) const {
1492   return InstructionPredicateMatcher::isIdentical(B) &&
1493          MMOIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->MMOIdx &&
1494          Relation == cast<MemoryVsLLTSizePredicateMatcher>(&B)->Relation &&
1495          OpIdx == cast<MemoryVsLLTSizePredicateMatcher>(&B)->OpIdx;
1496 }
1497
1498 void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
1499     MatchTable &Table, RuleMatcher &Rule) const {
1500   Table << MatchTable::Opcode(
1501                Relation == EqualTo       ? "GIM_CheckMemorySizeEqualToLLT"
1502                : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
1503                                          : "GIM_CheckMemorySizeLessThanLLT")
1504         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1505         << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
1506         << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
1507         << MatchTable::LineBreak;
1508 }
1509
1510 //===- VectorSplatImmPredicateMatcher -------------------------------------===//
1511
1512 void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
1513     MatchTable &Table, RuleMatcher &Rule) const {
1514   if (Kind == AllOnes)
1515     Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllOnes");
1516   else
1517     Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllZeros");
1518
1519   Table << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID);
1520   Table << MatchTable::LineBreak;
1521 }
1522
1523 //===- GenericInstructionPredicateMatcher ---------------------------------===//
1524
1525 GenericInstructionPredicateMatcher::GenericInstructionPredicateMatcher(
1526     unsigned InsnVarID, TreePredicateFn Predicate)
1527     : GenericInstructionPredicateMatcher(InsnVarID,
1528                                          getEnumNameForPredicate(Predicate)) {}
1529
1530 bool GenericInstructionPredicateMatcher::isIdentical(
1531     const PredicateMatcher &B) const {
1532   return InstructionPredicateMatcher::isIdentical(B) &&
1533          EnumVal ==
1534              static_cast<const GenericInstructionPredicateMatcher &>(B).EnumVal;
1535 }
1536 void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
1537     MatchTable &Table, RuleMatcher &Rule) const {
1538   Table << MatchTable::Opcode("GIM_CheckCxxInsnPredicate")
1539         << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
1540         << MatchTable::Comment("FnId") << MatchTable::NamedValue(EnumVal)
1541         << MatchTable::LineBreak;
1542 }
1543
1544 //===- InstructionMatcher -------------------------------------------------===//
1545
1546 OperandMatcher &
1547 InstructionMatcher::addOperand(unsigned OpIdx, const std::string &SymbolicName,
1548                                unsigned AllocatedTemporariesBaseID) {
1549   Operands.emplace_back(new OperandMatcher(*this, OpIdx, SymbolicName,
1550                                            AllocatedTemporariesBaseID));
1551   if (!SymbolicName.empty())
1552     Rule.defineOperand(SymbolicName, *Operands.back());
1553
1554   return *Operands.back();
1555 }
1556
1557 OperandMatcher &InstructionMatcher::getOperand(unsigned OpIdx) {
1558   auto I = llvm::find_if(Operands,
1559                          [&OpIdx](const std::unique_ptr<OperandMatcher> &X) {
1560                            return X->getOpIdx() == OpIdx;
1561                          });
1562   if (I != Operands.end())
1563     return **I;
1564   llvm_unreachable("Failed to lookup operand");
1565 }
1566
1567 OperandMatcher &InstructionMatcher::addPhysRegInput(Record *Reg, unsigned OpIdx,
1568                                                     unsigned TempOpIdx) {
1569   assert(SymbolicName.empty());
1570   OperandMatcher *OM = new OperandMatcher(*this, OpIdx, "", TempOpIdx);
1571   Operands.emplace_back(OM);
1572   Rule.definePhysRegOperand(Reg, *OM);
1573   PhysRegInputs.emplace_back(Reg, OpIdx);
1574   return *OM;
1575 }
1576
1577 void InstructionMatcher::emitPredicateOpcodes(MatchTable &Table,
1578                                               RuleMatcher &Rule) {
1579   if (NumOperandsCheck)
1580     InstructionNumOperandsMatcher(InsnVarID, getNumOperands())
1581         .emitPredicateOpcodes(Table, Rule);
1582
1583   // First emit all instruction level predicates need to be verified before we
1584   // can verify operands.
1585   emitFilteredPredicateListOpcodes(
1586       [](const PredicateMatcher &P) { return !P.dependsOnOperands(); }, Table,
1587       Rule);
1588
1589   // Emit all operand constraints.
1590   for (const auto &Operand : Operands)
1591     Operand->emitPredicateOpcodes(Table, Rule);
1592
1593   // All of the tablegen defined predicates should now be matched. Now emit
1594   // any custom predicates that rely on all generated checks.
1595   emitFilteredPredicateListOpcodes(
1596       [](const PredicateMatcher &P) { return P.dependsOnOperands(); }, Table,
1597       Rule);
1598 }
1599
1600 bool InstructionMatcher::isHigherPriorityThan(InstructionMatcher &B) {
1601   // Instruction matchers involving more operands have higher priority.
1602   if (Operands.size() > B.Operands.size())
1603     return true;
1604   if (Operands.size() < B.Operands.size())
1605     return false;
1606
1607   for (auto &&P : zip(predicates(), B.predicates())) {
1608     auto L = static_cast<InstructionPredicateMatcher *>(std::get<0>(P).get());
1609     auto R = static_cast<InstructionPredicateMatcher *>(std::get<1>(P).get());
1610     if (L->isHigherPriorityThan(*R))
1611       return true;
1612     if (R->isHigherPriorityThan(*L))
1613       return false;
1614   }
1615
1616   for (auto Operand : zip(Operands, B.Operands)) {
1617     if (std::get<0>(Operand)->isHigherPriorityThan(*std::get<1>(Operand)))
1618       return true;
1619     if (std::get<1>(Operand)->isHigherPriorityThan(*std::get<0>(Operand)))
1620       return false;
1621   }
1622
1623   return false;
1624 }
1625
1626 unsigned InstructionMatcher::countRendererFns() {
1627   return std::accumulate(
1628              predicates().begin(), predicates().end(), 0,
1629              [](unsigned A,
1630                 const std::unique_ptr<PredicateMatcher> &Predicate) {
1631                return A + Predicate->countRendererFns();
1632              }) +
1633          std::accumulate(
1634              Operands.begin(), Operands.end(), 0,
1635              [](unsigned A, const std::unique_ptr<OperandMatcher> &Operand) {
1636                return A + Operand->countRendererFns();
1637              });
1638 }
1639
1640 void InstructionMatcher::optimize() {
1641   SmallVector<std::unique_ptr<PredicateMatcher>, 8> Stash;
1642   const auto &OpcMatcher = getOpcodeMatcher();
1643
1644   Stash.push_back(predicates_pop_front());
1645   if (Stash.back().get() == &OpcMatcher) {
1646     if (NumOperandsCheck && OpcMatcher.isVariadicNumOperands() &&
1647         getNumOperands() != 0)
1648       Stash.emplace_back(
1649           new InstructionNumOperandsMatcher(InsnVarID, getNumOperands()));
1650     NumOperandsCheck = false;
1651
1652     for (auto &OM : Operands)
1653       for (auto &OP : OM->predicates())
1654         if (isa<IntrinsicIDOperandMatcher>(OP)) {
1655           Stash.push_back(std::move(OP));
1656           OM->eraseNullPredicates();
1657           break;
1658         }
1659   }
1660
1661   if (InsnVarID > 0) {
1662     assert(!Operands.empty() && "Nested instruction is expected to def a vreg");
1663     for (auto &OP : Operands[0]->predicates())
1664       OP.reset();
1665     Operands[0]->eraseNullPredicates();
1666   }
1667   for (auto &OM : Operands) {
1668     for (auto &OP : OM->predicates())
1669       if (isa<LLTOperandMatcher>(OP))
1670         Stash.push_back(std::move(OP));
1671     OM->eraseNullPredicates();
1672   }
1673   while (!Stash.empty())
1674     prependPredicate(Stash.pop_back_val());
1675 }
1676
1677 //===- InstructionOperandMatcher ------------------------------------------===//
1678
1679 void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table,
1680                                                    RuleMatcher &Rule) const {
1681   const unsigned NewInsnVarID = InsnMatcher->getInsnVarID();
1682   const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
1683   Table << MatchTable::Opcode(IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
1684                                            : "GIM_RecordInsn")
1685         << MatchTable::Comment("DefineMI") << MatchTable::IntValue(NewInsnVarID)
1686         << MatchTable::Comment("MI") << MatchTable::IntValue(getInsnVarID())
1687         << MatchTable::Comment("OpIdx") << MatchTable::IntValue(getOpIdx())
1688         << MatchTable::Comment("MIs[" + llvm::to_string(NewInsnVarID) + "]")
1689         << MatchTable::LineBreak;
1690 }
1691
1692 bool InstructionOperandMatcher::isHigherPriorityThan(
1693     const OperandPredicateMatcher &B) const {
1694   if (OperandPredicateMatcher::isHigherPriorityThan(B))
1695     return true;
1696   if (B.OperandPredicateMatcher::isHigherPriorityThan(*this))
1697     return false;
1698
1699   if (const InstructionOperandMatcher *BP =
1700           dyn_cast<InstructionOperandMatcher>(&B))
1701     if (InsnMatcher->isHigherPriorityThan(*BP->InsnMatcher))
1702       return true;
1703   return false;
1704 }
1705
1706 //===- OperandRenderer ----------------------------------------------------===//
1707
1708 OperandRenderer::~OperandRenderer() {}
1709
1710 //===- CopyRenderer -------------------------------------------------------===//
1711
1712 void CopyRenderer::emitRenderOpcodes(MatchTable &Table,
1713                                      RuleMatcher &Rule) const {
1714   const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
1715   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
1716   Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
1717         << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
1718         << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
1719         << MatchTable::IntValue(Operand.getOpIdx())
1720         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1721 }
1722
1723 //===- CopyPhysRegRenderer ------------------------------------------------===//
1724
1725 void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table,
1726                                             RuleMatcher &Rule) const {
1727   const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(PhysReg);
1728   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
1729   Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
1730         << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
1731         << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
1732         << MatchTable::IntValue(Operand.getOpIdx())
1733         << MatchTable::Comment(PhysReg->getName()) << MatchTable::LineBreak;
1734 }
1735
1736 //===- CopyOrAddZeroRegRenderer -------------------------------------------===//
1737
1738 void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table,
1739                                                  RuleMatcher &Rule) const {
1740   const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
1741   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
1742   Table << MatchTable::Opcode("GIR_CopyOrAddZeroReg")
1743         << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
1744         << MatchTable::Comment("OldInsnID")
1745         << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
1746         << MatchTable::IntValue(Operand.getOpIdx())
1747         << MatchTable::NamedValue(
1748                (ZeroRegisterDef->getValue("Namespace")
1749                     ? ZeroRegisterDef->getValueAsString("Namespace")
1750                     : ""),
1751                ZeroRegisterDef->getName())
1752         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1753 }
1754
1755 //===- CopyConstantAsImmRenderer ------------------------------------------===//
1756
1757 void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table,
1758                                                   RuleMatcher &Rule) const {
1759   InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
1760   unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
1761   Table << MatchTable::Opcode(Signed ? "GIR_CopyConstantAsSImm"
1762                                      : "GIR_CopyConstantAsUImm")
1763         << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
1764         << MatchTable::Comment("OldInsnID")
1765         << MatchTable::IntValue(OldInsnVarID)
1766         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1767 }
1768
1769 //===- CopyFConstantAsFPImmRenderer ---------------------------------------===//
1770
1771 void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table,
1772                                                      RuleMatcher &Rule) const {
1773   InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
1774   unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
1775   Table << MatchTable::Opcode("GIR_CopyFConstantAsFPImm")
1776         << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
1777         << MatchTable::Comment("OldInsnID")
1778         << MatchTable::IntValue(OldInsnVarID)
1779         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1780 }
1781
1782 //===- CopySubRegRenderer -------------------------------------------------===//
1783
1784 void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table,
1785                                            RuleMatcher &Rule) const {
1786   const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
1787   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
1788   Table << MatchTable::Opcode("GIR_CopySubReg")
1789         << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
1790         << MatchTable::Comment("OldInsnID")
1791         << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
1792         << MatchTable::IntValue(Operand.getOpIdx())
1793         << MatchTable::Comment("SubRegIdx")
1794         << MatchTable::IntValue(SubReg->EnumValue)
1795         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1796 }
1797
1798 //===- AddRegisterRenderer ------------------------------------------------===//
1799
1800 void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table,
1801                                             RuleMatcher &Rule) const {
1802   Table << MatchTable::Opcode("GIR_AddRegister")
1803         << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID);
1804   if (RegisterDef->getName() != "zero_reg") {
1805     Table << MatchTable::NamedValue(
1806         (RegisterDef->getValue("Namespace")
1807              ? RegisterDef->getValueAsString("Namespace")
1808              : ""),
1809         RegisterDef->getName());
1810   } else {
1811     Table << MatchTable::NamedValue(Target.getRegNamespace(), "NoRegister");
1812   }
1813   Table << MatchTable::Comment("AddRegisterRegFlags");
1814
1815   // TODO: This is encoded as a 64-bit element, but only 16 or 32-bits are
1816   // really needed for a physical register reference. We can pack the
1817   // register and flags in a single field.
1818   if (IsDef)
1819     Table << MatchTable::NamedValue("RegState::Define");
1820   else
1821     Table << MatchTable::IntValue(0);
1822   Table << MatchTable::LineBreak;
1823 }
1824
1825 //===- TempRegRenderer ----------------------------------------------------===//
1826
1827 void TempRegRenderer::emitRenderOpcodes(MatchTable &Table,
1828                                         RuleMatcher &Rule) const {
1829   if (SubRegIdx) {
1830     assert(!IsDef);
1831     Table << MatchTable::Opcode("GIR_AddTempSubRegister");
1832   } else
1833     Table << MatchTable::Opcode("GIR_AddTempRegister");
1834
1835   Table << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1836         << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
1837         << MatchTable::Comment("TempRegFlags");
1838
1839   if (IsDef) {
1840     SmallString<32> RegFlags;
1841     RegFlags += "RegState::Define";
1842     if (IsDead)
1843       RegFlags += "|RegState::Dead";
1844     Table << MatchTable::NamedValue(RegFlags);
1845   } else
1846     Table << MatchTable::IntValue(0);
1847
1848   if (SubRegIdx)
1849     Table << MatchTable::NamedValue(SubRegIdx->getQualifiedName());
1850   Table << MatchTable::LineBreak;
1851 }
1852
1853 //===- SubRegIndexRenderer ------------------------------------------------===//
1854
1855 void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table,
1856                                             RuleMatcher &Rule) const {
1857   Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
1858         << MatchTable::IntValue(InsnID) << MatchTable::Comment("SubRegIndex")
1859         << MatchTable::IntValue(SubRegIdx->EnumValue) << MatchTable::LineBreak;
1860 }
1861
1862 //===- RenderComplexPatternOperand ----------------------------------------===//
1863
1864 void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table,
1865                                                     RuleMatcher &Rule) const {
1866   Table << MatchTable::Opcode(
1867                SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
1868                                     : "GIR_ComplexSubOperandRenderer")
1869                           : "GIR_ComplexRenderer")
1870         << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1871         << MatchTable::Comment("RendererID")
1872         << MatchTable::IntValue(RendererID);
1873   if (SubOperand)
1874     Table << MatchTable::Comment("SubOperand")
1875           << MatchTable::IntValue(*SubOperand);
1876   if (SubReg)
1877     Table << MatchTable::Comment("SubRegIdx")
1878           << MatchTable::IntValue(SubReg->EnumValue);
1879   Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1880 }
1881
1882 //===- CustomRenderer -----------------------------------------------------===//
1883
1884 void CustomRenderer::emitRenderOpcodes(MatchTable &Table,
1885                                        RuleMatcher &Rule) const {
1886   InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
1887   unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
1888   Table << MatchTable::Opcode("GIR_CustomRenderer")
1889         << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1890         << MatchTable::Comment("OldInsnID")
1891         << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("Renderer")
1892         << MatchTable::NamedValue("GICR_" +
1893                                   Renderer.getValueAsString("RendererFn").str())
1894         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1895 }
1896
1897 //===- CustomOperandRenderer ----------------------------------------------===//
1898
1899 void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table,
1900                                               RuleMatcher &Rule) const {
1901   const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName);
1902   Table << MatchTable::Opcode("GIR_CustomOperandRenderer")
1903         << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1904         << MatchTable::Comment("OldInsnID")
1905         << MatchTable::IntValue(OpdMatcher.getInsnVarID())
1906         << MatchTable::Comment("OpIdx")
1907         << MatchTable::IntValue(OpdMatcher.getOpIdx())
1908         << MatchTable::Comment("OperandRenderer")
1909         << MatchTable::NamedValue("GICR_" +
1910                                   Renderer.getValueAsString("RendererFn").str())
1911         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
1912 }
1913
1914 //===- CustomCXXAction ----------------------------------------------------===//
1915
1916 void CustomCXXAction::emitActionOpcodes(MatchTable &Table,
1917                                         RuleMatcher &Rule) const {
1918   Table << MatchTable::Opcode("GIR_CustomAction")
1919         << MatchTable::NamedValue(FnEnumName) << MatchTable::LineBreak;
1920 }
1921
1922 //===- BuildMIAction ------------------------------------------------------===//
1923
1924 bool BuildMIAction::canMutate(RuleMatcher &Rule,
1925                               const InstructionMatcher *Insn) const {
1926   if (!Insn)
1927     return false;
1928
1929   if (OperandRenderers.size() != Insn->getNumOperands())
1930     return false;
1931
1932   for (const auto &Renderer : enumerate(OperandRenderers)) {
1933     if (const auto *Copy = dyn_cast<CopyRenderer>(&*Renderer.value())) {
1934       const OperandMatcher &OM =
1935           Rule.getOperandMatcher(Copy->getSymbolicName());
1936       if (Insn != &OM.getInstructionMatcher() ||
1937           OM.getOpIdx() != Renderer.index())
1938         return false;
1939     } else
1940       return false;
1941   }
1942
1943   return true;
1944 }
1945
1946 void BuildMIAction::chooseInsnToMutate(RuleMatcher &Rule) {
1947   for (auto *MutateCandidate : Rule.mutatable_insns()) {
1948     if (canMutate(Rule, MutateCandidate)) {
1949       // Take the first one we're offered that we're able to mutate.
1950       Rule.reserveInsnMatcherForMutation(MutateCandidate);
1951       Matched = MutateCandidate;
1952       return;
1953     }
1954   }
1955 }
1956
1957 void BuildMIAction::emitActionOpcodes(MatchTable &Table,
1958                                       RuleMatcher &Rule) const {
1959   if (Matched) {
1960     assert(canMutate(Rule, Matched) &&
1961            "Arranged to mutate an insn that isn't mutatable");
1962
1963     unsigned RecycleInsnID = Rule.getInsnVarID(*Matched);
1964     Table << MatchTable::Opcode("GIR_MutateOpcode")
1965           << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1966           << MatchTable::Comment("RecycleInsnID")
1967           << MatchTable::IntValue(RecycleInsnID)
1968           << MatchTable::Comment("Opcode")
1969           << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
1970           << MatchTable::LineBreak;
1971
1972     if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
1973       for (auto *Def : I->ImplicitDefs) {
1974         auto Namespace = Def->getValue("Namespace")
1975                              ? Def->getValueAsString("Namespace")
1976                              : "";
1977         const bool IsDead = DeadImplicitDefs.contains(Def);
1978         Table << MatchTable::Opcode("GIR_AddImplicitDef")
1979               << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1980               << MatchTable::NamedValue(Namespace, Def->getName())
1981               << (IsDead ? MatchTable::NamedValue("RegState", "Dead")
1982                          : MatchTable::IntValue(0))
1983               << MatchTable::LineBreak;
1984       }
1985       for (auto *Use : I->ImplicitUses) {
1986         auto Namespace = Use->getValue("Namespace")
1987                              ? Use->getValueAsString("Namespace")
1988                              : "";
1989         Table << MatchTable::Opcode("GIR_AddImplicitUse")
1990               << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
1991               << MatchTable::NamedValue(Namespace, Use->getName())
1992               << MatchTable::LineBreak;
1993       }
1994     }
1995     return;
1996   }
1997
1998   // TODO: Simple permutation looks like it could be almost as common as
1999   //       mutation due to commutative operations.
2000
2001   Table << MatchTable::Opcode("GIR_BuildMI") << MatchTable::Comment("InsnID")
2002         << MatchTable::IntValue(InsnID) << MatchTable::Comment("Opcode")
2003         << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
2004         << MatchTable::LineBreak;
2005   for (const auto &Renderer : OperandRenderers)
2006     Renderer->emitRenderOpcodes(Table, Rule);
2007
2008   for (auto [OpIdx, Def] : enumerate(I->ImplicitDefs)) {
2009     auto Namespace =
2010         Def->getValue("Namespace") ? Def->getValueAsString("Namespace") : "";
2011     if (DeadImplicitDefs.contains(Def)) {
2012       Table
2013           << MatchTable::Opcode("GIR_SetImplicitDefDead")
2014           << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
2015           << MatchTable::Comment(
2016                  ("OpIdx for " + Namespace + "::" + Def->getName() + "").str())
2017           << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
2018     }
2019   }
2020
2021   if (I->mayLoad || I->mayStore) {
2022     Table << MatchTable::Opcode("GIR_MergeMemOperands")
2023           << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
2024           << MatchTable::Comment("MergeInsnID's");
2025     // Emit the ID's for all the instructions that are matched by this rule.
2026     // TODO: Limit this to matched instructions that mayLoad/mayStore or have
2027     //       some other means of having a memoperand. Also limit this to
2028     //       emitted instructions that expect to have a memoperand too. For
2029     //       example, (G_SEXT (G_LOAD x)) that results in separate load and
2030     //       sign-extend instructions shouldn't put the memoperand on the
2031     //       sign-extend since it has no effect there.
2032     std::vector<unsigned> MergeInsnIDs;
2033     for (const auto &IDMatcherPair : Rule.defined_insn_vars())
2034       MergeInsnIDs.push_back(IDMatcherPair.second);
2035     llvm::sort(MergeInsnIDs);
2036     for (const auto &MergeInsnID : MergeInsnIDs)
2037       Table << MatchTable::IntValue(MergeInsnID);
2038     Table << MatchTable::NamedValue("GIU_MergeMemOperands_EndOfList")
2039           << MatchTable::LineBreak;
2040   }
2041
2042   // FIXME: This is a hack but it's sufficient for ISel. We'll need to do
2043   //        better for combines. Particularly when there are multiple match
2044   //        roots.
2045   if (InsnID == 0)
2046     EraseInstAction::emitActionOpcodes(Table, Rule, /*InsnID*/ 0);
2047 }
2048
2049 //===- BuildConstantAction ------------------------------------------------===//
2050
2051 void BuildConstantAction::emitActionOpcodes(MatchTable &Table,
2052                                             RuleMatcher &Rule) const {
2053   Table << MatchTable::Opcode("GIR_BuildConstant")
2054         << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
2055         << MatchTable::Comment("Val") << MatchTable::IntValue(Val)
2056         << MatchTable::LineBreak;
2057 }
2058
2059 //===- EraseInstAction ----------------------------------------------------===//
2060
2061 void EraseInstAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule,
2062                                         unsigned InsnID) {
2063   // Avoid erasing the same inst twice.
2064   if (!Rule.tryEraseInsnID(InsnID))
2065     return;
2066
2067   Table << MatchTable::Opcode("GIR_EraseFromParent")
2068         << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
2069         << MatchTable::LineBreak;
2070 }
2071
2072 void EraseInstAction::emitActionOpcodes(MatchTable &Table,
2073                                         RuleMatcher &Rule) const {
2074   emitActionOpcodes(Table, Rule, InsnID);
2075 }
2076
2077 //===- ReplaceRegAction ---------------------------------------------------===//
2078
2079 void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table,
2080                                                 RuleMatcher &Rule) const {
2081   if (TempRegID != (unsigned)-1)
2082     return;
2083
2084   Table << MatchTable::Opcode("GIM_CheckCanReplaceReg")
2085         << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID)
2086         << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx)
2087         << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId)
2088         << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx)
2089         << MatchTable::LineBreak;
2090 }
2091
2092 void ReplaceRegAction::emitActionOpcodes(MatchTable &Table,
2093                                          RuleMatcher &Rule) const {
2094   if (TempRegID != (unsigned)-1) {
2095     Table << MatchTable::Opcode("GIR_ReplaceRegWithTempReg")
2096           << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID)
2097           << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx)
2098           << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
2099           << MatchTable::LineBreak;
2100   } else {
2101     Table << MatchTable::Opcode("GIR_ReplaceReg")
2102           << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID)
2103           << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx)
2104           << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId)
2105           << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx)
2106           << MatchTable::LineBreak;
2107   }
2108 }
2109
2110 //===- ConstrainOperandToRegClassAction -----------------------------------===//
2111
2112 void ConstrainOperandToRegClassAction::emitActionOpcodes(
2113     MatchTable &Table, RuleMatcher &Rule) const {
2114   Table << MatchTable::Opcode("GIR_ConstrainOperandRC")
2115         << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
2116         << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
2117         << MatchTable::NamedValue(RC.getQualifiedIdName())
2118         << MatchTable::LineBreak;
2119 }
2120
2121 //===- MakeTempRegisterAction ---------------------------------------------===//
2122
2123 void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table,
2124                                                RuleMatcher &Rule) const {
2125   Table << MatchTable::Opcode("GIR_MakeTempReg")
2126         << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
2127         << MatchTable::Comment("TypeID") << Ty << MatchTable::LineBreak;
2128 }
2129
2130 } // namespace gi
2131 } // namespace llvm