llvm/lib/CodeGen/MIRParser/MIParser.cpp

   1 //===- MIParser.cpp - Machine instructions parser implementation ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements the parsing of machine instructions.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "llvm/CodeGen/MIRParser/MIParser.h"
  14 #include "MILexer.h"
  15 #include "llvm/ADT/APInt.h"
  16 #include "llvm/ADT/APSInt.h"
  17 #include "llvm/ADT/ArrayRef.h"
  18 #include "llvm/ADT/DenseMap.h"
  19 #include "llvm/ADT/SmallVector.h"
  20 #include "llvm/ADT/StringMap.h"
  21 #include "llvm/ADT/StringRef.h"
  22 #include "llvm/ADT/StringSwitch.h"
  23 #include "llvm/ADT/Twine.h"
  24 #include "llvm/Analysis/MemoryLocation.h"
  25 #include "llvm/AsmParser/Parser.h"
  26 #include "llvm/AsmParser/SlotMapping.h"
  27 #include "llvm/CodeGen/LowLevelType.h"
  28 #include "llvm/CodeGen/MIRFormatter.h"
  29 #include "llvm/CodeGen/MIRPrinter.h"
  30 #include "llvm/CodeGen/MachineBasicBlock.h"
  31 #include "llvm/CodeGen/MachineFrameInfo.h"
  32 #include "llvm/CodeGen/MachineFunction.h"
  33 #include "llvm/CodeGen/MachineInstr.h"
  34 #include "llvm/CodeGen/MachineInstrBuilder.h"
  35 #include "llvm/CodeGen/MachineMemOperand.h"
  36 #include "llvm/CodeGen/MachineOperand.h"
  37 #include "llvm/CodeGen/MachineRegisterInfo.h"
  38 #include "llvm/CodeGen/RegisterBank.h"
  39 #include "llvm/CodeGen/RegisterBankInfo.h"
  40 #include "llvm/CodeGen/TargetInstrInfo.h"
  41 #include "llvm/CodeGen/TargetRegisterInfo.h"
  42 #include "llvm/CodeGen/TargetSubtargetInfo.h"
  43 #include "llvm/IR/BasicBlock.h"
  44 #include "llvm/IR/Constants.h"
  45 #include "llvm/IR/DataLayout.h"
  46 #include "llvm/IR/DebugInfoMetadata.h"
  47 #include "llvm/IR/DebugLoc.h"
  48 #include "llvm/IR/Function.h"
  49 #include "llvm/IR/InstrTypes.h"
  50 #include "llvm/IR/Instructions.h"
  51 #include "llvm/IR/Intrinsics.h"
  52 #include "llvm/IR/Metadata.h"
  53 #include "llvm/IR/Module.h"
  54 #include "llvm/IR/ModuleSlotTracker.h"
  55 #include "llvm/IR/Type.h"
  56 #include "llvm/IR/Value.h"
  57 #include "llvm/IR/ValueSymbolTable.h"
  58 #include "llvm/MC/LaneBitmask.h"
  59 #include "llvm/MC/MCContext.h"
  60 #include "llvm/MC/MCDwarf.h"
  61 #include "llvm/MC/MCInstrDesc.h"
  62 #include "llvm/Support/AtomicOrdering.h"
  63 #include "llvm/Support/BranchProbability.h"
  64 #include "llvm/Support/Casting.h"
  65 #include "llvm/Support/ErrorHandling.h"
  66 #include "llvm/Support/MemoryBuffer.h"
  67 #include "llvm/Support/SMLoc.h"
  68 #include "llvm/Support/SourceMgr.h"
  69 #include "llvm/Target/TargetIntrinsicInfo.h"
  70 #include "llvm/Target/TargetMachine.h"
  71 #include <cassert>
  72 #include <cctype>
  73 #include <cstddef>
  74 #include <cstdint>
  75 #include <limits>
  76 #include <string>
  77 #include <utility>
  78
  79 using namespace llvm;
  80
  81 void PerTargetMIParsingState::setTarget(
  82   const TargetSubtargetInfo &NewSubtarget) {
  83
  84   // If the subtarget changed, over conservatively assume everything is invalid.
  85   if (&Subtarget == &NewSubtarget)
  86     return;
  87
  88   Names2InstrOpCodes.clear();
  89   Names2Regs.clear();
  90   Names2RegMasks.clear();
  91   Names2SubRegIndices.clear();
  92   Names2TargetIndices.clear();
  93   Names2DirectTargetFlags.clear();
  94   Names2BitmaskTargetFlags.clear();
  95   Names2MMOTargetFlags.clear();
  96
  97   initNames2RegClasses();
  98   initNames2RegBanks();
  99 }
 100
 101 void PerTargetMIParsingState::initNames2Regs() {
 102   if (!Names2Regs.empty())
 103     return;
 104
 105   // The '%noreg' register is the register 0.
 106   Names2Regs.insert(std::make_pair("noreg", 0));
 107   const auto *TRI = Subtarget.getRegisterInfo();
 108   assert(TRI && "Expected target register info");
 109
 110   for (unsigned I = 0, E = TRI->getNumRegs(); I < E; ++I) {
 111     bool WasInserted =
 112         Names2Regs.insert(std::make_pair(StringRef(TRI->getName(I)).lower(), I))
 113             .second;
 114     (void)WasInserted;
 115     assert(WasInserted && "Expected registers to be unique case-insensitively");
 116   }
 117 }
 118
 119 bool PerTargetMIParsingState::getRegisterByName(StringRef RegName,
 120                                                 Register &Reg) {
 121   initNames2Regs();
 122   auto RegInfo = Names2Regs.find(RegName);
 123   if (RegInfo == Names2Regs.end())
 124     return true;
 125   Reg = RegInfo->getValue();
 126   return false;
 127 }
 128
 129 void PerTargetMIParsingState::initNames2InstrOpCodes() {
 130   if (!Names2InstrOpCodes.empty())
 131     return;
 132   const auto *TII = Subtarget.getInstrInfo();
 133   assert(TII && "Expected target instruction info");
 134   for (unsigned I = 0, E = TII->getNumOpcodes(); I < E; ++I)
 135     Names2InstrOpCodes.insert(std::make_pair(StringRef(TII->getName(I)), I));
 136 }
 137
 138 bool PerTargetMIParsingState::parseInstrName(StringRef InstrName,
 139                                              unsigned &OpCode) {
 140   initNames2InstrOpCodes();
 141   auto InstrInfo = Names2InstrOpCodes.find(InstrName);
 142   if (InstrInfo == Names2InstrOpCodes.end())
 143     return true;
 144   OpCode = InstrInfo->getValue();
 145   return false;
 146 }
 147
 148 void PerTargetMIParsingState::initNames2RegMasks() {
 149   if (!Names2RegMasks.empty())
 150     return;
 151   const auto *TRI = Subtarget.getRegisterInfo();
 152   assert(TRI && "Expected target register info");
 153   ArrayRef<const uint32_t *> RegMasks = TRI->getRegMasks();
 154   ArrayRef<const char *> RegMaskNames = TRI->getRegMaskNames();
 155   assert(RegMasks.size() == RegMaskNames.size());
 156   for (size_t I = 0, E = RegMasks.size(); I < E; ++I)
 157     Names2RegMasks.insert(
 158         std::make_pair(StringRef(RegMaskNames[I]).lower(), RegMasks[I]));
 159 }
 160
 161 const uint32_t *PerTargetMIParsingState::getRegMask(StringRef Identifier) {
 162   initNames2RegMasks();
 163   auto RegMaskInfo = Names2RegMasks.find(Identifier);
 164   if (RegMaskInfo == Names2RegMasks.end())
 165     return nullptr;
 166   return RegMaskInfo->getValue();
 167 }
 168
 169 void PerTargetMIParsingState::initNames2SubRegIndices() {
 170   if (!Names2SubRegIndices.empty())
 171     return;
 172   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
 173   for (unsigned I = 1, E = TRI->getNumSubRegIndices(); I < E; ++I)
 174     Names2SubRegIndices.insert(
 175         std::make_pair(TRI->getSubRegIndexName(I), I));
 176 }
 177
 178 unsigned PerTargetMIParsingState::getSubRegIndex(StringRef Name) {
 179   initNames2SubRegIndices();
 180   auto SubRegInfo = Names2SubRegIndices.find(Name);
 181   if (SubRegInfo == Names2SubRegIndices.end())
 182     return 0;
 183   return SubRegInfo->getValue();
 184 }
 185
 186 void PerTargetMIParsingState::initNames2TargetIndices() {
 187   if (!Names2TargetIndices.empty())
 188     return;
 189   const auto *TII = Subtarget.getInstrInfo();
 190   assert(TII && "Expected target instruction info");
 191   auto Indices = TII->getSerializableTargetIndices();
 192   for (const auto &I : Indices)
 193     Names2TargetIndices.insert(std::make_pair(StringRef(I.second), I.first));
 194 }
 195
 196 bool PerTargetMIParsingState::getTargetIndex(StringRef Name, int &Index) {
 197   initNames2TargetIndices();
 198   auto IndexInfo = Names2TargetIndices.find(Name);
 199   if (IndexInfo == Names2TargetIndices.end())
 200     return true;
 201   Index = IndexInfo->second;
 202   return false;
 203 }
 204
 205 void PerTargetMIParsingState::initNames2DirectTargetFlags() {
 206   if (!Names2DirectTargetFlags.empty())
 207     return;
 208
 209   const auto *TII = Subtarget.getInstrInfo();
 210   assert(TII && "Expected target instruction info");
 211   auto Flags = TII->getSerializableDirectMachineOperandTargetFlags();
 212   for (const auto &I : Flags)
 213     Names2DirectTargetFlags.insert(
 214         std::make_pair(StringRef(I.second), I.first));
 215 }
 216
 217 bool PerTargetMIParsingState::getDirectTargetFlag(StringRef Name,
 218                                                   unsigned &Flag) {
 219   initNames2DirectTargetFlags();
 220   auto FlagInfo = Names2DirectTargetFlags.find(Name);
 221   if (FlagInfo == Names2DirectTargetFlags.end())
 222     return true;
 223   Flag = FlagInfo->second;
 224   return false;
 225 }
 226
 227 void PerTargetMIParsingState::initNames2BitmaskTargetFlags() {
 228   if (!Names2BitmaskTargetFlags.empty())
 229     return;
 230
 231   const auto *TII = Subtarget.getInstrInfo();
 232   assert(TII && "Expected target instruction info");
 233   auto Flags = TII->getSerializableBitmaskMachineOperandTargetFlags();
 234   for (const auto &I : Flags)
 235     Names2BitmaskTargetFlags.insert(
 236         std::make_pair(StringRef(I.second), I.first));
 237 }
 238
 239 bool PerTargetMIParsingState::getBitmaskTargetFlag(StringRef Name,
 240                                                    unsigned &Flag) {
 241   initNames2BitmaskTargetFlags();
 242   auto FlagInfo = Names2BitmaskTargetFlags.find(Name);
 243   if (FlagInfo == Names2BitmaskTargetFlags.end())
 244     return true;
 245   Flag = FlagInfo->second;
 246   return false;
 247 }
 248
 249 void PerTargetMIParsingState::initNames2MMOTargetFlags() {
 250   if (!Names2MMOTargetFlags.empty())
 251     return;
 252
 253   const auto *TII = Subtarget.getInstrInfo();
 254   assert(TII && "Expected target instruction info");
 255   auto Flags = TII->getSerializableMachineMemOperandTargetFlags();
 256   for (const auto &I : Flags)
 257     Names2MMOTargetFlags.insert(std::make_pair(StringRef(I.second), I.first));
 258 }
 259
 260 bool PerTargetMIParsingState::getMMOTargetFlag(StringRef Name,
 261                                                MachineMemOperand::Flags &Flag) {
 262   initNames2MMOTargetFlags();
 263   auto FlagInfo = Names2MMOTargetFlags.find(Name);
 264   if (FlagInfo == Names2MMOTargetFlags.end())
 265     return true;
 266   Flag = FlagInfo->second;
 267   return false;
 268 }
 269
 270 void PerTargetMIParsingState::initNames2RegClasses() {
 271   if (!Names2RegClasses.empty())
 272     return;
 273
 274   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
 275   for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; ++I) {
 276     const auto *RC = TRI->getRegClass(I);
 277     Names2RegClasses.insert(
 278         std::make_pair(StringRef(TRI->getRegClassName(RC)).lower(), RC));
 279   }
 280 }
 281
 282 void PerTargetMIParsingState::initNames2RegBanks() {
 283   if (!Names2RegBanks.empty())
 284     return;
 285
 286   const RegisterBankInfo *RBI = Subtarget.getRegBankInfo();
 287   // If the target does not support GlobalISel, we may not have a
 288   // register bank info.
 289   if (!RBI)
 290     return;
 291
 292   for (unsigned I = 0, E = RBI->getNumRegBanks(); I < E; ++I) {
 293     const auto &RegBank = RBI->getRegBank(I);
 294     Names2RegBanks.insert(
 295         std::make_pair(StringRef(RegBank.getName()).lower(), &RegBank));
 296   }
 297 }
 298
 299 const TargetRegisterClass *
 300 PerTargetMIParsingState::getRegClass(StringRef Name) {
 301   auto RegClassInfo = Names2RegClasses.find(Name);
 302   if (RegClassInfo == Names2RegClasses.end())
 303     return nullptr;
 304   return RegClassInfo->getValue();
 305 }
 306
 307 const RegisterBank *PerTargetMIParsingState::getRegBank(StringRef Name) {
 308   auto RegBankInfo = Names2RegBanks.find(Name);
 309   if (RegBankInfo == Names2RegBanks.end())
 310     return nullptr;
 311   return RegBankInfo->getValue();
 312 }
 313
 314 PerFunctionMIParsingState::PerFunctionMIParsingState(MachineFunction &MF,
 315     SourceMgr &SM, const SlotMapping &IRSlots, PerTargetMIParsingState &T)
 316   : MF(MF), SM(&SM), IRSlots(IRSlots), Target(T) {
 317 }
 318
 319 VRegInfo &PerFunctionMIParsingState::getVRegInfo(Register Num) {
 320   auto I = VRegInfos.insert(std::make_pair(Num, nullptr));
 321   if (I.second) {
 322     MachineRegisterInfo &MRI = MF.getRegInfo();
 323     VRegInfo *Info = new (Allocator) VRegInfo;
 324     Info->VReg = MRI.createIncompleteVirtualRegister();
 325     I.first->second = Info;
 326   }
 327   return *I.first->second;
 328 }
 329
 330 VRegInfo &PerFunctionMIParsingState::getVRegInfoNamed(StringRef RegName) {
 331   assert(RegName != "" && "Expected named reg.");
 332
 333   auto I = VRegInfosNamed.insert(std::make_pair(RegName.str(), nullptr));
 334   if (I.second) {
 335     VRegInfo *Info = new (Allocator) VRegInfo;
 336     Info->VReg = MF.getRegInfo().createIncompleteVirtualRegister(RegName);
 337     I.first->second = Info;
 338   }
 339   return *I.first->second;
 340 }
 341
 342 static void mapValueToSlot(const Value *V, ModuleSlotTracker &MST,
 343                            DenseMap<unsigned, const Value *> &Slots2Values) {
 344   int Slot = MST.getLocalSlot(V);
 345   if (Slot == -1)
 346     return;
 347   Slots2Values.insert(std::make_pair(unsigned(Slot), V));
 348 }
 349
 350 /// Creates the mapping from slot numbers to function's unnamed IR values.
 351 static void initSlots2Values(const Function &F,
 352                              DenseMap<unsigned, const Value *> &Slots2Values) {
 353   ModuleSlotTracker MST(F.getParent(), /*ShouldInitializeAllMetadata=*/false);
 354   MST.incorporateFunction(F);
 355   for (const auto &Arg : F.args())
 356     mapValueToSlot(&Arg, MST, Slots2Values);
 357   for (const auto &BB : F) {
 358     mapValueToSlot(&BB, MST, Slots2Values);
 359     for (const auto &I : BB)
 360       mapValueToSlot(&I, MST, Slots2Values);
 361   }
 362 }
 363
 364 const Value* PerFunctionMIParsingState::getIRValue(unsigned Slot) {
 365   if (Slots2Values.empty())
 366     initSlots2Values(MF.getFunction(), Slots2Values);
 367   return Slots2Values.lookup(Slot);
 368 }
 369
 370 namespace {
 371
 372 /// A wrapper struct around the 'MachineOperand' struct that includes a source
 373 /// range and other attributes.
 374 struct ParsedMachineOperand {
 375   MachineOperand Operand;
 376   StringRef::iterator Begin;
 377   StringRef::iterator End;
 378   std::optional<unsigned> TiedDefIdx;
 379
 380   ParsedMachineOperand(const MachineOperand &Operand, StringRef::iterator Begin,
 381                        StringRef::iterator End,
 382                        std::optional<unsigned> &TiedDefIdx)
 383       : Operand(Operand), Begin(Begin), End(End), TiedDefIdx(TiedDefIdx) {
 384     if (TiedDefIdx)
 385       assert(Operand.isReg() && Operand.isUse() &&
 386              "Only used register operands can be tied");
 387   }
 388 };
 389
 390 class MIParser {
 391   MachineFunction &MF;
 392   SMDiagnostic &Error;
 393   StringRef Source, CurrentSource;
 394   SMRange SourceRange;
 395   MIToken Token;
 396   PerFunctionMIParsingState &PFS;
 397   /// Maps from slot numbers to function's unnamed basic blocks.
 398   DenseMap<unsigned, const BasicBlock *> Slots2BasicBlocks;
 399
 400 public:
 401   MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
 402            StringRef Source);
 403   MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
 404            StringRef Source, SMRange SourceRange);
 405
 406   /// \p SkipChar gives the number of characters to skip before looking
 407   /// for the next token.
 408   void lex(unsigned SkipChar = 0);
 409
 410   /// Report an error at the current location with the given message.
 411   ///
 412   /// This function always return true.
 413   bool error(const Twine &Msg);
 414
 415   /// Report an error at the given location with the given message.
 416   ///
 417   /// This function always return true.
 418   bool error(StringRef::iterator Loc, const Twine &Msg);
 419
 420   bool
 421   parseBasicBlockDefinitions(DenseMap<unsigned, MachineBasicBlock *> &MBBSlots);
 422   bool parseBasicBlocks();
 423   bool parse(MachineInstr *&MI);
 424   bool parseStandaloneMBB(MachineBasicBlock *&MBB);
 425   bool parseStandaloneNamedRegister(Register &Reg);
 426   bool parseStandaloneVirtualRegister(VRegInfo *&Info);
 427   bool parseStandaloneRegister(Register &Reg);
 428   bool parseStandaloneStackObject(int &FI);
 429   bool parseStandaloneMDNode(MDNode *&Node);
 430   bool parseMachineMetadata();
 431   bool parseMDTuple(MDNode *&MD, bool IsDistinct);
 432   bool parseMDNodeVector(SmallVectorImpl<Metadata *> &Elts);
 433   bool parseMetadata(Metadata *&MD);
 434
 435   bool
 436   parseBasicBlockDefinition(DenseMap<unsigned, MachineBasicBlock *> &MBBSlots);
 437   bool parseBasicBlock(MachineBasicBlock &MBB,
 438                        MachineBasicBlock *&AddFalthroughFrom);
 439   bool parseBasicBlockLiveins(MachineBasicBlock &MBB);
 440   bool parseBasicBlockSuccessors(MachineBasicBlock &MBB);
 441
 442   bool parseNamedRegister(Register &Reg);
 443   bool parseVirtualRegister(VRegInfo *&Info);
 444   bool parseNamedVirtualRegister(VRegInfo *&Info);
 445   bool parseRegister(Register &Reg, VRegInfo *&VRegInfo);
 446   bool parseRegisterFlag(unsigned &Flags);
 447   bool parseRegisterClassOrBank(VRegInfo &RegInfo);
 448   bool parseSubRegisterIndex(unsigned &SubReg);
 449   bool parseRegisterTiedDefIndex(unsigned &TiedDefIdx);
 450   bool parseRegisterOperand(MachineOperand &Dest,
 451                             std::optional<unsigned> &TiedDefIdx,
 452                             bool IsDef = false);
 453   bool parseImmediateOperand(MachineOperand &Dest);
 454   bool parseIRConstant(StringRef::iterator Loc, StringRef StringValue,
 455                        const Constant *&C);
 456   bool parseIRConstant(StringRef::iterator Loc, const Constant *&C);
 457   bool parseLowLevelType(StringRef::iterator Loc, LLT &Ty);
 458   bool parseTypedImmediateOperand(MachineOperand &Dest);
 459   bool parseFPImmediateOperand(MachineOperand &Dest);
 460   bool parseMBBReference(MachineBasicBlock *&MBB);
 461   bool parseMBBOperand(MachineOperand &Dest);
 462   bool parseStackFrameIndex(int &FI);
 463   bool parseStackObjectOperand(MachineOperand &Dest);
 464   bool parseFixedStackFrameIndex(int &FI);
 465   bool parseFixedStackObjectOperand(MachineOperand &Dest);
 466   bool parseGlobalValue(GlobalValue *&GV);
 467   bool parseGlobalAddressOperand(MachineOperand &Dest);
 468   bool parseConstantPoolIndexOperand(MachineOperand &Dest);
 469   bool parseSubRegisterIndexOperand(MachineOperand &Dest);
 470   bool parseJumpTableIndexOperand(MachineOperand &Dest);
 471   bool parseExternalSymbolOperand(MachineOperand &Dest);
 472   bool parseMCSymbolOperand(MachineOperand &Dest);
 473   [[nodiscard]] bool parseMDNode(MDNode *&Node);
 474   bool parseDIExpression(MDNode *&Expr);
 475   bool parseDILocation(MDNode *&Expr);
 476   bool parseMetadataOperand(MachineOperand &Dest);
 477   bool parseCFIOffset(int &Offset);
 478   bool parseCFIRegister(Register &Reg);
 479   bool parseCFIAddressSpace(unsigned &AddressSpace);
 480   bool parseCFIEscapeValues(std::string& Values);
 481   bool parseCFIOperand(MachineOperand &Dest);
 482   bool parseIRBlock(BasicBlock *&BB, const Function &F);
 483   bool parseBlockAddressOperand(MachineOperand &Dest);
 484   bool parseIntrinsicOperand(MachineOperand &Dest);
 485   bool parsePredicateOperand(MachineOperand &Dest);
 486   bool parseShuffleMaskOperand(MachineOperand &Dest);
 487   bool parseTargetIndexOperand(MachineOperand &Dest);
 488   bool parseDbgInstrRefOperand(MachineOperand &Dest);
 489   bool parseCustomRegisterMaskOperand(MachineOperand &Dest);
 490   bool parseLiveoutRegisterMaskOperand(MachineOperand &Dest);
 491   bool parseMachineOperand(const unsigned OpCode, const unsigned OpIdx,
 492                            MachineOperand &Dest,
 493                            std::optional<unsigned> &TiedDefIdx);
 494   bool parseMachineOperandAndTargetFlags(const unsigned OpCode,
 495                                          const unsigned OpIdx,
 496                                          MachineOperand &Dest,
 497                                          std::optional<unsigned> &TiedDefIdx);
 498   bool parseOffset(int64_t &Offset);
 499   bool parseIRBlockAddressTaken(BasicBlock *&BB);
 500   bool parseAlignment(uint64_t &Alignment);
 501   bool parseAddrspace(unsigned &Addrspace);
 502   bool parseSectionID(std::optional<MBBSectionID> &SID);
 503   bool parseBBID(std::optional<UniqueBBID> &BBID);
 504   bool parseCallFrameSize(unsigned &CallFrameSize);
 505   bool parseOperandsOffset(MachineOperand &Op);
 506   bool parseIRValue(const Value *&V);
 507   bool parseMemoryOperandFlag(MachineMemOperand::Flags &Flags);
 508   bool parseMemoryPseudoSourceValue(const PseudoSourceValue *&PSV);
 509   bool parseMachinePointerInfo(MachinePointerInfo &Dest);
 510   bool parseOptionalScope(LLVMContext &Context, SyncScope::ID &SSID);
 511   bool parseOptionalAtomicOrdering(AtomicOrdering &Order);
 512   bool parseMachineMemoryOperand(MachineMemOperand *&Dest);
 513   bool parsePreOrPostInstrSymbol(MCSymbol *&Symbol);
 514   bool parseHeapAllocMarker(MDNode *&Node);
 515   bool parsePCSections(MDNode *&Node);
 516
 517   bool parseTargetImmMnemonic(const unsigned OpCode, const unsigned OpIdx,
 518                               MachineOperand &Dest, const MIRFormatter &MF);
 519
 520 private:
 521   /// Convert the integer literal in the current token into an unsigned integer.
 522   ///
 523   /// Return true if an error occurred.
 524   bool getUnsigned(unsigned &Result);
 525
 526   /// Convert the integer literal in the current token into an uint64.
 527   ///
 528   /// Return true if an error occurred.
 529   bool getUint64(uint64_t &Result);
 530
 531   /// Convert the hexadecimal literal in the current token into an unsigned
 532   ///  APInt with a minimum bitwidth required to represent the value.
 533   ///
 534   /// Return true if the literal does not represent an integer value.
 535   bool getHexUint(APInt &Result);
 536
 537   /// If the current token is of the given kind, consume it and return false.
 538   /// Otherwise report an error and return true.
 539   bool expectAndConsume(MIToken::TokenKind TokenKind);
 540
 541   /// If the current token is of the given kind, consume it and return true.
 542   /// Otherwise return false.
 543   bool consumeIfPresent(MIToken::TokenKind TokenKind);
 544
 545   bool parseInstruction(unsigned &OpCode, unsigned &Flags);
 546
 547   bool assignRegisterTies(MachineInstr &MI,
 548                           ArrayRef<ParsedMachineOperand> Operands);
 549
 550   bool verifyImplicitOperands(ArrayRef<ParsedMachineOperand> Operands,
 551                               const MCInstrDesc &MCID);
 552
 553   const BasicBlock *getIRBlock(unsigned Slot);
 554   const BasicBlock *getIRBlock(unsigned Slot, const Function &F);
 555
 556   /// Get or create an MCSymbol for a given name.
 557   MCSymbol *getOrCreateMCSymbol(StringRef Name);
 558
 559   /// parseStringConstant
 560   ///   ::= StringConstant
 561   bool parseStringConstant(std::string &Result);
 562
 563   /// Map the location in the MI string to the corresponding location specified
 564   /// in `SourceRange`.
 565   SMLoc mapSMLoc(StringRef::iterator Loc);
 566 };
 567
 568 } // end anonymous namespace
 569
 570 MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
 571                    StringRef Source)
 572     : MF(PFS.MF), Error(Error), Source(Source), CurrentSource(Source), PFS(PFS)
 573 {}
 574
 575 MIParser::MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error,
 576                    StringRef Source, SMRange SourceRange)
 577     : MF(PFS.MF), Error(Error), Source(Source), CurrentSource(Source),
 578       SourceRange(SourceRange), PFS(PFS) {}
 579
 580 void MIParser::lex(unsigned SkipChar) {
 581   CurrentSource = lexMIToken(
 582       CurrentSource.slice(SkipChar, StringRef::npos), Token,
 583       [this](StringRef::iterator Loc, const Twine &Msg) { error(Loc, Msg); });
 584 }
 585
 586 bool MIParser::error(const Twine &Msg) { return error(Token.location(), Msg); }
 587
 588 bool MIParser::error(StringRef::iterator Loc, const Twine &Msg) {
 589   const SourceMgr &SM = *PFS.SM;
 590   assert(Loc >= Source.data() && Loc <= (Source.data() + Source.size()));
 591   const MemoryBuffer &Buffer = *SM.getMemoryBuffer(SM.getMainFileID());
 592   if (Loc >= Buffer.getBufferStart() && Loc <= Buffer.getBufferEnd()) {
 593     // Create an ordinary diagnostic when the source manager's buffer is the
 594     // source string.
 595     Error = SM.GetMessage(SMLoc::getFromPointer(Loc), SourceMgr::DK_Error, Msg);
 596     return true;
 597   }
 598   // Create a diagnostic for a YAML string literal.
 599   Error = SMDiagnostic(SM, SMLoc(), Buffer.getBufferIdentifier(), 1,
 600                        Loc - Source.data(), SourceMgr::DK_Error, Msg.str(),
 601                        Source, std::nullopt, std::nullopt);
 602   return true;
 603 }
 604
 605 SMLoc MIParser::mapSMLoc(StringRef::iterator Loc) {
 606   assert(SourceRange.isValid() && "Invalid source range");
 607   assert(Loc >= Source.data() && Loc <= (Source.data() + Source.size()));
 608   return SMLoc::getFromPointer(SourceRange.Start.getPointer() +
 609                                (Loc - Source.data()));
 610 }
 611
 612 typedef function_ref<bool(StringRef::iterator Loc, const Twine &)>
 613     ErrorCallbackType;
 614
 615 static const char *toString(MIToken::TokenKind TokenKind) {
 616   switch (TokenKind) {
 617   case MIToken::comma:
 618     return "','";
 619   case MIToken::equal:
 620     return "'='";
 621   case MIToken::colon:
 622     return "':'";
 623   case MIToken::lparen:
 624     return "'('";
 625   case MIToken::rparen:
 626     return "')'";
 627   default:
 628     return "<unknown token>";
 629   }
 630 }
 631
 632 bool MIParser::expectAndConsume(MIToken::TokenKind TokenKind) {
 633   if (Token.isNot(TokenKind))
 634     return error(Twine("expected ") + toString(TokenKind));
 635   lex();
 636   return false;
 637 }
 638
 639 bool MIParser::consumeIfPresent(MIToken::TokenKind TokenKind) {
 640   if (Token.isNot(TokenKind))
 641     return false;
 642   lex();
 643   return true;
 644 }
 645
 646 // Parse Machine Basic Block Section ID.
 647 bool MIParser::parseSectionID(std::optional<MBBSectionID> &SID) {
 648   assert(Token.is(MIToken::kw_bbsections));
 649   lex();
 650   if (Token.is(MIToken::IntegerLiteral)) {
 651     unsigned Value = 0;
 652     if (getUnsigned(Value))
 653       return error("Unknown Section ID");
 654     SID = MBBSectionID{Value};
 655   } else {
 656     const StringRef &S = Token.stringValue();
 657     if (S == "Exception")
 658       SID = MBBSectionID::ExceptionSectionID;
 659     else if (S == "Cold")
 660       SID = MBBSectionID::ColdSectionID;
 661     else
 662       return error("Unknown Section ID");
 663   }
 664   lex();
 665   return false;
 666 }
 667
 668 // Parse Machine Basic Block ID.
 669 bool MIParser::parseBBID(std::optional<UniqueBBID> &BBID) {
 670   assert(Token.is(MIToken::kw_bb_id));
 671   lex();
 672   unsigned BaseID = 0;
 673   unsigned CloneID = 0;
 674   if (getUnsigned(BaseID))
 675     return error("Unknown BB ID");
 676   lex();
 677   if (Token.is(MIToken::IntegerLiteral)) {
 678     if (getUnsigned(CloneID))
 679       return error("Unknown Clone ID");
 680     lex();
 681   }
 682   BBID = {BaseID, CloneID};
 683   return false;
 684 }
 685
 686 // Parse basic block call frame size.
 687 bool MIParser::parseCallFrameSize(unsigned &CallFrameSize) {
 688   assert(Token.is(MIToken::kw_call_frame_size));
 689   lex();
 690   unsigned Value = 0;
 691   if (getUnsigned(Value))
 692     return error("Unknown call frame size");
 693   CallFrameSize = Value;
 694   lex();
 695   return false;
 696 }
 697
 698 bool MIParser::parseBasicBlockDefinition(
 699     DenseMap<unsigned, MachineBasicBlock *> &MBBSlots) {
 700   assert(Token.is(MIToken::MachineBasicBlockLabel));
 701   unsigned ID = 0;
 702   if (getUnsigned(ID))
 703     return true;
 704   auto Loc = Token.location();
 705   auto Name = Token.stringValue();
 706   lex();
 707   bool MachineBlockAddressTaken = false;
 708   BasicBlock *AddressTakenIRBlock = nullptr;
 709   bool IsLandingPad = false;
 710   bool IsInlineAsmBrIndirectTarget = false;
 711   bool IsEHFuncletEntry = false;
 712   std::optional<MBBSectionID> SectionID;
 713   uint64_t Alignment = 0;
 714   std::optional<UniqueBBID> BBID;
 715   unsigned CallFrameSize = 0;
 716   BasicBlock *BB = nullptr;
 717   if (consumeIfPresent(MIToken::lparen)) {
 718     do {
 719       // TODO: Report an error when multiple same attributes are specified.
 720       switch (Token.kind()) {
 721       case MIToken::kw_machine_block_address_taken:
 722         MachineBlockAddressTaken = true;
 723         lex();
 724         break;
 725       case MIToken::kw_ir_block_address_taken:
 726         if (parseIRBlockAddressTaken(AddressTakenIRBlock))
 727           return true;
 728         break;
 729       case MIToken::kw_landing_pad:
 730         IsLandingPad = true;
 731         lex();
 732         break;
 733       case MIToken::kw_inlineasm_br_indirect_target:
 734         IsInlineAsmBrIndirectTarget = true;
 735         lex();
 736         break;
 737       case MIToken::kw_ehfunclet_entry:
 738         IsEHFuncletEntry = true;
 739         lex();
 740         break;
 741       case MIToken::kw_align:
 742         if (parseAlignment(Alignment))
 743           return true;
 744         break;
 745       case MIToken::IRBlock:
 746       case MIToken::NamedIRBlock:
 747         // TODO: Report an error when both name and ir block are specified.
 748         if (parseIRBlock(BB, MF.getFunction()))
 749           return true;
 750         lex();
 751         break;
 752       case MIToken::kw_bbsections:
 753         if (parseSectionID(SectionID))
 754           return true;
 755         break;
 756       case MIToken::kw_bb_id:
 757         if (parseBBID(BBID))
 758           return true;
 759         break;
 760       case MIToken::kw_call_frame_size:
 761         if (parseCallFrameSize(CallFrameSize))
 762           return true;
 763         break;
 764       default:
 765         break;
 766       }
 767     } while (consumeIfPresent(MIToken::comma));
 768     if (expectAndConsume(MIToken::rparen))
 769       return true;
 770   }
 771   if (expectAndConsume(MIToken::colon))
 772     return true;
 773
 774   if (!Name.empty()) {
 775     BB = dyn_cast_or_null<BasicBlock>(
 776         MF.getFunction().getValueSymbolTable()->lookup(Name));
 777     if (!BB)
 778       return error(Loc, Twine("basic block '") + Name +
 779                             "' is not defined in the function '" +
 780                             MF.getName() + "'");
 781   }
 782   auto *MBB = MF.CreateMachineBasicBlock(BB);
 783   MF.insert(MF.end(), MBB);
 784   bool WasInserted = MBBSlots.insert(std::make_pair(ID, MBB)).second;
 785   if (!WasInserted)
 786     return error(Loc, Twine("redefinition of machine basic block with id #") +
 787                           Twine(ID));
 788   if (Alignment)
 789     MBB->setAlignment(Align(Alignment));
 790   if (MachineBlockAddressTaken)
 791     MBB->setMachineBlockAddressTaken();
 792   if (AddressTakenIRBlock)
 793     MBB->setAddressTakenIRBlock(AddressTakenIRBlock);
 794   MBB->setIsEHPad(IsLandingPad);
 795   MBB->setIsInlineAsmBrIndirectTarget(IsInlineAsmBrIndirectTarget);
 796   MBB->setIsEHFuncletEntry(IsEHFuncletEntry);
 797   if (SectionID) {
 798     MBB->setSectionID(*SectionID);
 799     MF.setBBSectionsType(BasicBlockSection::List);
 800   }
 801   if (BBID.has_value()) {
 802     // BBSectionsType is set to `List` if any basic blocks has `SectionID`.
 803     // Here, we set it to `Labels` if it hasn't been set above.
 804     if (!MF.hasBBSections())
 805       MF.setBBSectionsType(BasicBlockSection::Labels);
 806     MBB->setBBID(BBID.value());
 807   }
 808   MBB->setCallFrameSize(CallFrameSize);
 809   return false;
 810 }
 811
 812 bool MIParser::parseBasicBlockDefinitions(
 813     DenseMap<unsigned, MachineBasicBlock *> &MBBSlots) {
 814   lex();
 815   // Skip until the first machine basic block.
 816   while (Token.is(MIToken::Newline))
 817     lex();
 818   if (Token.isErrorOrEOF())
 819     return Token.isError();
 820   if (Token.isNot(MIToken::MachineBasicBlockLabel))
 821     return error("expected a basic block definition before instructions");
 822   unsigned BraceDepth = 0;
 823   do {
 824     if (parseBasicBlockDefinition(MBBSlots))
 825       return true;
 826     bool IsAfterNewline = false;
 827     // Skip until the next machine basic block.
 828     while (true) {
 829       if ((Token.is(MIToken::MachineBasicBlockLabel) && IsAfterNewline) ||
 830           Token.isErrorOrEOF())
 831         break;
 832       else if (Token.is(MIToken::MachineBasicBlockLabel))
 833         return error("basic block definition should be located at the start of "
 834                      "the line");
 835       else if (consumeIfPresent(MIToken::Newline)) {
 836         IsAfterNewline = true;
 837         continue;
 838       }
 839       IsAfterNewline = false;
 840       if (Token.is(MIToken::lbrace))
 841         ++BraceDepth;
 842       if (Token.is(MIToken::rbrace)) {
 843         if (!BraceDepth)
 844           return error("extraneous closing brace ('}')");
 845         --BraceDepth;
 846       }
 847       lex();
 848     }
 849     // Verify that we closed all of the '{' at the end of a file or a block.
 850     if (!Token.isError() && BraceDepth)
 851       return error("expected '}'"); // FIXME: Report a note that shows '{'.
 852   } while (!Token.isErrorOrEOF());
 853   return Token.isError();
 854 }
 855
 856 bool MIParser::parseBasicBlockLiveins(MachineBasicBlock &MBB) {
 857   assert(Token.is(MIToken::kw_liveins));
 858   lex();
 859   if (expectAndConsume(MIToken::colon))
 860     return true;
 861   if (Token.isNewlineOrEOF()) // Allow an empty list of liveins.
 862     return false;
 863   do {
 864     if (Token.isNot(MIToken::NamedRegister))
 865       return error("expected a named register");
 866     Register Reg;
 867     if (parseNamedRegister(Reg))
 868       return true;
 869     lex();
 870     LaneBitmask Mask = LaneBitmask::getAll();
 871     if (consumeIfPresent(MIToken::colon)) {
 872       // Parse lane mask.
 873       if (Token.isNot(MIToken::IntegerLiteral) &&
 874           Token.isNot(MIToken::HexLiteral))
 875         return error("expected a lane mask");
 876       static_assert(sizeof(LaneBitmask::Type) == sizeof(uint64_t),
 877                     "Use correct get-function for lane mask");
 878       LaneBitmask::Type V;
 879       if (getUint64(V))
 880         return error("invalid lane mask value");
 881       Mask = LaneBitmask(V);
 882       lex();
 883     }
 884     MBB.addLiveIn(Reg, Mask);
 885   } while (consumeIfPresent(MIToken::comma));
 886   return false;
 887 }
 888
 889 bool MIParser::parseBasicBlockSuccessors(MachineBasicBlock &MBB) {
 890   assert(Token.is(MIToken::kw_successors));
 891   lex();
 892   if (expectAndConsume(MIToken::colon))
 893     return true;
 894   if (Token.isNewlineOrEOF()) // Allow an empty list of successors.
 895     return false;
 896   do {
 897     if (Token.isNot(MIToken::MachineBasicBlock))
 898       return error("expected a machine basic block reference");
 899     MachineBasicBlock *SuccMBB = nullptr;
 900     if (parseMBBReference(SuccMBB))
 901       return true;
 902     lex();
 903     unsigned Weight = 0;
 904     if (consumeIfPresent(MIToken::lparen)) {
 905       if (Token.isNot(MIToken::IntegerLiteral) &&
 906           Token.isNot(MIToken::HexLiteral))
 907         return error("expected an integer literal after '('");
 908       if (getUnsigned(Weight))
 909         return true;
 910       lex();
 911       if (expectAndConsume(MIToken::rparen))
 912         return true;
 913     }
 914     MBB.addSuccessor(SuccMBB, BranchProbability::getRaw(Weight));
 915   } while (consumeIfPresent(MIToken::comma));
 916   MBB.normalizeSuccProbs();
 917   return false;
 918 }
 919
 920 bool MIParser::parseBasicBlock(MachineBasicBlock &MBB,
 921                                MachineBasicBlock *&AddFalthroughFrom) {
 922   // Skip the definition.
 923   assert(Token.is(MIToken::MachineBasicBlockLabel));
 924   lex();
 925   if (consumeIfPresent(MIToken::lparen)) {
 926     while (Token.isNot(MIToken::rparen) && !Token.isErrorOrEOF())
 927       lex();
 928     consumeIfPresent(MIToken::rparen);
 929   }
 930   consumeIfPresent(MIToken::colon);
 931
 932   // Parse the liveins and successors.
 933   // N.B: Multiple lists of successors and liveins are allowed and they're
 934   // merged into one.
 935   // Example:
 936   //   liveins: $edi
 937   //   liveins: $esi
 938   //
 939   // is equivalent to
 940   //   liveins: $edi, $esi
 941   bool ExplicitSuccessors = false;
 942   while (true) {
 943     if (Token.is(MIToken::kw_successors)) {
 944       if (parseBasicBlockSuccessors(MBB))
 945         return true;
 946       ExplicitSuccessors = true;
 947     } else if (Token.is(MIToken::kw_liveins)) {
 948       if (parseBasicBlockLiveins(MBB))
 949         return true;
 950     } else if (consumeIfPresent(MIToken::Newline)) {
 951       continue;
 952     } else
 953       break;
 954     if (!Token.isNewlineOrEOF())
 955       return error("expected line break at the end of a list");
 956     lex();
 957   }
 958
 959   // Parse the instructions.
 960   bool IsInBundle = false;
 961   MachineInstr *PrevMI = nullptr;
 962   while (!Token.is(MIToken::MachineBasicBlockLabel) &&
 963          !Token.is(MIToken::Eof)) {
 964     if (consumeIfPresent(MIToken::Newline))
 965       continue;
 966     if (consumeIfPresent(MIToken::rbrace)) {
 967       // The first parsing pass should verify that all closing '}' have an
 968       // opening '{'.
 969       assert(IsInBundle);
 970       IsInBundle = false;
 971       continue;
 972     }
 973     MachineInstr *MI = nullptr;
 974     if (parse(MI))
 975       return true;
 976     MBB.insert(MBB.end(), MI);
 977     if (IsInBundle) {
 978       PrevMI->setFlag(MachineInstr::BundledSucc);
 979       MI->setFlag(MachineInstr::BundledPred);
 980     }
 981     PrevMI = MI;
 982     if (Token.is(MIToken::lbrace)) {
 983       if (IsInBundle)
 984         return error("nested instruction bundles are not allowed");
 985       lex();
 986       // This instruction is the start of the bundle.
 987       MI->setFlag(MachineInstr::BundledSucc);
 988       IsInBundle = true;
 989       if (!Token.is(MIToken::Newline))
 990         // The next instruction can be on the same line.
 991         continue;
 992     }
 993     assert(Token.isNewlineOrEOF() && "MI is not fully parsed");
 994     lex();
 995   }
 996
 997   // Construct successor list by searching for basic block machine operands.
 998   if (!ExplicitSuccessors) {
 999     SmallVector<MachineBasicBlock*,4> Successors;
1000     bool IsFallthrough;
1001     guessSuccessors(MBB, Successors, IsFallthrough);
1002     for (MachineBasicBlock *Succ : Successors)
1003       MBB.addSuccessor(Succ);
1004
1005     if (IsFallthrough) {
1006       AddFalthroughFrom = &MBB;
1007     } else {
1008       MBB.normalizeSuccProbs();
1009     }
1010   }
1011
1012   return false;
1013 }
1014
1015 bool MIParser::parseBasicBlocks() {
1016   lex();
1017   // Skip until the first machine basic block.
1018   while (Token.is(MIToken::Newline))
1019     lex();
1020   if (Token.isErrorOrEOF())
1021     return Token.isError();
1022   // The first parsing pass should have verified that this token is a MBB label
1023   // in the 'parseBasicBlockDefinitions' method.
1024   assert(Token.is(MIToken::MachineBasicBlockLabel));
1025   MachineBasicBlock *AddFalthroughFrom = nullptr;
1026   do {
1027     MachineBasicBlock *MBB = nullptr;
1028     if (parseMBBReference(MBB))
1029       return true;
1030     if (AddFalthroughFrom) {
1031       if (!AddFalthroughFrom->isSuccessor(MBB))
1032         AddFalthroughFrom->addSuccessor(MBB);
1033       AddFalthroughFrom->normalizeSuccProbs();
1034       AddFalthroughFrom = nullptr;
1035     }
1036     if (parseBasicBlock(*MBB, AddFalthroughFrom))
1037       return true;
1038     // The method 'parseBasicBlock' should parse the whole block until the next
1039     // block or the end of file.
1040     assert(Token.is(MIToken::MachineBasicBlockLabel) || Token.is(MIToken::Eof));
1041   } while (Token.isNot(MIToken::Eof));
1042   return false;
1043 }
1044
1045 bool MIParser::parse(MachineInstr *&MI) {
1046   // Parse any register operands before '='
1047   MachineOperand MO = MachineOperand::CreateImm(0);
1048   SmallVector<ParsedMachineOperand, 8> Operands;
1049   while (Token.isRegister() || Token.isRegisterFlag()) {
1050     auto Loc = Token.location();
1051     std::optional<unsigned> TiedDefIdx;
1052     if (parseRegisterOperand(MO, TiedDefIdx, /*IsDef=*/true))
1053       return true;
1054     Operands.push_back(
1055         ParsedMachineOperand(MO, Loc, Token.location(), TiedDefIdx));
1056     if (Token.isNot(MIToken::comma))
1057       break;
1058     lex();
1059   }
1060   if (!Operands.empty() && expectAndConsume(MIToken::equal))
1061     return true;
1062
1063   unsigned OpCode, Flags = 0;
1064   if (Token.isError() || parseInstruction(OpCode, Flags))
1065     return true;
1066
1067   // Parse the remaining machine operands.
1068   while (!Token.isNewlineOrEOF() && Token.isNot(MIToken::kw_pre_instr_symbol) &&
1069          Token.isNot(MIToken::kw_post_instr_symbol) &&
1070          Token.isNot(MIToken::kw_heap_alloc_marker) &&
1071          Token.isNot(MIToken::kw_pcsections) &&
1072          Token.isNot(MIToken::kw_cfi_type) &&
1073          Token.isNot(MIToken::kw_debug_location) &&
1074          Token.isNot(MIToken::kw_debug_instr_number) &&
1075          Token.isNot(MIToken::coloncolon) && Token.isNot(MIToken::lbrace)) {
1076     auto Loc = Token.location();
1077     std::optional<unsigned> TiedDefIdx;
1078     if (parseMachineOperandAndTargetFlags(OpCode, Operands.size(), MO, TiedDefIdx))
1079       return true;
1080     Operands.push_back(
1081         ParsedMachineOperand(MO, Loc, Token.location(), TiedDefIdx));
1082     if (Token.isNewlineOrEOF() || Token.is(MIToken::coloncolon) ||
1083         Token.is(MIToken::lbrace))
1084       break;
1085     if (Token.isNot(MIToken::comma))
1086       return error("expected ',' before the next machine operand");
1087     lex();
1088   }
1089
1090   MCSymbol *PreInstrSymbol = nullptr;
1091   if (Token.is(MIToken::kw_pre_instr_symbol))
1092     if (parsePreOrPostInstrSymbol(PreInstrSymbol))
1093       return true;
1094   MCSymbol *PostInstrSymbol = nullptr;
1095   if (Token.is(MIToken::kw_post_instr_symbol))
1096     if (parsePreOrPostInstrSymbol(PostInstrSymbol))
1097       return true;
1098   MDNode *HeapAllocMarker = nullptr;
1099   if (Token.is(MIToken::kw_heap_alloc_marker))
1100     if (parseHeapAllocMarker(HeapAllocMarker))
1101       return true;
1102   MDNode *PCSections = nullptr;
1103   if (Token.is(MIToken::kw_pcsections))
1104     if (parsePCSections(PCSections))
1105       return true;
1106
1107   unsigned CFIType = 0;
1108   if (Token.is(MIToken::kw_cfi_type)) {
1109     lex();
1110     if (Token.isNot(MIToken::IntegerLiteral))
1111       return error("expected an integer literal after 'cfi-type'");
1112     // getUnsigned is sufficient for 32-bit integers.
1113     if (getUnsigned(CFIType))
1114       return true;
1115     lex();
1116     // Lex past trailing comma if present.
1117     if (Token.is(MIToken::comma))
1118       lex();
1119   }
1120
1121   unsigned InstrNum = 0;
1122   if (Token.is(MIToken::kw_debug_instr_number)) {
1123     lex();
1124     if (Token.isNot(MIToken::IntegerLiteral))
1125       return error("expected an integer literal after 'debug-instr-number'");
1126     if (getUnsigned(InstrNum))
1127       return true;
1128     lex();
1129     // Lex past trailing comma if present.
1130     if (Token.is(MIToken::comma))
1131       lex();
1132   }
1133
1134   DebugLoc DebugLocation;
1135   if (Token.is(MIToken::kw_debug_location)) {
1136     lex();
1137     MDNode *Node = nullptr;
1138     if (Token.is(MIToken::exclaim)) {
1139       if (parseMDNode(Node))
1140         return true;
1141     } else if (Token.is(MIToken::md_dilocation)) {
1142       if (parseDILocation(Node))
1143         return true;
1144     } else
1145       return error("expected a metadata node after 'debug-location'");
1146     if (!isa<DILocation>(Node))
1147       return error("referenced metadata is not a DILocation");
1148     DebugLocation = DebugLoc(Node);
1149   }
1150
1151   // Parse the machine memory operands.
1152   SmallVector<MachineMemOperand *, 2> MemOperands;
1153   if (Token.is(MIToken::coloncolon)) {
1154     lex();
1155     while (!Token.isNewlineOrEOF()) {
1156       MachineMemOperand *MemOp = nullptr;
1157       if (parseMachineMemoryOperand(MemOp))
1158         return true;
1159       MemOperands.push_back(MemOp);
1160       if (Token.isNewlineOrEOF())
1161         break;
1162       if (Token.isNot(MIToken::comma))
1163         return error("expected ',' before the next machine memory operand");
1164       lex();
1165     }
1166   }
1167
1168   const auto &MCID = MF.getSubtarget().getInstrInfo()->get(OpCode);
1169   if (!MCID.isVariadic()) {
1170     // FIXME: Move the implicit operand verification to the machine verifier.
1171     if (verifyImplicitOperands(Operands, MCID))
1172       return true;
1173   }
1174
1175   MI = MF.CreateMachineInstr(MCID, DebugLocation, /*NoImplicit=*/true);
1176   MI->setFlags(Flags);
1177
1178   unsigned NumExplicitOps = 0;
1179   for (const auto &Operand : Operands) {
1180     bool IsImplicitOp = Operand.Operand.isReg() && Operand.Operand.isImplicit();
1181     if (!IsImplicitOp) {
1182       if (!MCID.isVariadic() && NumExplicitOps >= MCID.getNumOperands() &&
1183           !Operand.Operand.isValidExcessOperand())
1184         return error(Operand.Begin, "too many operands for instruction");
1185
1186       ++NumExplicitOps;
1187     }
1188
1189     MI->addOperand(MF, Operand.Operand);
1190   }
1191
1192   if (assignRegisterTies(*MI, Operands))
1193     return true;
1194   if (PreInstrSymbol)
1195     MI->setPreInstrSymbol(MF, PreInstrSymbol);
1196   if (PostInstrSymbol)
1197     MI->setPostInstrSymbol(MF, PostInstrSymbol);
1198   if (HeapAllocMarker)
1199     MI->setHeapAllocMarker(MF, HeapAllocMarker);
1200   if (PCSections)
1201     MI->setPCSections(MF, PCSections);
1202   if (CFIType)
1203     MI->setCFIType(MF, CFIType);
1204   if (!MemOperands.empty())
1205     MI->setMemRefs(MF, MemOperands);
1206   if (InstrNum)
1207     MI->setDebugInstrNum(InstrNum);
1208   return false;
1209 }
1210
1211 bool MIParser::parseStandaloneMBB(MachineBasicBlock *&MBB) {
1212   lex();
1213   if (Token.isNot(MIToken::MachineBasicBlock))
1214     return error("expected a machine basic block reference");
1215   if (parseMBBReference(MBB))
1216     return true;
1217   lex();
1218   if (Token.isNot(MIToken::Eof))
1219     return error(
1220         "expected end of string after the machine basic block reference");
1221   return false;
1222 }
1223
1224 bool MIParser::parseStandaloneNamedRegister(Register &Reg) {
1225   lex();
1226   if (Token.isNot(MIToken::NamedRegister))
1227     return error("expected a named register");
1228   if (parseNamedRegister(Reg))
1229     return true;
1230   lex();
1231   if (Token.isNot(MIToken::Eof))
1232     return error("expected end of string after the register reference");
1233   return false;
1234 }
1235
1236 bool MIParser::parseStandaloneVirtualRegister(VRegInfo *&Info) {
1237   lex();
1238   if (Token.isNot(MIToken::VirtualRegister))
1239     return error("expected a virtual register");
1240   if (parseVirtualRegister(Info))
1241     return true;
1242   lex();
1243   if (Token.isNot(MIToken::Eof))
1244     return error("expected end of string after the register reference");
1245   return false;
1246 }
1247
1248 bool MIParser::parseStandaloneRegister(Register &Reg) {
1249   lex();
1250   if (Token.isNot(MIToken::NamedRegister) &&
1251       Token.isNot(MIToken::VirtualRegister))
1252     return error("expected either a named or virtual register");
1253
1254   VRegInfo *Info;
1255   if (parseRegister(Reg, Info))
1256     return true;
1257
1258   lex();
1259   if (Token.isNot(MIToken::Eof))
1260     return error("expected end of string after the register reference");
1261   return false;
1262 }
1263
1264 bool MIParser::parseStandaloneStackObject(int &FI) {
1265   lex();
1266   if (Token.isNot(MIToken::StackObject))
1267     return error("expected a stack object");
1268   if (parseStackFrameIndex(FI))
1269     return true;
1270   if (Token.isNot(MIToken::Eof))
1271     return error("expected end of string after the stack object reference");
1272   return false;
1273 }
1274
1275 bool MIParser::parseStandaloneMDNode(MDNode *&Node) {
1276   lex();
1277   if (Token.is(MIToken::exclaim)) {
1278     if (parseMDNode(Node))
1279       return true;
1280   } else if (Token.is(MIToken::md_diexpr)) {
1281     if (parseDIExpression(Node))
1282       return true;
1283   } else if (Token.is(MIToken::md_dilocation)) {
1284     if (parseDILocation(Node))
1285       return true;
1286   } else
1287     return error("expected a metadata node");
1288   if (Token.isNot(MIToken::Eof))
1289     return error("expected end of string after the metadata node");
1290   return false;
1291 }
1292
1293 bool MIParser::parseMachineMetadata() {
1294   lex();
1295   if (Token.isNot(MIToken::exclaim))
1296     return error("expected a metadata node");
1297
1298   lex();
1299   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
1300     return error("expected metadata id after '!'");
1301   unsigned ID = 0;
1302   if (getUnsigned(ID))
1303     return true;
1304   lex();
1305   if (expectAndConsume(MIToken::equal))
1306     return true;
1307   bool IsDistinct = Token.is(MIToken::kw_distinct);
1308   if (IsDistinct)
1309     lex();
1310   if (Token.isNot(MIToken::exclaim))
1311     return error("expected a metadata node");
1312   lex();
1313
1314   MDNode *MD;
1315   if (parseMDTuple(MD, IsDistinct))
1316     return true;
1317
1318   auto FI = PFS.MachineForwardRefMDNodes.find(ID);
1319   if (FI != PFS.MachineForwardRefMDNodes.end()) {
1320     FI->second.first->replaceAllUsesWith(MD);
1321     PFS.MachineForwardRefMDNodes.erase(FI);
1322
1323     assert(PFS.MachineMetadataNodes[ID] == MD && "Tracking VH didn't work");
1324   } else {
1325     if (PFS.MachineMetadataNodes.count(ID))
1326       return error("Metadata id is already used");
1327     PFS.MachineMetadataNodes[ID].reset(MD);
1328   }
1329
1330   return false;
1331 }
1332
1333 bool MIParser::parseMDTuple(MDNode *&MD, bool IsDistinct) {
1334   SmallVector<Metadata *, 16> Elts;
1335   if (parseMDNodeVector(Elts))
1336     return true;
1337   MD = (IsDistinct ? MDTuple::getDistinct
1338                    : MDTuple::get)(MF.getFunction().getContext(), Elts);
1339   return false;
1340 }
1341
1342 bool MIParser::parseMDNodeVector(SmallVectorImpl<Metadata *> &Elts) {
1343   if (Token.isNot(MIToken::lbrace))
1344     return error("expected '{' here");
1345   lex();
1346
1347   if (Token.is(MIToken::rbrace)) {
1348     lex();
1349     return false;
1350   }
1351
1352   do {
1353     Metadata *MD;
1354     if (parseMetadata(MD))
1355       return true;
1356
1357     Elts.push_back(MD);
1358
1359     if (Token.isNot(MIToken::comma))
1360       break;
1361     lex();
1362   } while (true);
1363
1364   if (Token.isNot(MIToken::rbrace))
1365     return error("expected end of metadata node");
1366   lex();
1367
1368   return false;
1369 }
1370
1371 // ::= !42
1372 // ::= !"string"
1373 bool MIParser::parseMetadata(Metadata *&MD) {
1374   if (Token.isNot(MIToken::exclaim))
1375     return error("expected '!' here");
1376   lex();
1377
1378   if (Token.is(MIToken::StringConstant)) {
1379     std::string Str;
1380     if (parseStringConstant(Str))
1381       return true;
1382     MD = MDString::get(MF.getFunction().getContext(), Str);
1383     return false;
1384   }
1385
1386   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
1387     return error("expected metadata id after '!'");
1388
1389   SMLoc Loc = mapSMLoc(Token.location());
1390
1391   unsigned ID = 0;
1392   if (getUnsigned(ID))
1393     return true;
1394   lex();
1395
1396   auto NodeInfo = PFS.IRSlots.MetadataNodes.find(ID);
1397   if (NodeInfo != PFS.IRSlots.MetadataNodes.end()) {
1398     MD = NodeInfo->second.get();
1399     return false;
1400   }
1401   // Check machine metadata.
1402   NodeInfo = PFS.MachineMetadataNodes.find(ID);
1403   if (NodeInfo != PFS.MachineMetadataNodes.end()) {
1404     MD = NodeInfo->second.get();
1405     return false;
1406   }
1407   // Forward reference.
1408   auto &FwdRef = PFS.MachineForwardRefMDNodes[ID];
1409   FwdRef = std::make_pair(
1410       MDTuple::getTemporary(MF.getFunction().getContext(), std::nullopt), Loc);
1411   PFS.MachineMetadataNodes[ID].reset(FwdRef.first.get());
1412   MD = FwdRef.first.get();
1413
1414   return false;
1415 }
1416
1417 static const char *printImplicitRegisterFlag(const MachineOperand &MO) {
1418   assert(MO.isImplicit());
1419   return MO.isDef() ? "implicit-def" : "implicit";
1420 }
1421
1422 static std::string getRegisterName(const TargetRegisterInfo *TRI,
1423                                    Register Reg) {
1424   assert(Reg.isPhysical() && "expected phys reg");
1425   return StringRef(TRI->getName(Reg)).lower();
1426 }
1427
1428 /// Return true if the parsed machine operands contain a given machine operand.
1429 static bool isImplicitOperandIn(const MachineOperand &ImplicitOperand,
1430                                 ArrayRef<ParsedMachineOperand> Operands) {
1431   for (const auto &I : Operands) {
1432     if (ImplicitOperand.isIdenticalTo(I.Operand))
1433       return true;
1434   }
1435   return false;
1436 }
1437
1438 bool MIParser::verifyImplicitOperands(ArrayRef<ParsedMachineOperand> Operands,
1439                                       const MCInstrDesc &MCID) {
1440   if (MCID.isCall())
1441     // We can't verify call instructions as they can contain arbitrary implicit
1442     // register and register mask operands.
1443     return false;
1444
1445   // Gather all the expected implicit operands.
1446   SmallVector<MachineOperand, 4> ImplicitOperands;
1447   for (MCPhysReg ImpDef : MCID.implicit_defs())
1448     ImplicitOperands.push_back(MachineOperand::CreateReg(ImpDef, true, true));
1449   for (MCPhysReg ImpUse : MCID.implicit_uses())
1450     ImplicitOperands.push_back(MachineOperand::CreateReg(ImpUse, false, true));
1451
1452   const auto *TRI = MF.getSubtarget().getRegisterInfo();
1453   assert(TRI && "Expected target register info");
1454   for (const auto &I : ImplicitOperands) {
1455     if (isImplicitOperandIn(I, Operands))
1456       continue;
1457     return error(Operands.empty() ? Token.location() : Operands.back().End,
1458                  Twine("missing implicit register operand '") +
1459                      printImplicitRegisterFlag(I) + " $" +
1460                      getRegisterName(TRI, I.getReg()) + "'");
1461   }
1462   return false;
1463 }
1464
1465 bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) {
1466   // Allow frame and fast math flags for OPCODE
1467   // clang-format off
1468   while (Token.is(MIToken::kw_frame_setup) ||
1469          Token.is(MIToken::kw_frame_destroy) ||
1470          Token.is(MIToken::kw_nnan) ||
1471          Token.is(MIToken::kw_ninf) ||
1472          Token.is(MIToken::kw_nsz) ||
1473          Token.is(MIToken::kw_arcp) ||
1474          Token.is(MIToken::kw_contract) ||
1475          Token.is(MIToken::kw_afn) ||
1476          Token.is(MIToken::kw_reassoc) ||
1477          Token.is(MIToken::kw_nuw) ||
1478          Token.is(MIToken::kw_nsw) ||
1479          Token.is(MIToken::kw_exact) ||
1480          Token.is(MIToken::kw_nofpexcept) ||
1481          Token.is(MIToken::kw_noconvergent) ||
1482          Token.is(MIToken::kw_unpredictable)) {
1483     // clang-format on
1484     // Mine frame and fast math flags
1485     if (Token.is(MIToken::kw_frame_setup))
1486       Flags |= MachineInstr::FrameSetup;
1487     if (Token.is(MIToken::kw_frame_destroy))
1488       Flags |= MachineInstr::FrameDestroy;
1489     if (Token.is(MIToken::kw_nnan))
1490       Flags |= MachineInstr::FmNoNans;
1491     if (Token.is(MIToken::kw_ninf))
1492       Flags |= MachineInstr::FmNoInfs;
1493     if (Token.is(MIToken::kw_nsz))
1494       Flags |= MachineInstr::FmNsz;
1495     if (Token.is(MIToken::kw_arcp))
1496       Flags |= MachineInstr::FmArcp;
1497     if (Token.is(MIToken::kw_contract))
1498       Flags |= MachineInstr::FmContract;
1499     if (Token.is(MIToken::kw_afn))
1500       Flags |= MachineInstr::FmAfn;
1501     if (Token.is(MIToken::kw_reassoc))
1502       Flags |= MachineInstr::FmReassoc;
1503     if (Token.is(MIToken::kw_nuw))
1504       Flags |= MachineInstr::NoUWrap;
1505     if (Token.is(MIToken::kw_nsw))
1506       Flags |= MachineInstr::NoSWrap;
1507     if (Token.is(MIToken::kw_exact))
1508       Flags |= MachineInstr::IsExact;
1509     if (Token.is(MIToken::kw_nofpexcept))
1510       Flags |= MachineInstr::NoFPExcept;
1511     if (Token.is(MIToken::kw_unpredictable))
1512       Flags |= MachineInstr::Unpredictable;
1513     if (Token.is(MIToken::kw_noconvergent))
1514       Flags |= MachineInstr::NoConvergent;
1515
1516     lex();
1517   }
1518   if (Token.isNot(MIToken::Identifier))
1519     return error("expected a machine instruction");
1520   StringRef InstrName = Token.stringValue();
1521   if (PFS.Target.parseInstrName(InstrName, OpCode))
1522     return error(Twine("unknown machine instruction name '") + InstrName + "'");
1523   lex();
1524   return false;
1525 }
1526
1527 bool MIParser::parseNamedRegister(Register &Reg) {
1528   assert(Token.is(MIToken::NamedRegister) && "Needs NamedRegister token");
1529   StringRef Name = Token.stringValue();
1530   if (PFS.Target.getRegisterByName(Name, Reg))
1531     return error(Twine("unknown register name '") + Name + "'");
1532   return false;
1533 }
1534
1535 bool MIParser::parseNamedVirtualRegister(VRegInfo *&Info) {
1536   assert(Token.is(MIToken::NamedVirtualRegister) && "Expected NamedVReg token");
1537   StringRef Name = Token.stringValue();
1538   // TODO: Check that the VReg name is not the same as a physical register name.
1539   //       If it is, then print a warning (when warnings are implemented).
1540   Info = &PFS.getVRegInfoNamed(Name);
1541   return false;
1542 }
1543
1544 bool MIParser::parseVirtualRegister(VRegInfo *&Info) {
1545   if (Token.is(MIToken::NamedVirtualRegister))
1546     return parseNamedVirtualRegister(Info);
1547   assert(Token.is(MIToken::VirtualRegister) && "Needs VirtualRegister token");
1548   unsigned ID;
1549   if (getUnsigned(ID))
1550     return true;
1551   Info = &PFS.getVRegInfo(ID);
1552   return false;
1553 }
1554
1555 bool MIParser::parseRegister(Register &Reg, VRegInfo *&Info) {
1556   switch (Token.kind()) {
1557   case MIToken::underscore:
1558     Reg = 0;
1559     return false;
1560   case MIToken::NamedRegister:
1561     return parseNamedRegister(Reg);
1562   case MIToken::NamedVirtualRegister:
1563   case MIToken::VirtualRegister:
1564     if (parseVirtualRegister(Info))
1565       return true;
1566     Reg = Info->VReg;
1567     return false;
1568   // TODO: Parse other register kinds.
1569   default:
1570     llvm_unreachable("The current token should be a register");
1571   }
1572 }
1573
1574 bool MIParser::parseRegisterClassOrBank(VRegInfo &RegInfo) {
1575   if (Token.isNot(MIToken::Identifier) && Token.isNot(MIToken::underscore))
1576     return error("expected '_', register class, or register bank name");
1577   StringRef::iterator Loc = Token.location();
1578   StringRef Name = Token.stringValue();
1579
1580   // Was it a register class?
1581   const TargetRegisterClass *RC = PFS.Target.getRegClass(Name);
1582   if (RC) {
1583     lex();
1584
1585     switch (RegInfo.Kind) {
1586     case VRegInfo::UNKNOWN:
1587     case VRegInfo::NORMAL:
1588       RegInfo.Kind = VRegInfo::NORMAL;
1589       if (RegInfo.Explicit && RegInfo.D.RC != RC) {
1590         const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
1591         return error(Loc, Twine("conflicting register classes, previously: ") +
1592                      Twine(TRI.getRegClassName(RegInfo.D.RC)));
1593       }
1594       RegInfo.D.RC = RC;
1595       RegInfo.Explicit = true;
1596       return false;
1597
1598     case VRegInfo::GENERIC:
1599     case VRegInfo::REGBANK:
1600       return error(Loc, "register class specification on generic register");
1601     }
1602     llvm_unreachable("Unexpected register kind");
1603   }
1604
1605   // Should be a register bank or a generic register.
1606   const RegisterBank *RegBank = nullptr;
1607   if (Name != "_") {
1608     RegBank = PFS.Target.getRegBank(Name);
1609     if (!RegBank)
1610       return error(Loc, "expected '_', register class, or register bank name");
1611   }
1612
1613   lex();
1614
1615   switch (RegInfo.Kind) {
1616   case VRegInfo::UNKNOWN:
1617   case VRegInfo::GENERIC:
1618   case VRegInfo::REGBANK:
1619     RegInfo.Kind = RegBank ? VRegInfo::REGBANK : VRegInfo::GENERIC;
1620     if (RegInfo.Explicit && RegInfo.D.RegBank != RegBank)
1621       return error(Loc, "conflicting generic register banks");
1622     RegInfo.D.RegBank = RegBank;
1623     RegInfo.Explicit = true;
1624     return false;
1625
1626   case VRegInfo::NORMAL:
1627     return error(Loc, "register bank specification on normal register");
1628   }
1629   llvm_unreachable("Unexpected register kind");
1630 }
1631
1632 bool MIParser::parseRegisterFlag(unsigned &Flags) {
1633   const unsigned OldFlags = Flags;
1634   switch (Token.kind()) {
1635   case MIToken::kw_implicit:
1636     Flags |= RegState::Implicit;
1637     break;
1638   case MIToken::kw_implicit_define:
1639     Flags |= RegState::ImplicitDefine;
1640     break;
1641   case MIToken::kw_def:
1642     Flags |= RegState::Define;
1643     break;
1644   case MIToken::kw_dead:
1645     Flags |= RegState::Dead;
1646     break;
1647   case MIToken::kw_killed:
1648     Flags |= RegState::Kill;
1649     break;
1650   case MIToken::kw_undef:
1651     Flags |= RegState::Undef;
1652     break;
1653   case MIToken::kw_internal:
1654     Flags |= RegState::InternalRead;
1655     break;
1656   case MIToken::kw_early_clobber:
1657     Flags |= RegState::EarlyClobber;
1658     break;
1659   case MIToken::kw_debug_use:
1660     Flags |= RegState::Debug;
1661     break;
1662   case MIToken::kw_renamable:
1663     Flags |= RegState::Renamable;
1664     break;
1665   default:
1666     llvm_unreachable("The current token should be a register flag");
1667   }
1668   if (OldFlags == Flags)
1669     // We know that the same flag is specified more than once when the flags
1670     // weren't modified.
1671     return error("duplicate '" + Token.stringValue() + "' register flag");
1672   lex();
1673   return false;
1674 }
1675
1676 bool MIParser::parseSubRegisterIndex(unsigned &SubReg) {
1677   assert(Token.is(MIToken::dot));
1678   lex();
1679   if (Token.isNot(MIToken::Identifier))
1680     return error("expected a subregister index after '.'");
1681   auto Name = Token.stringValue();
1682   SubReg = PFS.Target.getSubRegIndex(Name);
1683   if (!SubReg)
1684     return error(Twine("use of unknown subregister index '") + Name + "'");
1685   lex();
1686   return false;
1687 }
1688
1689 bool MIParser::parseRegisterTiedDefIndex(unsigned &TiedDefIdx) {
1690   if (!consumeIfPresent(MIToken::kw_tied_def))
1691     return true;
1692   if (Token.isNot(MIToken::IntegerLiteral))
1693     return error("expected an integer literal after 'tied-def'");
1694   if (getUnsigned(TiedDefIdx))
1695     return true;
1696   lex();
1697   if (expectAndConsume(MIToken::rparen))
1698     return true;
1699   return false;
1700 }
1701
1702 bool MIParser::assignRegisterTies(MachineInstr &MI,
1703                                   ArrayRef<ParsedMachineOperand> Operands) {
1704   SmallVector<std::pair<unsigned, unsigned>, 4> TiedRegisterPairs;
1705   for (unsigned I = 0, E = Operands.size(); I != E; ++I) {
1706     if (!Operands[I].TiedDefIdx)
1707       continue;
1708     // The parser ensures that this operand is a register use, so we just have
1709     // to check the tied-def operand.
1710     unsigned DefIdx = *Operands[I].TiedDefIdx;
1711     if (DefIdx >= E)
1712       return error(Operands[I].Begin,
1713                    Twine("use of invalid tied-def operand index '" +
1714                          Twine(DefIdx) + "'; instruction has only ") +
1715                        Twine(E) + " operands");
1716     const auto &DefOperand = Operands[DefIdx].Operand;
1717     if (!DefOperand.isReg() || !DefOperand.isDef())
1718       // FIXME: add note with the def operand.
1719       return error(Operands[I].Begin,
1720                    Twine("use of invalid tied-def operand index '") +
1721                        Twine(DefIdx) + "'; the operand #" + Twine(DefIdx) +
1722                        " isn't a defined register");
1723     // Check that the tied-def operand wasn't tied elsewhere.
1724     for (const auto &TiedPair : TiedRegisterPairs) {
1725       if (TiedPair.first == DefIdx)
1726         return error(Operands[I].Begin,
1727                      Twine("the tied-def operand #") + Twine(DefIdx) +
1728                          " is already tied with another register operand");
1729     }
1730     TiedRegisterPairs.push_back(std::make_pair(DefIdx, I));
1731   }
1732   // FIXME: Verify that for non INLINEASM instructions, the def and use tied
1733   // indices must be less than tied max.
1734   for (const auto &TiedPair : TiedRegisterPairs)
1735     MI.tieOperands(TiedPair.first, TiedPair.second);
1736   return false;
1737 }
1738
1739 bool MIParser::parseRegisterOperand(MachineOperand &Dest,
1740                                     std::optional<unsigned> &TiedDefIdx,
1741                                     bool IsDef) {
1742   unsigned Flags = IsDef ? RegState::Define : 0;
1743   while (Token.isRegisterFlag()) {
1744     if (parseRegisterFlag(Flags))
1745       return true;
1746   }
1747   if (!Token.isRegister())
1748     return error("expected a register after register flags");
1749   Register Reg;
1750   VRegInfo *RegInfo;
1751   if (parseRegister(Reg, RegInfo))
1752     return true;
1753   lex();
1754   unsigned SubReg = 0;
1755   if (Token.is(MIToken::dot)) {
1756     if (parseSubRegisterIndex(SubReg))
1757       return true;
1758     if (!Reg.isVirtual())
1759       return error("subregister index expects a virtual register");
1760   }
1761   if (Token.is(MIToken::colon)) {
1762     if (!Reg.isVirtual())
1763       return error("register class specification expects a virtual register");
1764     lex();
1765     if (parseRegisterClassOrBank(*RegInfo))
1766         return true;
1767   }
1768   MachineRegisterInfo &MRI = MF.getRegInfo();
1769   if ((Flags & RegState::Define) == 0) {
1770     if (consumeIfPresent(MIToken::lparen)) {
1771       unsigned Idx;
1772       if (!parseRegisterTiedDefIndex(Idx))
1773         TiedDefIdx = Idx;
1774       else {
1775         // Try a redundant low-level type.
1776         LLT Ty;
1777         if (parseLowLevelType(Token.location(), Ty))
1778           return error("expected tied-def or low-level type after '('");
1779
1780         if (expectAndConsume(MIToken::rparen))
1781           return true;
1782
1783         if (MRI.getType(Reg).isValid() && MRI.getType(Reg) != Ty)
1784           return error("inconsistent type for generic virtual register");
1785
1786         MRI.setRegClassOrRegBank(Reg, static_cast<RegisterBank *>(nullptr));
1787         MRI.setType(Reg, Ty);
1788       }
1789     }
1790   } else if (consumeIfPresent(MIToken::lparen)) {
1791     // Virtual registers may have a tpe with GlobalISel.
1792     if (!Reg.isVirtual())
1793       return error("unexpected type on physical register");
1794
1795     LLT Ty;
1796     if (parseLowLevelType(Token.location(), Ty))
1797       return true;
1798
1799     if (expectAndConsume(MIToken::rparen))
1800       return true;
1801
1802     if (MRI.getType(Reg).isValid() && MRI.getType(Reg) != Ty)
1803       return error("inconsistent type for generic virtual register");
1804
1805     MRI.setRegClassOrRegBank(Reg, static_cast<RegisterBank *>(nullptr));
1806     MRI.setType(Reg, Ty);
1807   } else if (Reg.isVirtual()) {
1808     // Generic virtual registers must have a type.
1809     // If we end up here this means the type hasn't been specified and
1810     // this is bad!
1811     if (RegInfo->Kind == VRegInfo::GENERIC ||
1812         RegInfo->Kind == VRegInfo::REGBANK)
1813       return error("generic virtual registers must have a type");
1814   }
1815
1816   if (Flags & RegState::Define) {
1817     if (Flags & RegState::Kill)
1818       return error("cannot have a killed def operand");
1819   } else {
1820     if (Flags & RegState::Dead)
1821       return error("cannot have a dead use operand");
1822   }
1823
1824   Dest = MachineOperand::CreateReg(
1825       Reg, Flags & RegState::Define, Flags & RegState::Implicit,
1826       Flags & RegState::Kill, Flags & RegState::Dead, Flags & RegState::Undef,
1827       Flags & RegState::EarlyClobber, SubReg, Flags & RegState::Debug,
1828       Flags & RegState::InternalRead, Flags & RegState::Renamable);
1829
1830   return false;
1831 }
1832
1833 bool MIParser::parseImmediateOperand(MachineOperand &Dest) {
1834   assert(Token.is(MIToken::IntegerLiteral));
1835   const APSInt &Int = Token.integerValue();
1836   if (auto SImm = Int.trySExtValue(); Int.isSigned() && SImm.has_value())
1837     Dest = MachineOperand::CreateImm(*SImm);
1838   else if (auto UImm = Int.tryZExtValue(); !Int.isSigned() && UImm.has_value())
1839     Dest = MachineOperand::CreateImm(*UImm);
1840   else
1841     return error("integer literal is too large to be an immediate operand");
1842   lex();
1843   return false;
1844 }
1845
1846 bool MIParser::parseTargetImmMnemonic(const unsigned OpCode,
1847                                       const unsigned OpIdx,
1848                                       MachineOperand &Dest,
1849                                       const MIRFormatter &MF) {
1850   assert(Token.is(MIToken::dot));
1851   auto Loc = Token.location(); // record start position
1852   size_t Len = 1;              // for "."
1853   lex();
1854
1855   // Handle the case that mnemonic starts with number.
1856   if (Token.is(MIToken::IntegerLiteral)) {
1857     Len += Token.range().size();
1858     lex();
1859   }
1860
1861   StringRef Src;
1862   if (Token.is(MIToken::comma))
1863     Src = StringRef(Loc, Len);
1864   else {
1865     assert(Token.is(MIToken::Identifier));
1866     Src = StringRef(Loc, Len + Token.stringValue().size());
1867   }
1868   int64_t Val;
1869   if (MF.parseImmMnemonic(OpCode, OpIdx, Src, Val,
1870                           [this](StringRef::iterator Loc, const Twine &Msg)
1871                               -> bool { return error(Loc, Msg); }))
1872     return true;
1873
1874   Dest = MachineOperand::CreateImm(Val);
1875   if (!Token.is(MIToken::comma))
1876     lex();
1877   return false;
1878 }
1879
1880 static bool parseIRConstant(StringRef::iterator Loc, StringRef StringValue,
1881                             PerFunctionMIParsingState &PFS, const Constant *&C,
1882                             ErrorCallbackType ErrCB) {
1883   auto Source = StringValue.str(); // The source has to be null terminated.
1884   SMDiagnostic Err;
1885   C = parseConstantValue(Source, Err, *PFS.MF.getFunction().getParent(),
1886                          &PFS.IRSlots);
1887   if (!C)
1888     return ErrCB(Loc + Err.getColumnNo(), Err.getMessage());
1889   return false;
1890 }
1891
1892 bool MIParser::parseIRConstant(StringRef::iterator Loc, StringRef StringValue,
1893                                const Constant *&C) {
1894   return ::parseIRConstant(
1895       Loc, StringValue, PFS, C,
1896       [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
1897         return error(Loc, Msg);
1898       });
1899 }
1900
1901 bool MIParser::parseIRConstant(StringRef::iterator Loc, const Constant *&C) {
1902   if (parseIRConstant(Loc, StringRef(Loc, Token.range().end() - Loc), C))
1903     return true;
1904   lex();
1905   return false;
1906 }
1907
1908 // See LLT implementation for bit size limits.
1909 static bool verifyScalarSize(uint64_t Size) {
1910   return Size != 0 && isUInt<16>(Size);
1911 }
1912
1913 static bool verifyVectorElementCount(uint64_t NumElts) {
1914   return NumElts != 0 && isUInt<16>(NumElts);
1915 }
1916
1917 static bool verifyAddrSpace(uint64_t AddrSpace) {
1918   return isUInt<24>(AddrSpace);
1919 }
1920
1921 bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) {
1922   if (Token.range().front() == 's' || Token.range().front() == 'p') {
1923     StringRef SizeStr = Token.range().drop_front();
1924     if (SizeStr.size() == 0 || !llvm::all_of(SizeStr, isdigit))
1925       return error("expected integers after 's'/'p' type character");
1926   }
1927
1928   if (Token.range().front() == 's') {
1929     auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue();
1930     if (!verifyScalarSize(ScalarSize))
1931       return error("invalid size for scalar type");
1932
1933     Ty = LLT::scalar(ScalarSize);
1934     lex();
1935     return false;
1936   } else if (Token.range().front() == 'p') {
1937     const DataLayout &DL = MF.getDataLayout();
1938     uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue();
1939     if (!verifyAddrSpace(AS))
1940       return error("invalid address space number");
1941
1942     Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
1943     lex();
1944     return false;
1945   }
1946
1947   // Now we're looking for a vector.
1948   if (Token.isNot(MIToken::less))
1949     return error(Loc,
1950                  "expected sN, pA, <M x sN>, or <M x pA> for GlobalISel type");
1951   lex();
1952
1953   if (Token.isNot(MIToken::IntegerLiteral))
1954     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
1955   uint64_t NumElements = Token.integerValue().getZExtValue();
1956   if (!verifyVectorElementCount(NumElements))
1957     return error("invalid number of vector elements");
1958
1959   lex();
1960
1961   if (Token.isNot(MIToken::Identifier) || Token.stringValue() != "x")
1962     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
1963   lex();
1964
1965   if (Token.range().front() != 's' && Token.range().front() != 'p')
1966     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
1967   StringRef SizeStr = Token.range().drop_front();
1968   if (SizeStr.size() == 0 || !llvm::all_of(SizeStr, isdigit))
1969     return error("expected integers after 's'/'p' type character");
1970
1971   if (Token.range().front() == 's') {
1972     auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue();
1973     if (!verifyScalarSize(ScalarSize))
1974       return error("invalid size for scalar type");
1975     Ty = LLT::scalar(ScalarSize);
1976   } else if (Token.range().front() == 'p') {
1977     const DataLayout &DL = MF.getDataLayout();
1978     uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue();
1979     if (!verifyAddrSpace(AS))
1980       return error("invalid address space number");
1981
1982     Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS));
1983   } else
1984     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
1985   lex();
1986
1987   if (Token.isNot(MIToken::greater))
1988     return error(Loc, "expected <M x sN> or <M x pA> for vector type");
1989   lex();
1990
1991   Ty = LLT::fixed_vector(NumElements, Ty);
1992   return false;
1993 }
1994
1995 bool MIParser::parseTypedImmediateOperand(MachineOperand &Dest) {
1996   assert(Token.is(MIToken::Identifier));
1997   StringRef TypeStr = Token.range();
1998   if (TypeStr.front() != 'i' && TypeStr.front() != 's' &&
1999       TypeStr.front() != 'p')
2000     return error(
2001         "a typed immediate operand should start with one of 'i', 's', or 'p'");
2002   StringRef SizeStr = Token.range().drop_front();
2003   if (SizeStr.size() == 0 || !llvm::all_of(SizeStr, isdigit))
2004     return error("expected integers after 'i'/'s'/'p' type character");
2005
2006   auto Loc = Token.location();
2007   lex();
2008   if (Token.isNot(MIToken::IntegerLiteral)) {
2009     if (Token.isNot(MIToken::Identifier) ||
2010         !(Token.range() == "true" || Token.range() == "false"))
2011       return error("expected an integer literal");
2012   }
2013   const Constant *C = nullptr;
2014   if (parseIRConstant(Loc, C))
2015     return true;
2016   Dest = MachineOperand::CreateCImm(cast<ConstantInt>(C));
2017   return false;
2018 }
2019
2020 bool MIParser::parseFPImmediateOperand(MachineOperand &Dest) {
2021   auto Loc = Token.location();
2022   lex();
2023   if (Token.isNot(MIToken::FloatingPointLiteral) &&
2024       Token.isNot(MIToken::HexLiteral))
2025     return error("expected a floating point literal");
2026   const Constant *C = nullptr;
2027   if (parseIRConstant(Loc, C))
2028     return true;
2029   Dest = MachineOperand::CreateFPImm(cast<ConstantFP>(C));
2030   return false;
2031 }
2032
2033 static bool getHexUint(const MIToken &Token, APInt &Result) {
2034   assert(Token.is(MIToken::HexLiteral));
2035   StringRef S = Token.range();
2036   assert(S[0] == '0' && tolower(S[1]) == 'x');
2037   // This could be a floating point literal with a special prefix.
2038   if (!isxdigit(S[2]))
2039     return true;
2040   StringRef V = S.substr(2);
2041   APInt A(V.size()*4, V, 16);
2042
2043   // If A is 0, then A.getActiveBits() is 0. This isn't a valid bitwidth. Make
2044   // sure it isn't the case before constructing result.
2045   unsigned NumBits = (A == 0) ? 32 : A.getActiveBits();
2046   Result = APInt(NumBits, ArrayRef<uint64_t>(A.getRawData(), A.getNumWords()));
2047   return false;
2048 }
2049
2050 static bool getUnsigned(const MIToken &Token, unsigned &Result,
2051                         ErrorCallbackType ErrCB) {
2052   if (Token.hasIntegerValue()) {
2053     const uint64_t Limit = uint64_t(std::numeric_limits<unsigned>::max()) + 1;
2054     uint64_t Val64 = Token.integerValue().getLimitedValue(Limit);
2055     if (Val64 == Limit)
2056       return ErrCB(Token.location(), "expected 32-bit integer (too large)");
2057     Result = Val64;
2058     return false;
2059   }
2060   if (Token.is(MIToken::HexLiteral)) {
2061     APInt A;
2062     if (getHexUint(Token, A))
2063       return true;
2064     if (A.getBitWidth() > 32)
2065       return ErrCB(Token.location(), "expected 32-bit integer (too large)");
2066     Result = A.getZExtValue();
2067     return false;
2068   }
2069   return true;
2070 }
2071
2072 bool MIParser::getUnsigned(unsigned &Result) {
2073   return ::getUnsigned(
2074       Token, Result, [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
2075         return error(Loc, Msg);
2076       });
2077 }
2078
2079 bool MIParser::parseMBBReference(MachineBasicBlock *&MBB) {
2080   assert(Token.is(MIToken::MachineBasicBlock) ||
2081          Token.is(MIToken::MachineBasicBlockLabel));
2082   unsigned Number;
2083   if (getUnsigned(Number))
2084     return true;
2085   auto MBBInfo = PFS.MBBSlots.find(Number);
2086   if (MBBInfo == PFS.MBBSlots.end())
2087     return error(Twine("use of undefined machine basic block #") +
2088                  Twine(Number));
2089   MBB = MBBInfo->second;
2090   // TODO: Only parse the name if it's a MachineBasicBlockLabel. Deprecate once
2091   // we drop the <irname> from the bb.<id>.<irname> format.
2092   if (!Token.stringValue().empty() && Token.stringValue() != MBB->getName())
2093     return error(Twine("the name of machine basic block #") + Twine(Number) +
2094                  " isn't '" + Token.stringValue() + "'");
2095   return false;
2096 }
2097
2098 bool MIParser::parseMBBOperand(MachineOperand &Dest) {
2099   MachineBasicBlock *MBB;
2100   if (parseMBBReference(MBB))
2101     return true;
2102   Dest = MachineOperand::CreateMBB(MBB);
2103   lex();
2104   return false;
2105 }
2106
2107 bool MIParser::parseStackFrameIndex(int &FI) {
2108   assert(Token.is(MIToken::StackObject));
2109   unsigned ID;
2110   if (getUnsigned(ID))
2111     return true;
2112   auto ObjectInfo = PFS.StackObjectSlots.find(ID);
2113   if (ObjectInfo == PFS.StackObjectSlots.end())
2114     return error(Twine("use of undefined stack object '%stack.") + Twine(ID) +
2115                  "'");
2116   StringRef Name;
2117   if (const auto *Alloca =
2118           MF.getFrameInfo().getObjectAllocation(ObjectInfo->second))
2119     Name = Alloca->getName();
2120   if (!Token.stringValue().empty() && Token.stringValue() != Name)
2121     return error(Twine("the name of the stack object '%stack.") + Twine(ID) +
2122                  "' isn't '" + Token.stringValue() + "'");
2123   lex();
2124   FI = ObjectInfo->second;
2125   return false;
2126 }
2127
2128 bool MIParser::parseStackObjectOperand(MachineOperand &Dest) {
2129   int FI;
2130   if (parseStackFrameIndex(FI))
2131     return true;
2132   Dest = MachineOperand::CreateFI(FI);
2133   return false;
2134 }
2135
2136 bool MIParser::parseFixedStackFrameIndex(int &FI) {
2137   assert(Token.is(MIToken::FixedStackObject));
2138   unsigned ID;
2139   if (getUnsigned(ID))
2140     return true;
2141   auto ObjectInfo = PFS.FixedStackObjectSlots.find(ID);
2142   if (ObjectInfo == PFS.FixedStackObjectSlots.end())
2143     return error(Twine("use of undefined fixed stack object '%fixed-stack.") +
2144                  Twine(ID) + "'");
2145   lex();
2146   FI = ObjectInfo->second;
2147   return false;
2148 }
2149
2150 bool MIParser::parseFixedStackObjectOperand(MachineOperand &Dest) {
2151   int FI;
2152   if (parseFixedStackFrameIndex(FI))
2153     return true;
2154   Dest = MachineOperand::CreateFI(FI);
2155   return false;
2156 }
2157
2158 static bool parseGlobalValue(const MIToken &Token,
2159                              PerFunctionMIParsingState &PFS, GlobalValue *&GV,
2160                              ErrorCallbackType ErrCB) {
2161   switch (Token.kind()) {
2162   case MIToken::NamedGlobalValue: {
2163     const Module *M = PFS.MF.getFunction().getParent();
2164     GV = M->getNamedValue(Token.stringValue());
2165     if (!GV)
2166       return ErrCB(Token.location(), Twine("use of undefined global value '") +
2167                                          Token.range() + "'");
2168     break;
2169   }
2170   case MIToken::GlobalValue: {
2171     unsigned GVIdx;
2172     if (getUnsigned(Token, GVIdx, ErrCB))
2173       return true;
2174     if (GVIdx >= PFS.IRSlots.GlobalValues.size())
2175       return ErrCB(Token.location(), Twine("use of undefined global value '@") +
2176                                          Twine(GVIdx) + "'");
2177     GV = PFS.IRSlots.GlobalValues[GVIdx];
2178     break;
2179   }
2180   default:
2181     llvm_unreachable("The current token should be a global value");
2182   }
2183   return false;
2184 }
2185
2186 bool MIParser::parseGlobalValue(GlobalValue *&GV) {
2187   return ::parseGlobalValue(
2188       Token, PFS, GV,
2189       [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
2190         return error(Loc, Msg);
2191       });
2192 }
2193
2194 bool MIParser::parseGlobalAddressOperand(MachineOperand &Dest) {
2195   GlobalValue *GV = nullptr;
2196   if (parseGlobalValue(GV))
2197     return true;
2198   lex();
2199   Dest = MachineOperand::CreateGA(GV, /*Offset=*/0);
2200   if (parseOperandsOffset(Dest))
2201     return true;
2202   return false;
2203 }
2204
2205 bool MIParser::parseConstantPoolIndexOperand(MachineOperand &Dest) {
2206   assert(Token.is(MIToken::ConstantPoolItem));
2207   unsigned ID;
2208   if (getUnsigned(ID))
2209     return true;
2210   auto ConstantInfo = PFS.ConstantPoolSlots.find(ID);
2211   if (ConstantInfo == PFS.ConstantPoolSlots.end())
2212     return error("use of undefined constant '%const." + Twine(ID) + "'");
2213   lex();
2214   Dest = MachineOperand::CreateCPI(ID, /*Offset=*/0);
2215   if (parseOperandsOffset(Dest))
2216     return true;
2217   return false;
2218 }
2219
2220 bool MIParser::parseJumpTableIndexOperand(MachineOperand &Dest) {
2221   assert(Token.is(MIToken::JumpTableIndex));
2222   unsigned ID;
2223   if (getUnsigned(ID))
2224     return true;
2225   auto JumpTableEntryInfo = PFS.JumpTableSlots.find(ID);
2226   if (JumpTableEntryInfo == PFS.JumpTableSlots.end())
2227     return error("use of undefined jump table '%jump-table." + Twine(ID) + "'");
2228   lex();
2229   Dest = MachineOperand::CreateJTI(JumpTableEntryInfo->second);
2230   return false;
2231 }
2232
2233 bool MIParser::parseExternalSymbolOperand(MachineOperand &Dest) {
2234   assert(Token.is(MIToken::ExternalSymbol));
2235   const char *Symbol = MF.createExternalSymbolName(Token.stringValue());
2236   lex();
2237   Dest = MachineOperand::CreateES(Symbol);
2238   if (parseOperandsOffset(Dest))
2239     return true;
2240   return false;
2241 }
2242
2243 bool MIParser::parseMCSymbolOperand(MachineOperand &Dest) {
2244   assert(Token.is(MIToken::MCSymbol));
2245   MCSymbol *Symbol = getOrCreateMCSymbol(Token.stringValue());
2246   lex();
2247   Dest = MachineOperand::CreateMCSymbol(Symbol);
2248   if (parseOperandsOffset(Dest))
2249     return true;
2250   return false;
2251 }
2252
2253 bool MIParser::parseSubRegisterIndexOperand(MachineOperand &Dest) {
2254   assert(Token.is(MIToken::SubRegisterIndex));
2255   StringRef Name = Token.stringValue();
2256   unsigned SubRegIndex = PFS.Target.getSubRegIndex(Token.stringValue());
2257   if (SubRegIndex == 0)
2258     return error(Twine("unknown subregister index '") + Name + "'");
2259   lex();
2260   Dest = MachineOperand::CreateImm(SubRegIndex);
2261   return false;
2262 }
2263
2264 bool MIParser::parseMDNode(MDNode *&Node) {
2265   assert(Token.is(MIToken::exclaim));
2266
2267   auto Loc = Token.location();
2268   lex();
2269   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
2270     return error("expected metadata id after '!'");
2271   unsigned ID;
2272   if (getUnsigned(ID))
2273     return true;
2274   auto NodeInfo = PFS.IRSlots.MetadataNodes.find(ID);
2275   if (NodeInfo == PFS.IRSlots.MetadataNodes.end()) {
2276     NodeInfo = PFS.MachineMetadataNodes.find(ID);
2277     if (NodeInfo == PFS.MachineMetadataNodes.end())
2278       return error(Loc, "use of undefined metadata '!" + Twine(ID) + "'");
2279   }
2280   lex();
2281   Node = NodeInfo->second.get();
2282   return false;
2283 }
2284
2285 bool MIParser::parseDIExpression(MDNode *&Expr) {
2286   assert(Token.is(MIToken::md_diexpr));
2287   lex();
2288
2289   // FIXME: Share this parsing with the IL parser.
2290   SmallVector<uint64_t, 8> Elements;
2291
2292   if (expectAndConsume(MIToken::lparen))
2293     return true;
2294
2295   if (Token.isNot(MIToken::rparen)) {
2296     do {
2297       if (Token.is(MIToken::Identifier)) {
2298         if (unsigned Op = dwarf::getOperationEncoding(Token.stringValue())) {
2299           lex();
2300           Elements.push_back(Op);
2301           continue;
2302         }
2303         if (unsigned Enc = dwarf::getAttributeEncoding(Token.stringValue())) {
2304           lex();
2305           Elements.push_back(Enc);
2306           continue;
2307         }
2308         return error(Twine("invalid DWARF op '") + Token.stringValue() + "'");
2309       }
2310
2311       if (Token.isNot(MIToken::IntegerLiteral) ||
2312           Token.integerValue().isSigned())
2313         return error("expected unsigned integer");
2314
2315       auto &U = Token.integerValue();
2316       if (U.ugt(UINT64_MAX))
2317         return error("element too large, limit is " + Twine(UINT64_MAX));
2318       Elements.push_back(U.getZExtValue());
2319       lex();
2320
2321     } while (consumeIfPresent(MIToken::comma));
2322   }
2323
2324   if (expectAndConsume(MIToken::rparen))
2325     return true;
2326
2327   Expr = DIExpression::get(MF.getFunction().getContext(), Elements);
2328   return false;
2329 }
2330
2331 bool MIParser::parseDILocation(MDNode *&Loc) {
2332   assert(Token.is(MIToken::md_dilocation));
2333   lex();
2334
2335   bool HaveLine = false;
2336   unsigned Line = 0;
2337   unsigned Column = 0;
2338   MDNode *Scope = nullptr;
2339   MDNode *InlinedAt = nullptr;
2340   bool ImplicitCode = false;
2341
2342   if (expectAndConsume(MIToken::lparen))
2343     return true;
2344
2345   if (Token.isNot(MIToken::rparen)) {
2346     do {
2347       if (Token.is(MIToken::Identifier)) {
2348         if (Token.stringValue() == "line") {
2349           lex();
2350           if (expectAndConsume(MIToken::colon))
2351             return true;
2352           if (Token.isNot(MIToken::IntegerLiteral) ||
2353               Token.integerValue().isSigned())
2354             return error("expected unsigned integer");
2355           Line = Token.integerValue().getZExtValue();
2356           HaveLine = true;
2357           lex();
2358           continue;
2359         }
2360         if (Token.stringValue() == "column") {
2361           lex();
2362           if (expectAndConsume(MIToken::colon))
2363             return true;
2364           if (Token.isNot(MIToken::IntegerLiteral) ||
2365               Token.integerValue().isSigned())
2366             return error("expected unsigned integer");
2367           Column = Token.integerValue().getZExtValue();
2368           lex();
2369           continue;
2370         }
2371         if (Token.stringValue() == "scope") {
2372           lex();
2373           if (expectAndConsume(MIToken::colon))
2374             return true;
2375           if (parseMDNode(Scope))
2376             return error("expected metadata node");
2377           if (!isa<DIScope>(Scope))
2378             return error("expected DIScope node");
2379           continue;
2380         }
2381         if (Token.stringValue() == "inlinedAt") {
2382           lex();
2383           if (expectAndConsume(MIToken::colon))
2384             return true;
2385           if (Token.is(MIToken::exclaim)) {
2386             if (parseMDNode(InlinedAt))
2387               return true;
2388           } else if (Token.is(MIToken::md_dilocation)) {
2389             if (parseDILocation(InlinedAt))
2390               return true;
2391           } else
2392             return error("expected metadata node");
2393           if (!isa<DILocation>(InlinedAt))
2394             return error("expected DILocation node");
2395           continue;
2396         }
2397         if (Token.stringValue() == "isImplicitCode") {
2398           lex();
2399           if (expectAndConsume(MIToken::colon))
2400             return true;
2401           if (!Token.is(MIToken::Identifier))
2402             return error("expected true/false");
2403           // As far as I can see, we don't have any existing need for parsing
2404           // true/false in MIR yet. Do it ad-hoc until there's something else
2405           // that needs it.
2406           if (Token.stringValue() == "true")
2407             ImplicitCode = true;
2408           else if (Token.stringValue() == "false")
2409             ImplicitCode = false;
2410           else
2411             return error("expected true/false");
2412           lex();
2413           continue;
2414         }
2415       }
2416       return error(Twine("invalid DILocation argument '") +
2417                    Token.stringValue() + "'");
2418     } while (consumeIfPresent(MIToken::comma));
2419   }
2420
2421   if (expectAndConsume(MIToken::rparen))
2422     return true;
2423
2424   if (!HaveLine)
2425     return error("DILocation requires line number");
2426   if (!Scope)
2427     return error("DILocation requires a scope");
2428
2429   Loc = DILocation::get(MF.getFunction().getContext(), Line, Column, Scope,
2430                         InlinedAt, ImplicitCode);
2431   return false;
2432 }
2433
2434 bool MIParser::parseMetadataOperand(MachineOperand &Dest) {
2435   MDNode *Node = nullptr;
2436   if (Token.is(MIToken::exclaim)) {
2437     if (parseMDNode(Node))
2438       return true;
2439   } else if (Token.is(MIToken::md_diexpr)) {
2440     if (parseDIExpression(Node))
2441       return true;
2442   }
2443   Dest = MachineOperand::CreateMetadata(Node);
2444   return false;
2445 }
2446
2447 bool MIParser::parseCFIOffset(int &Offset) {
2448   if (Token.isNot(MIToken::IntegerLiteral))
2449     return error("expected a cfi offset");
2450   if (Token.integerValue().getSignificantBits() > 32)
2451     return error("expected a 32 bit integer (the cfi offset is too large)");
2452   Offset = (int)Token.integerValue().getExtValue();
2453   lex();
2454   return false;
2455 }
2456
2457 bool MIParser::parseCFIRegister(Register &Reg) {
2458   if (Token.isNot(MIToken::NamedRegister))
2459     return error("expected a cfi register");
2460   Register LLVMReg;
2461   if (parseNamedRegister(LLVMReg))
2462     return true;
2463   const auto *TRI = MF.getSubtarget().getRegisterInfo();
2464   assert(TRI && "Expected target register info");
2465   int DwarfReg = TRI->getDwarfRegNum(LLVMReg, true);
2466   if (DwarfReg < 0)
2467     return error("invalid DWARF register");
2468   Reg = (unsigned)DwarfReg;
2469   lex();
2470   return false;
2471 }
2472
2473 bool MIParser::parseCFIAddressSpace(unsigned &AddressSpace) {
2474   if (Token.isNot(MIToken::IntegerLiteral))
2475     return error("expected a cfi address space literal");
2476   if (Token.integerValue().isSigned())
2477     return error("expected an unsigned integer (cfi address space)");
2478   AddressSpace = Token.integerValue().getZExtValue();
2479   lex();
2480   return false;
2481 }
2482
2483 bool MIParser::parseCFIEscapeValues(std::string &Values) {
2484   do {
2485     if (Token.isNot(MIToken::HexLiteral))
2486       return error("expected a hexadecimal literal");
2487     unsigned Value;
2488     if (getUnsigned(Value))
2489       return true;
2490     if (Value > UINT8_MAX)
2491       return error("expected a 8-bit integer (too large)");
2492     Values.push_back(static_cast<uint8_t>(Value));
2493     lex();
2494   } while (consumeIfPresent(MIToken::comma));
2495   return false;
2496 }
2497
2498 bool MIParser::parseCFIOperand(MachineOperand &Dest) {
2499   auto Kind = Token.kind();
2500   lex();
2501   int Offset;
2502   Register Reg;
2503   unsigned AddressSpace;
2504   unsigned CFIIndex;
2505   switch (Kind) {
2506   case MIToken::kw_cfi_same_value:
2507     if (parseCFIRegister(Reg))
2508       return true;
2509     CFIIndex = MF.addFrameInst(MCCFIInstruction::createSameValue(nullptr, Reg));
2510     break;
2511   case MIToken::kw_cfi_offset:
2512     if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
2513         parseCFIOffset(Offset))
2514       return true;
2515     CFIIndex =
2516         MF.addFrameInst(MCCFIInstruction::createOffset(nullptr, Reg, Offset));
2517     break;
2518   case MIToken::kw_cfi_rel_offset:
2519     if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
2520         parseCFIOffset(Offset))
2521       return true;
2522     CFIIndex = MF.addFrameInst(
2523         MCCFIInstruction::createRelOffset(nullptr, Reg, Offset));
2524     break;
2525   case MIToken::kw_cfi_def_cfa_register:
2526     if (parseCFIRegister(Reg))
2527       return true;
2528     CFIIndex =
2529         MF.addFrameInst(MCCFIInstruction::createDefCfaRegister(nullptr, Reg));
2530     break;
2531   case MIToken::kw_cfi_def_cfa_offset:
2532     if (parseCFIOffset(Offset))
2533       return true;
2534     CFIIndex =
2535         MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset(nullptr, Offset));
2536     break;
2537   case MIToken::kw_cfi_adjust_cfa_offset:
2538     if (parseCFIOffset(Offset))
2539       return true;
2540     CFIIndex = MF.addFrameInst(
2541         MCCFIInstruction::createAdjustCfaOffset(nullptr, Offset));
2542     break;
2543   case MIToken::kw_cfi_def_cfa:
2544     if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
2545         parseCFIOffset(Offset))
2546       return true;
2547     CFIIndex =
2548         MF.addFrameInst(MCCFIInstruction::cfiDefCfa(nullptr, Reg, Offset));
2549     break;
2550   case MIToken::kw_cfi_llvm_def_aspace_cfa:
2551     if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
2552         parseCFIOffset(Offset) || expectAndConsume(MIToken::comma) ||
2553         parseCFIAddressSpace(AddressSpace))
2554       return true;
2555     CFIIndex = MF.addFrameInst(MCCFIInstruction::createLLVMDefAspaceCfa(
2556         nullptr, Reg, Offset, AddressSpace, SMLoc()));
2557     break;
2558   case MIToken::kw_cfi_remember_state:
2559     CFIIndex = MF.addFrameInst(MCCFIInstruction::createRememberState(nullptr));
2560     break;
2561   case MIToken::kw_cfi_restore:
2562     if (parseCFIRegister(Reg))
2563       return true;
2564     CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestore(nullptr, Reg));
2565     break;
2566   case MIToken::kw_cfi_restore_state:
2567     CFIIndex = MF.addFrameInst(MCCFIInstruction::createRestoreState(nullptr));
2568     break;
2569   case MIToken::kw_cfi_undefined:
2570     if (parseCFIRegister(Reg))
2571       return true;
2572     CFIIndex = MF.addFrameInst(MCCFIInstruction::createUndefined(nullptr, Reg));
2573     break;
2574   case MIToken::kw_cfi_register: {
2575     Register Reg2;
2576     if (parseCFIRegister(Reg) || expectAndConsume(MIToken::comma) ||
2577         parseCFIRegister(Reg2))
2578       return true;
2579
2580     CFIIndex =
2581         MF.addFrameInst(MCCFIInstruction::createRegister(nullptr, Reg, Reg2));
2582     break;
2583   }
2584   case MIToken::kw_cfi_window_save:
2585     CFIIndex = MF.addFrameInst(MCCFIInstruction::createWindowSave(nullptr));
2586     break;
2587   case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
2588     CFIIndex = MF.addFrameInst(MCCFIInstruction::createNegateRAState(nullptr));
2589     break;
2590   case MIToken::kw_cfi_escape: {
2591     std::string Values;
2592     if (parseCFIEscapeValues(Values))
2593       return true;
2594     CFIIndex = MF.addFrameInst(MCCFIInstruction::createEscape(nullptr, Values));
2595     break;
2596   }
2597   default:
2598     // TODO: Parse the other CFI operands.
2599     llvm_unreachable("The current token should be a cfi operand");
2600   }
2601   Dest = MachineOperand::CreateCFIIndex(CFIIndex);
2602   return false;
2603 }
2604
2605 bool MIParser::parseIRBlock(BasicBlock *&BB, const Function &F) {
2606   switch (Token.kind()) {
2607   case MIToken::NamedIRBlock: {
2608     BB = dyn_cast_or_null<BasicBlock>(
2609         F.getValueSymbolTable()->lookup(Token.stringValue()));
2610     if (!BB)
2611       return error(Twine("use of undefined IR block '") + Token.range() + "'");
2612     break;
2613   }
2614   case MIToken::IRBlock: {
2615     unsigned SlotNumber = 0;
2616     if (getUnsigned(SlotNumber))
2617       return true;
2618     BB = const_cast<BasicBlock *>(getIRBlock(SlotNumber, F));
2619     if (!BB)
2620       return error(Twine("use of undefined IR block '%ir-block.") +
2621                    Twine(SlotNumber) + "'");
2622     break;
2623   }
2624   default:
2625     llvm_unreachable("The current token should be an IR block reference");
2626   }
2627   return false;
2628 }
2629
2630 bool MIParser::parseBlockAddressOperand(MachineOperand &Dest) {
2631   assert(Token.is(MIToken::kw_blockaddress));
2632   lex();
2633   if (expectAndConsume(MIToken::lparen))
2634     return true;
2635   if (Token.isNot(MIToken::GlobalValue) &&
2636       Token.isNot(MIToken::NamedGlobalValue))
2637     return error("expected a global value");
2638   GlobalValue *GV = nullptr;
2639   if (parseGlobalValue(GV))
2640     return true;
2641   auto *F = dyn_cast<Function>(GV);
2642   if (!F)
2643     return error("expected an IR function reference");
2644   lex();
2645   if (expectAndConsume(MIToken::comma))
2646     return true;
2647   BasicBlock *BB = nullptr;
2648   if (Token.isNot(MIToken::IRBlock) && Token.isNot(MIToken::NamedIRBlock))
2649     return error("expected an IR block reference");
2650   if (parseIRBlock(BB, *F))
2651     return true;
2652   lex();
2653   if (expectAndConsume(MIToken::rparen))
2654     return true;
2655   Dest = MachineOperand::CreateBA(BlockAddress::get(F, BB), /*Offset=*/0);
2656   if (parseOperandsOffset(Dest))
2657     return true;
2658   return false;
2659 }
2660
2661 bool MIParser::parseIntrinsicOperand(MachineOperand &Dest) {
2662   assert(Token.is(MIToken::kw_intrinsic));
2663   lex();
2664   if (expectAndConsume(MIToken::lparen))
2665     return error("expected syntax intrinsic(@llvm.whatever)");
2666
2667   if (Token.isNot(MIToken::NamedGlobalValue))
2668     return error("expected syntax intrinsic(@llvm.whatever)");
2669
2670   std::string Name = std::string(Token.stringValue());
2671   lex();
2672
2673   if (expectAndConsume(MIToken::rparen))
2674     return error("expected ')' to terminate intrinsic name");
2675
2676   // Find out what intrinsic we're dealing with, first try the global namespace
2677   // and then the target's private intrinsics if that fails.
2678   const TargetIntrinsicInfo *TII = MF.getTarget().getIntrinsicInfo();
2679   Intrinsic::ID ID = Function::lookupIntrinsicID(Name);
2680   if (ID == Intrinsic::not_intrinsic && TII)
2681     ID = static_cast<Intrinsic::ID>(TII->lookupName(Name));
2682
2683   if (ID == Intrinsic::not_intrinsic)
2684     return error("unknown intrinsic name");
2685   Dest = MachineOperand::CreateIntrinsicID(ID);
2686
2687   return false;
2688 }
2689
2690 bool MIParser::parsePredicateOperand(MachineOperand &Dest) {
2691   assert(Token.is(MIToken::kw_intpred) || Token.is(MIToken::kw_floatpred));
2692   bool IsFloat = Token.is(MIToken::kw_floatpred);
2693   lex();
2694
2695   if (expectAndConsume(MIToken::lparen))
2696     return error("expected syntax intpred(whatever) or floatpred(whatever");
2697
2698   if (Token.isNot(MIToken::Identifier))
2699     return error("whatever");
2700
2701   CmpInst::Predicate Pred;
2702   if (IsFloat) {
2703     Pred = StringSwitch<CmpInst::Predicate>(Token.stringValue())
2704                .Case("false", CmpInst::FCMP_FALSE)
2705                .Case("oeq", CmpInst::FCMP_OEQ)
2706                .Case("ogt", CmpInst::FCMP_OGT)
2707                .Case("oge", CmpInst::FCMP_OGE)
2708                .Case("olt", CmpInst::FCMP_OLT)
2709                .Case("ole", CmpInst::FCMP_OLE)
2710                .Case("one", CmpInst::FCMP_ONE)
2711                .Case("ord", CmpInst::FCMP_ORD)
2712                .Case("uno", CmpInst::FCMP_UNO)
2713                .Case("ueq", CmpInst::FCMP_UEQ)
2714                .Case("ugt", CmpInst::FCMP_UGT)
2715                .Case("uge", CmpInst::FCMP_UGE)
2716                .Case("ult", CmpInst::FCMP_ULT)
2717                .Case("ule", CmpInst::FCMP_ULE)
2718                .Case("une", CmpInst::FCMP_UNE)
2719                .Case("true", CmpInst::FCMP_TRUE)
2720                .Default(CmpInst::BAD_FCMP_PREDICATE);
2721     if (!CmpInst::isFPPredicate(Pred))
2722       return error("invalid floating-point predicate");
2723   } else {
2724     Pred = StringSwitch<CmpInst::Predicate>(Token.stringValue())
2725                .Case("eq", CmpInst::ICMP_EQ)
2726                .Case("ne", CmpInst::ICMP_NE)
2727                .Case("sgt", CmpInst::ICMP_SGT)
2728                .Case("sge", CmpInst::ICMP_SGE)
2729                .Case("slt", CmpInst::ICMP_SLT)
2730                .Case("sle", CmpInst::ICMP_SLE)
2731                .Case("ugt", CmpInst::ICMP_UGT)
2732                .Case("uge", CmpInst::ICMP_UGE)
2733                .Case("ult", CmpInst::ICMP_ULT)
2734                .Case("ule", CmpInst::ICMP_ULE)
2735                .Default(CmpInst::BAD_ICMP_PREDICATE);
2736     if (!CmpInst::isIntPredicate(Pred))
2737       return error("invalid integer predicate");
2738   }
2739
2740   lex();
2741   Dest = MachineOperand::CreatePredicate(Pred);
2742   if (expectAndConsume(MIToken::rparen))
2743     return error("predicate should be terminated by ')'.");
2744
2745   return false;
2746 }
2747
2748 bool MIParser::parseShuffleMaskOperand(MachineOperand &Dest) {
2749   assert(Token.is(MIToken::kw_shufflemask));
2750
2751   lex();
2752   if (expectAndConsume(MIToken::lparen))
2753     return error("expected syntax shufflemask(<integer or undef>, ...)");
2754
2755   SmallVector<int, 32> ShufMask;
2756   do {
2757     if (Token.is(MIToken::kw_undef)) {
2758       ShufMask.push_back(-1);
2759     } else if (Token.is(MIToken::IntegerLiteral)) {
2760       const APSInt &Int = Token.integerValue();
2761       ShufMask.push_back(Int.getExtValue());
2762     } else
2763       return error("expected integer constant");
2764
2765     lex();
2766   } while (consumeIfPresent(MIToken::comma));
2767
2768   if (expectAndConsume(MIToken::rparen))
2769     return error("shufflemask should be terminated by ')'.");
2770
2771   ArrayRef<int> MaskAlloc = MF.allocateShuffleMask(ShufMask);
2772   Dest = MachineOperand::CreateShuffleMask(MaskAlloc);
2773   return false;
2774 }
2775
2776 bool MIParser::parseDbgInstrRefOperand(MachineOperand &Dest) {
2777   assert(Token.is(MIToken::kw_dbg_instr_ref));
2778
2779   lex();
2780   if (expectAndConsume(MIToken::lparen))
2781     return error("expected syntax dbg-instr-ref(<unsigned>, <unsigned>)");
2782
2783   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isNegative())
2784     return error("expected unsigned integer for instruction index");
2785   uint64_t InstrIdx = Token.integerValue().getZExtValue();
2786   assert(InstrIdx <= std::numeric_limits<unsigned>::max() &&
2787          "Instruction reference's instruction index is too large");
2788   lex();
2789
2790   if (expectAndConsume(MIToken::comma))
2791     return error("expected syntax dbg-instr-ref(<unsigned>, <unsigned>)");
2792
2793   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isNegative())
2794     return error("expected unsigned integer for operand index");
2795   uint64_t OpIdx = Token.integerValue().getZExtValue();
2796   assert(OpIdx <= std::numeric_limits<unsigned>::max() &&
2797          "Instruction reference's operand index is too large");
2798   lex();
2799
2800   if (expectAndConsume(MIToken::rparen))
2801     return error("expected syntax dbg-instr-ref(<unsigned>, <unsigned>)");
2802
2803   Dest = MachineOperand::CreateDbgInstrRef(InstrIdx, OpIdx);
2804   return false;
2805 }
2806
2807 bool MIParser::parseTargetIndexOperand(MachineOperand &Dest) {
2808   assert(Token.is(MIToken::kw_target_index));
2809   lex();
2810   if (expectAndConsume(MIToken::lparen))
2811     return true;
2812   if (Token.isNot(MIToken::Identifier))
2813     return error("expected the name of the target index");
2814   int Index = 0;
2815   if (PFS.Target.getTargetIndex(Token.stringValue(), Index))
2816     return error("use of undefined target index '" + Token.stringValue() + "'");
2817   lex();
2818   if (expectAndConsume(MIToken::rparen))
2819     return true;
2820   Dest = MachineOperand::CreateTargetIndex(unsigned(Index), /*Offset=*/0);
2821   if (parseOperandsOffset(Dest))
2822     return true;
2823   return false;
2824 }
2825
2826 bool MIParser::parseCustomRegisterMaskOperand(MachineOperand &Dest) {
2827   assert(Token.stringValue() == "CustomRegMask" && "Expected a custom RegMask");
2828   lex();
2829   if (expectAndConsume(MIToken::lparen))
2830     return true;
2831
2832   uint32_t *Mask = MF.allocateRegMask();
2833   do {
2834     if (Token.isNot(MIToken::rparen)) {
2835       if (Token.isNot(MIToken::NamedRegister))
2836         return error("expected a named register");
2837       Register Reg;
2838       if (parseNamedRegister(Reg))
2839         return true;
2840       lex();
2841       Mask[Reg / 32] |= 1U << (Reg % 32);
2842     }
2843
2844     // TODO: Report an error if the same register is used more than once.
2845   } while (consumeIfPresent(MIToken::comma));
2846
2847   if (expectAndConsume(MIToken::rparen))
2848     return true;
2849   Dest = MachineOperand::CreateRegMask(Mask);
2850   return false;
2851 }
2852
2853 bool MIParser::parseLiveoutRegisterMaskOperand(MachineOperand &Dest) {
2854   assert(Token.is(MIToken::kw_liveout));
2855   uint32_t *Mask = MF.allocateRegMask();
2856   lex();
2857   if (expectAndConsume(MIToken::lparen))
2858     return true;
2859   while (true) {
2860     if (Token.isNot(MIToken::NamedRegister))
2861       return error("expected a named register");
2862     Register Reg;
2863     if (parseNamedRegister(Reg))
2864       return true;
2865     lex();
2866     Mask[Reg / 32] |= 1U << (Reg % 32);
2867     // TODO: Report an error if the same register is used more than once.
2868     if (Token.isNot(MIToken::comma))
2869       break;
2870     lex();
2871   }
2872   if (expectAndConsume(MIToken::rparen))
2873     return true;
2874   Dest = MachineOperand::CreateRegLiveOut(Mask);
2875   return false;
2876 }
2877
2878 bool MIParser::parseMachineOperand(const unsigned OpCode, const unsigned OpIdx,
2879                                    MachineOperand &Dest,
2880                                    std::optional<unsigned> &TiedDefIdx) {
2881   switch (Token.kind()) {
2882   case MIToken::kw_implicit:
2883   case MIToken::kw_implicit_define:
2884   case MIToken::kw_def:
2885   case MIToken::kw_dead:
2886   case MIToken::kw_killed:
2887   case MIToken::kw_undef:
2888   case MIToken::kw_internal:
2889   case MIToken::kw_early_clobber:
2890   case MIToken::kw_debug_use:
2891   case MIToken::kw_renamable:
2892   case MIToken::underscore:
2893   case MIToken::NamedRegister:
2894   case MIToken::VirtualRegister:
2895   case MIToken::NamedVirtualRegister:
2896     return parseRegisterOperand(Dest, TiedDefIdx);
2897   case MIToken::IntegerLiteral:
2898     return parseImmediateOperand(Dest);
2899   case MIToken::kw_half:
2900   case MIToken::kw_float:
2901   case MIToken::kw_double:
2902   case MIToken::kw_x86_fp80:
2903   case MIToken::kw_fp128:
2904   case MIToken::kw_ppc_fp128:
2905     return parseFPImmediateOperand(Dest);
2906   case MIToken::MachineBasicBlock:
2907     return parseMBBOperand(Dest);
2908   case MIToken::StackObject:
2909     return parseStackObjectOperand(Dest);
2910   case MIToken::FixedStackObject:
2911     return parseFixedStackObjectOperand(Dest);
2912   case MIToken::GlobalValue:
2913   case MIToken::NamedGlobalValue:
2914     return parseGlobalAddressOperand(Dest);
2915   case MIToken::ConstantPoolItem:
2916     return parseConstantPoolIndexOperand(Dest);
2917   case MIToken::JumpTableIndex:
2918     return parseJumpTableIndexOperand(Dest);
2919   case MIToken::ExternalSymbol:
2920     return parseExternalSymbolOperand(Dest);
2921   case MIToken::MCSymbol:
2922     return parseMCSymbolOperand(Dest);
2923   case MIToken::SubRegisterIndex:
2924     return parseSubRegisterIndexOperand(Dest);
2925   case MIToken::md_diexpr:
2926   case MIToken::exclaim:
2927     return parseMetadataOperand(Dest);
2928   case MIToken::kw_cfi_same_value:
2929   case MIToken::kw_cfi_offset:
2930   case MIToken::kw_cfi_rel_offset:
2931   case MIToken::kw_cfi_def_cfa_register:
2932   case MIToken::kw_cfi_def_cfa_offset:
2933   case MIToken::kw_cfi_adjust_cfa_offset:
2934   case MIToken::kw_cfi_escape:
2935   case MIToken::kw_cfi_def_cfa:
2936   case MIToken::kw_cfi_llvm_def_aspace_cfa:
2937   case MIToken::kw_cfi_register:
2938   case MIToken::kw_cfi_remember_state:
2939   case MIToken::kw_cfi_restore:
2940   case MIToken::kw_cfi_restore_state:
2941   case MIToken::kw_cfi_undefined:
2942   case MIToken::kw_cfi_window_save:
2943   case MIToken::kw_cfi_aarch64_negate_ra_sign_state:
2944     return parseCFIOperand(Dest);
2945   case MIToken::kw_blockaddress:
2946     return parseBlockAddressOperand(Dest);
2947   case MIToken::kw_intrinsic:
2948     return parseIntrinsicOperand(Dest);
2949   case MIToken::kw_target_index:
2950     return parseTargetIndexOperand(Dest);
2951   case MIToken::kw_liveout:
2952     return parseLiveoutRegisterMaskOperand(Dest);
2953   case MIToken::kw_floatpred:
2954   case MIToken::kw_intpred:
2955     return parsePredicateOperand(Dest);
2956   case MIToken::kw_shufflemask:
2957     return parseShuffleMaskOperand(Dest);
2958   case MIToken::kw_dbg_instr_ref:
2959     return parseDbgInstrRefOperand(Dest);
2960   case MIToken::Error:
2961     return true;
2962   case MIToken::Identifier:
2963     if (const auto *RegMask = PFS.Target.getRegMask(Token.stringValue())) {
2964       Dest = MachineOperand::CreateRegMask(RegMask);
2965       lex();
2966       break;
2967     } else if (Token.stringValue() == "CustomRegMask") {
2968       return parseCustomRegisterMaskOperand(Dest);
2969     } else
2970       return parseTypedImmediateOperand(Dest);
2971   case MIToken::dot: {
2972     const auto *TII = MF.getSubtarget().getInstrInfo();
2973     if (const auto *Formatter = TII->getMIRFormatter()) {
2974       return parseTargetImmMnemonic(OpCode, OpIdx, Dest, *Formatter);
2975     }
2976     [[fallthrough]];
2977   }
2978   default:
2979     // FIXME: Parse the MCSymbol machine operand.
2980     return error("expected a machine operand");
2981   }
2982   return false;
2983 }
2984
2985 bool MIParser::parseMachineOperandAndTargetFlags(
2986     const unsigned OpCode, const unsigned OpIdx, MachineOperand &Dest,
2987     std::optional<unsigned> &TiedDefIdx) {
2988   unsigned TF = 0;
2989   bool HasTargetFlags = false;
2990   if (Token.is(MIToken::kw_target_flags)) {
2991     HasTargetFlags = true;
2992     lex();
2993     if (expectAndConsume(MIToken::lparen))
2994       return true;
2995     if (Token.isNot(MIToken::Identifier))
2996       return error("expected the name of the target flag");
2997     if (PFS.Target.getDirectTargetFlag(Token.stringValue(), TF)) {
2998       if (PFS.Target.getBitmaskTargetFlag(Token.stringValue(), TF))
2999         return error("use of undefined target flag '" + Token.stringValue() +
3000                      "'");
3001     }
3002     lex();
3003     while (Token.is(MIToken::comma)) {
3004       lex();
3005       if (Token.isNot(MIToken::Identifier))
3006         return error("expected the name of the target flag");
3007       unsigned BitFlag = 0;
3008       if (PFS.Target.getBitmaskTargetFlag(Token.stringValue(), BitFlag))
3009         return error("use of undefined target flag '" + Token.stringValue() +
3010                      "'");
3011       // TODO: Report an error when using a duplicate bit target flag.
3012       TF |= BitFlag;
3013       lex();
3014     }
3015     if (expectAndConsume(MIToken::rparen))
3016       return true;
3017   }
3018   auto Loc = Token.location();
3019   if (parseMachineOperand(OpCode, OpIdx, Dest, TiedDefIdx))
3020     return true;
3021   if (!HasTargetFlags)
3022     return false;
3023   if (Dest.isReg())
3024     return error(Loc, "register operands can't have target flags");
3025   Dest.setTargetFlags(TF);
3026   return false;
3027 }
3028
3029 bool MIParser::parseOffset(int64_t &Offset) {
3030   if (Token.isNot(MIToken::plus) && Token.isNot(MIToken::minus))
3031     return false;
3032   StringRef Sign = Token.range();
3033   bool IsNegative = Token.is(MIToken::minus);
3034   lex();
3035   if (Token.isNot(MIToken::IntegerLiteral))
3036     return error("expected an integer literal after '" + Sign + "'");
3037   if (Token.integerValue().getSignificantBits() > 64)
3038     return error("expected 64-bit integer (too large)");
3039   Offset = Token.integerValue().getExtValue();
3040   if (IsNegative)
3041     Offset = -Offset;
3042   lex();
3043   return false;
3044 }
3045
3046 bool MIParser::parseIRBlockAddressTaken(BasicBlock *&BB) {
3047   assert(Token.is(MIToken::kw_ir_block_address_taken));
3048   lex();
3049   if (Token.isNot(MIToken::IRBlock) && Token.isNot(MIToken::NamedIRBlock))
3050     return error("expected basic block after 'ir_block_address_taken'");
3051
3052   if (parseIRBlock(BB, MF.getFunction()))
3053     return true;
3054
3055   lex();
3056   return false;
3057 }
3058
3059 bool MIParser::parseAlignment(uint64_t &Alignment) {
3060   assert(Token.is(MIToken::kw_align) || Token.is(MIToken::kw_basealign));
3061   lex();
3062   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
3063     return error("expected an integer literal after 'align'");
3064   if (getUint64(Alignment))
3065     return true;
3066   lex();
3067
3068   if (!isPowerOf2_64(Alignment))
3069     return error("expected a power-of-2 literal after 'align'");
3070
3071   return false;
3072 }
3073
3074 bool MIParser::parseAddrspace(unsigned &Addrspace) {
3075   assert(Token.is(MIToken::kw_addrspace));
3076   lex();
3077   if (Token.isNot(MIToken::IntegerLiteral) || Token.integerValue().isSigned())
3078     return error("expected an integer literal after 'addrspace'");
3079   if (getUnsigned(Addrspace))
3080     return true;
3081   lex();
3082   return false;
3083 }
3084
3085 bool MIParser::parseOperandsOffset(MachineOperand &Op) {
3086   int64_t Offset = 0;
3087   if (parseOffset(Offset))
3088     return true;
3089   Op.setOffset(Offset);
3090   return false;
3091 }
3092
3093 static bool parseIRValue(const MIToken &Token, PerFunctionMIParsingState &PFS,
3094                          const Value *&V, ErrorCallbackType ErrCB) {
3095   switch (Token.kind()) {
3096   case MIToken::NamedIRValue: {
3097     V = PFS.MF.getFunction().getValueSymbolTable()->lookup(Token.stringValue());
3098     break;
3099   }
3100   case MIToken::IRValue: {
3101     unsigned SlotNumber = 0;
3102     if (getUnsigned(Token, SlotNumber, ErrCB))
3103       return true;
3104     V = PFS.getIRValue(SlotNumber);
3105     break;
3106   }
3107   case MIToken::NamedGlobalValue:
3108   case MIToken::GlobalValue: {
3109     GlobalValue *GV = nullptr;
3110     if (parseGlobalValue(Token, PFS, GV, ErrCB))
3111       return true;
3112     V = GV;
3113     break;
3114   }
3115   case MIToken::QuotedIRValue: {
3116     const Constant *C = nullptr;
3117     if (parseIRConstant(Token.location(), Token.stringValue(), PFS, C, ErrCB))
3118       return true;
3119     V = C;
3120     break;
3121   }
3122   case MIToken::kw_unknown_address:
3123     V = nullptr;
3124     return false;
3125   default:
3126     llvm_unreachable("The current token should be an IR block reference");
3127   }
3128   if (!V)
3129     return ErrCB(Token.location(), Twine("use of undefined IR value '") + Token.range() + "'");
3130   return false;
3131 }
3132
3133 bool MIParser::parseIRValue(const Value *&V) {
3134   return ::parseIRValue(
3135       Token, PFS, V, [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
3136         return error(Loc, Msg);
3137       });
3138 }
3139
3140 bool MIParser::getUint64(uint64_t &Result) {
3141   if (Token.hasIntegerValue()) {
3142     if (Token.integerValue().getActiveBits() > 64)
3143       return error("expected 64-bit integer (too large)");
3144     Result = Token.integerValue().getZExtValue();
3145     return false;
3146   }
3147   if (Token.is(MIToken::HexLiteral)) {
3148     APInt A;
3149     if (getHexUint(A))
3150       return true;
3151     if (A.getBitWidth() > 64)
3152       return error("expected 64-bit integer (too large)");
3153     Result = A.getZExtValue();
3154     return false;
3155   }
3156   return true;
3157 }
3158
3159 bool MIParser::getHexUint(APInt &Result) {
3160   return ::getHexUint(Token, Result);
3161 }
3162
3163 bool MIParser::parseMemoryOperandFlag(MachineMemOperand::Flags &Flags) {
3164   const auto OldFlags = Flags;
3165   switch (Token.kind()) {
3166   case MIToken::kw_volatile:
3167     Flags |= MachineMemOperand::MOVolatile;
3168     break;
3169   case MIToken::kw_non_temporal:
3170     Flags |= MachineMemOperand::MONonTemporal;
3171     break;
3172   case MIToken::kw_dereferenceable:
3173     Flags |= MachineMemOperand::MODereferenceable;
3174     break;
3175   case MIToken::kw_invariant:
3176     Flags |= MachineMemOperand::MOInvariant;
3177     break;
3178   case MIToken::StringConstant: {
3179     MachineMemOperand::Flags TF;
3180     if (PFS.Target.getMMOTargetFlag(Token.stringValue(), TF))
3181       return error("use of undefined target MMO flag '" + Token.stringValue() +
3182                    "'");
3183     Flags |= TF;
3184     break;
3185   }
3186   default:
3187     llvm_unreachable("The current token should be a memory operand flag");
3188   }
3189   if (OldFlags == Flags)
3190     // We know that the same flag is specified more than once when the flags
3191     // weren't modified.
3192     return error("duplicate '" + Token.stringValue() + "' memory operand flag");
3193   lex();
3194   return false;
3195 }
3196
3197 bool MIParser::parseMemoryPseudoSourceValue(const PseudoSourceValue *&PSV) {
3198   switch (Token.kind()) {
3199   case MIToken::kw_stack:
3200     PSV = MF.getPSVManager().getStack();
3201     break;
3202   case MIToken::kw_got:
3203     PSV = MF.getPSVManager().getGOT();
3204     break;
3205   case MIToken::kw_jump_table:
3206     PSV = MF.getPSVManager().getJumpTable();
3207     break;
3208   case MIToken::kw_constant_pool:
3209     PSV = MF.getPSVManager().getConstantPool();
3210     break;
3211   case MIToken::FixedStackObject: {
3212     int FI;
3213     if (parseFixedStackFrameIndex(FI))
3214       return true;
3215     PSV = MF.getPSVManager().getFixedStack(FI);
3216     // The token was already consumed, so use return here instead of break.
3217     return false;
3218   }
3219   case MIToken::StackObject: {
3220     int FI;
3221     if (parseStackFrameIndex(FI))
3222       return true;
3223     PSV = MF.getPSVManager().getFixedStack(FI);
3224     // The token was already consumed, so use return here instead of break.
3225     return false;
3226   }
3227   case MIToken::kw_call_entry:
3228     lex();
3229     switch (Token.kind()) {
3230     case MIToken::GlobalValue:
3231     case MIToken::NamedGlobalValue: {
3232       GlobalValue *GV = nullptr;
3233       if (parseGlobalValue(GV))
3234         return true;
3235       PSV = MF.getPSVManager().getGlobalValueCallEntry(GV);
3236       break;
3237     }
3238     case MIToken::ExternalSymbol:
3239       PSV = MF.getPSVManager().getExternalSymbolCallEntry(
3240           MF.createExternalSymbolName(Token.stringValue()));
3241       break;
3242     default:
3243       return error(
3244           "expected a global value or an external symbol after 'call-entry'");
3245     }
3246     break;
3247   case MIToken::kw_custom: {
3248     lex();
3249     const auto *TII = MF.getSubtarget().getInstrInfo();
3250     if (const auto *Formatter = TII->getMIRFormatter()) {
3251       if (Formatter->parseCustomPseudoSourceValue(
3252               Token.stringValue(), MF, PFS, PSV,
3253               [this](StringRef::iterator Loc, const Twine &Msg) -> bool {
3254                 return error(Loc, Msg);
3255               }))
3256         return true;
3257     } else
3258       return error("unable to parse target custom pseudo source value");
3259     break;
3260   }
3261   default:
3262     llvm_unreachable("The current token should be pseudo source value");
3263   }
3264   lex();
3265   return false;
3266 }
3267
3268 bool MIParser::parseMachinePointerInfo(MachinePointerInfo &Dest) {
3269   if (Token.is(MIToken::kw_constant_pool) || Token.is(MIToken::kw_stack) ||
3270       Token.is(MIToken::kw_got) || Token.is(MIToken::kw_jump_table) ||
3271       Token.is(MIToken::FixedStackObject) || Token.is(MIToken::StackObject) ||
3272       Token.is(MIToken::kw_call_entry) || Token.is(MIToken::kw_custom)) {
3273     const PseudoSourceValue *PSV = nullptr;
3274     if (parseMemoryPseudoSourceValue(PSV))
3275       return true;
3276     int64_t Offset = 0;
3277     if (parseOffset(Offset))
3278       return true;
3279     Dest = MachinePointerInfo(PSV, Offset);
3280     return false;
3281   }
3282   if (Token.isNot(MIToken::NamedIRValue) && Token.isNot(MIToken::IRValue) &&
3283       Token.isNot(MIToken::GlobalValue) &&
3284       Token.isNot(MIToken::NamedGlobalValue) &&
3285       Token.isNot(MIToken::QuotedIRValue) &&
3286       Token.isNot(MIToken::kw_unknown_address))
3287     return error("expected an IR value reference");
3288   const Value *V = nullptr;
3289   if (parseIRValue(V))
3290     return true;
3291   if (V && !V->getType()->isPointerTy())
3292     return error("expected a pointer IR value");
3293   lex();
3294   int64_t Offset = 0;
3295   if (parseOffset(Offset))
3296     return true;
3297   Dest = MachinePointerInfo(V, Offset);
3298   return false;
3299 }
3300
3301 bool MIParser::parseOptionalScope(LLVMContext &Context,
3302                                   SyncScope::ID &SSID) {
3303   SSID = SyncScope::System;
3304   if (Token.is(MIToken::Identifier) && Token.stringValue() == "syncscope") {
3305     lex();
3306     if (expectAndConsume(MIToken::lparen))
3307       return error("expected '(' in syncscope");
3308
3309     std::string SSN;
3310     if (parseStringConstant(SSN))
3311       return true;
3312
3313     SSID = Context.getOrInsertSyncScopeID(SSN);
3314     if (expectAndConsume(MIToken::rparen))
3315       return error("expected ')' in syncscope");
3316   }
3317
3318   return false;
3319 }
3320
3321 bool MIParser::parseOptionalAtomicOrdering(AtomicOrdering &Order) {
3322   Order = AtomicOrdering::NotAtomic;
3323   if (Token.isNot(MIToken::Identifier))
3324     return false;
3325
3326   Order = StringSwitch<AtomicOrdering>(Token.stringValue())
3327               .Case("unordered", AtomicOrdering::Unordered)
3328               .Case("monotonic", AtomicOrdering::Monotonic)
3329               .Case("acquire", AtomicOrdering::Acquire)
3330               .Case("release", AtomicOrdering::Release)
3331               .Case("acq_rel", AtomicOrdering::AcquireRelease)
3332               .Case("seq_cst", AtomicOrdering::SequentiallyConsistent)
3333               .Default(AtomicOrdering::NotAtomic);
3334
3335   if (Order != AtomicOrdering::NotAtomic) {
3336     lex();
3337     return false;
3338   }
3339
3340   return error("expected an atomic scope, ordering or a size specification");
3341 }
3342
3343 bool MIParser::parseMachineMemoryOperand(MachineMemOperand *&Dest) {
3344   if (expectAndConsume(MIToken::lparen))
3345     return true;
3346   MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
3347   while (Token.isMemoryOperandFlag()) {
3348     if (parseMemoryOperandFlag(Flags))
3349       return true;
3350   }
3351   if (Token.isNot(MIToken::Identifier) ||
3352       (Token.stringValue() != "load" && Token.stringValue() != "store"))
3353     return error("expected 'load' or 'store' memory operation");
3354   if (Token.stringValue() == "load")
3355     Flags |= MachineMemOperand::MOLoad;
3356   else
3357     Flags |= MachineMemOperand::MOStore;
3358   lex();
3359
3360   // Optional 'store' for operands that both load and store.
3361   if (Token.is(MIToken::Identifier) && Token.stringValue() == "store") {
3362     Flags |= MachineMemOperand::MOStore;
3363     lex();
3364   }
3365
3366   // Optional synchronization scope.
3367   SyncScope::ID SSID;
3368   if (parseOptionalScope(MF.getFunction().getContext(), SSID))
3369     return true;
3370
3371   // Up to two atomic orderings (cmpxchg provides guarantees on failure).
3372   AtomicOrdering Order, FailureOrder;
3373   if (parseOptionalAtomicOrdering(Order))
3374     return true;
3375
3376   if (parseOptionalAtomicOrdering(FailureOrder))
3377     return true;
3378
3379   LLT MemoryType;
3380   if (Token.isNot(MIToken::IntegerLiteral) &&
3381       Token.isNot(MIToken::kw_unknown_size) &&
3382       Token.isNot(MIToken::lparen))
3383     return error("expected memory LLT, the size integer literal or 'unknown-size' after "
3384                  "memory operation");
3385
3386   uint64_t Size = MemoryLocation::UnknownSize;
3387   if (Token.is(MIToken::IntegerLiteral)) {
3388     if (getUint64(Size))
3389       return true;
3390
3391     // Convert from bytes to bits for storage.
3392     MemoryType = LLT::scalar(8 * Size);
3393     lex();
3394   } else if (Token.is(MIToken::kw_unknown_size)) {
3395     Size = MemoryLocation::UnknownSize;
3396     lex();
3397   } else {
3398     if (expectAndConsume(MIToken::lparen))
3399       return true;
3400     if (parseLowLevelType(Token.location(), MemoryType))
3401       return true;
3402     if (expectAndConsume(MIToken::rparen))
3403       return true;
3404
3405     Size = MemoryType.getSizeInBytes();
3406   }
3407
3408   MachinePointerInfo Ptr = MachinePointerInfo();
3409   if (Token.is(MIToken::Identifier)) {
3410     const char *Word =
3411         ((Flags & MachineMemOperand::MOLoad) &&
3412          (Flags & MachineMemOperand::MOStore))
3413             ? "on"
3414             : Flags & MachineMemOperand::MOLoad ? "from" : "into";
3415     if (Token.stringValue() != Word)
3416       return error(Twine("expected '") + Word + "'");
3417     lex();
3418
3419     if (parseMachinePointerInfo(Ptr))
3420       return true;
3421   }
3422   uint64_t BaseAlignment =
3423       (Size != MemoryLocation::UnknownSize ? PowerOf2Ceil(Size) : 1);
3424   AAMDNodes AAInfo;
3425   MDNode *Range = nullptr;
3426   while (consumeIfPresent(MIToken::comma)) {
3427     switch (Token.kind()) {
3428     case MIToken::kw_align: {
3429       // align is printed if it is different than size.
3430       uint64_t Alignment;
3431       if (parseAlignment(Alignment))
3432         return true;
3433       if (Ptr.Offset & (Alignment - 1)) {
3434         // MachineMemOperand::getAlign never returns a value greater than the
3435         // alignment of offset, so this just guards against hand-written MIR
3436         // that specifies a large "align" value when it should probably use
3437         // "basealign" instead.
3438         return error("specified alignment is more aligned than offset");
3439       }
3440       BaseAlignment = Alignment;
3441       break;
3442     }
3443     case MIToken::kw_basealign:
3444       // basealign is printed if it is different than align.
3445       if (parseAlignment(BaseAlignment))
3446         return true;
3447       break;
3448     case MIToken::kw_addrspace:
3449       if (parseAddrspace(Ptr.AddrSpace))
3450         return true;
3451       break;
3452     case MIToken::md_tbaa:
3453       lex();
3454       if (parseMDNode(AAInfo.TBAA))
3455         return true;
3456       break;
3457     case MIToken::md_alias_scope:
3458       lex();
3459       if (parseMDNode(AAInfo.Scope))
3460         return true;
3461       break;
3462     case MIToken::md_noalias:
3463       lex();
3464       if (parseMDNode(AAInfo.NoAlias))
3465         return true;
3466       break;
3467     case MIToken::md_range:
3468       lex();
3469       if (parseMDNode(Range))
3470         return true;
3471       break;
3472     // TODO: Report an error on duplicate metadata nodes.
3473     default:
3474       return error("expected 'align' or '!tbaa' or '!alias.scope' or "
3475                    "'!noalias' or '!range'");
3476     }
3477   }
3478   if (expectAndConsume(MIToken::rparen))
3479     return true;
3480   Dest = MF.getMachineMemOperand(Ptr, Flags, MemoryType, Align(BaseAlignment),
3481                                  AAInfo, Range, SSID, Order, FailureOrder);
3482   return false;
3483 }
3484
3485 bool MIParser::parsePreOrPostInstrSymbol(MCSymbol *&Symbol) {
3486   assert((Token.is(MIToken::kw_pre_instr_symbol) ||
3487           Token.is(MIToken::kw_post_instr_symbol)) &&
3488          "Invalid token for a pre- post-instruction symbol!");
3489   lex();
3490   if (Token.isNot(MIToken::MCSymbol))
3491     return error("expected a symbol after 'pre-instr-symbol'");
3492   Symbol = getOrCreateMCSymbol(Token.stringValue());
3493   lex();
3494   if (Token.isNewlineOrEOF() || Token.is(MIToken::coloncolon) ||
3495       Token.is(MIToken::lbrace))
3496     return false;
3497   if (Token.isNot(MIToken::comma))
3498     return error("expected ',' before the next machine operand");
3499   lex();
3500   return false;
3501 }
3502
3503 bool MIParser::parseHeapAllocMarker(MDNode *&Node) {
3504   assert(Token.is(MIToken::kw_heap_alloc_marker) &&
3505          "Invalid token for a heap alloc marker!");
3506   lex();
3507   if (parseMDNode(Node))
3508     return true;
3509   if (!Node)
3510     return error("expected a MDNode after 'heap-alloc-marker'");
3511   if (Token.isNewlineOrEOF() || Token.is(MIToken::coloncolon) ||
3512       Token.is(MIToken::lbrace))
3513     return false;
3514   if (Token.isNot(MIToken::comma))
3515     return error("expected ',' before the next machine operand");
3516   lex();
3517   return false;
3518 }
3519
3520 bool MIParser::parsePCSections(MDNode *&Node) {
3521   assert(Token.is(MIToken::kw_pcsections) &&
3522          "Invalid token for a PC sections!");
3523   lex();
3524   if (parseMDNode(Node))
3525     return true;
3526   if (!Node)
3527     return error("expected a MDNode after 'pcsections'");
3528   if (Token.isNewlineOrEOF() || Token.is(MIToken::coloncolon) ||
3529       Token.is(MIToken::lbrace))
3530     return false;
3531   if (Token.isNot(MIToken::comma))
3532     return error("expected ',' before the next machine operand");
3533   lex();
3534   return false;
3535 }
3536
3537 static void initSlots2BasicBlocks(
3538     const Function &F,
3539     DenseMap<unsigned, const BasicBlock *> &Slots2BasicBlocks) {
3540   ModuleSlotTracker MST(F.getParent(), /*ShouldInitializeAllMetadata=*/false);
3541   MST.incorporateFunction(F);
3542   for (const auto &BB : F) {
3543     if (BB.hasName())
3544       continue;
3545     int Slot = MST.getLocalSlot(&BB);
3546     if (Slot == -1)
3547       continue;
3548     Slots2BasicBlocks.insert(std::make_pair(unsigned(Slot), &BB));
3549   }
3550 }
3551
3552 static const BasicBlock *getIRBlockFromSlot(
3553     unsigned Slot,
3554     const DenseMap<unsigned, const BasicBlock *> &Slots2BasicBlocks) {
3555   return Slots2BasicBlocks.lookup(Slot);
3556 }
3557
3558 const BasicBlock *MIParser::getIRBlock(unsigned Slot) {
3559   if (Slots2BasicBlocks.empty())
3560     initSlots2BasicBlocks(MF.getFunction(), Slots2BasicBlocks);
3561   return getIRBlockFromSlot(Slot, Slots2BasicBlocks);
3562 }
3563
3564 const BasicBlock *MIParser::getIRBlock(unsigned Slot, const Function &F) {
3565   if (&F == &MF.getFunction())
3566     return getIRBlock(Slot);
3567   DenseMap<unsigned, const BasicBlock *> CustomSlots2BasicBlocks;
3568   initSlots2BasicBlocks(F, CustomSlots2BasicBlocks);
3569   return getIRBlockFromSlot(Slot, CustomSlots2BasicBlocks);
3570 }
3571
3572 MCSymbol *MIParser::getOrCreateMCSymbol(StringRef Name) {
3573   // FIXME: Currently we can't recognize temporary or local symbols and call all
3574   // of the appropriate forms to create them. However, this handles basic cases
3575   // well as most of the special aspects are recognized by a prefix on their
3576   // name, and the input names should already be unique. For test cases, keeping
3577   // the symbol name out of the symbol table isn't terribly important.
3578   return MF.getContext().getOrCreateSymbol(Name);
3579 }
3580
3581 bool MIParser::parseStringConstant(std::string &Result) {
3582   if (Token.isNot(MIToken::StringConstant))
3583     return error("expected string constant");
3584   Result = std::string(Token.stringValue());
3585   lex();
3586   return false;
3587 }
3588
3589 bool llvm::parseMachineBasicBlockDefinitions(PerFunctionMIParsingState &PFS,
3590                                              StringRef Src,
3591                                              SMDiagnostic &Error) {
3592   return MIParser(PFS, Error, Src).parseBasicBlockDefinitions(PFS.MBBSlots);
3593 }
3594
3595 bool llvm::parseMachineInstructions(PerFunctionMIParsingState &PFS,
3596                                     StringRef Src, SMDiagnostic &Error) {
3597   return MIParser(PFS, Error, Src).parseBasicBlocks();
3598 }
3599
3600 bool llvm::parseMBBReference(PerFunctionMIParsingState &PFS,
3601                              MachineBasicBlock *&MBB, StringRef Src,
3602                              SMDiagnostic &Error) {
3603   return MIParser(PFS, Error, Src).parseStandaloneMBB(MBB);
3604 }
3605
3606 bool llvm::parseRegisterReference(PerFunctionMIParsingState &PFS,
3607                                   Register &Reg, StringRef Src,
3608                                   SMDiagnostic &Error) {
3609   return MIParser(PFS, Error, Src).parseStandaloneRegister(Reg);
3610 }
3611
3612 bool llvm::parseNamedRegisterReference(PerFunctionMIParsingState &PFS,
3613                                        Register &Reg, StringRef Src,
3614                                        SMDiagnostic &Error) {
3615   return MIParser(PFS, Error, Src).parseStandaloneNamedRegister(Reg);
3616 }
3617
3618 bool llvm::parseVirtualRegisterReference(PerFunctionMIParsingState &PFS,
3619                                          VRegInfo *&Info, StringRef Src,
3620                                          SMDiagnostic &Error) {
3621   return MIParser(PFS, Error, Src).parseStandaloneVirtualRegister(Info);
3622 }
3623
3624 bool llvm::parseStackObjectReference(PerFunctionMIParsingState &PFS,
3625                                      int &FI, StringRef Src,
3626                                      SMDiagnostic &Error) {
3627   return MIParser(PFS, Error, Src).parseStandaloneStackObject(FI);
3628 }
3629
3630 bool llvm::parseMDNode(PerFunctionMIParsingState &PFS,
3631                        MDNode *&Node, StringRef Src, SMDiagnostic &Error) {
3632   return MIParser(PFS, Error, Src).parseStandaloneMDNode(Node);
3633 }
3634
3635 bool llvm::parseMachineMetadata(PerFunctionMIParsingState &PFS, StringRef Src,
3636                                 SMRange SrcRange, SMDiagnostic &Error) {
3637   return MIParser(PFS, Error, Src, SrcRange).parseMachineMetadata();
3638 }
3639
3640 bool MIRFormatter::parseIRValue(StringRef Src, MachineFunction &MF,
3641                                 PerFunctionMIParsingState &PFS, const Value *&V,
3642                                 ErrorCallbackType ErrorCallback) {
3643   MIToken Token;
3644   Src = lexMIToken(Src, Token, [&](StringRef::iterator Loc, const Twine &Msg) {
3645     ErrorCallback(Loc, Msg);
3646   });
3647   V = nullptr;
3648
3649   return ::parseIRValue(Token, PFS, V, ErrorCallback);
3650 }