lib/Target/AArch64/AArch64AsmPrinter.cpp

   1 //===- AArch64AsmPrinter.cpp - AArch64 LLVM assembly writer ---------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a printer that converts from our internal representation
  10 // of machine-dependent LLVM code to the AArch64 assembly language.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64.h"
  15 #include "AArch64MCInstLower.h"
  16 #include "AArch64MachineFunctionInfo.h"
  17 #include "AArch64RegisterInfo.h"
  18 #include "AArch64Subtarget.h"
  19 #include "AArch64TargetObjectFile.h"
  20 #include "InstPrinter/AArch64InstPrinter.h"
  21 #include "MCTargetDesc/AArch64AddressingModes.h"
  22 #include "MCTargetDesc/AArch64MCTargetDesc.h"
  23 #include "MCTargetDesc/AArch64TargetStreamer.h"
  24 #include "Utils/AArch64BaseInfo.h"
  25 #include "llvm/ADT/SmallString.h"
  26 #include "llvm/ADT/SmallVector.h"
  27 #include "llvm/ADT/StringRef.h"
  28 #include "llvm/ADT/Triple.h"
  29 #include "llvm/ADT/Twine.h"
  30 #include "llvm/BinaryFormat/COFF.h"
  31 #include "llvm/BinaryFormat/ELF.h"
  32 #include "llvm/CodeGen/AsmPrinter.h"
  33 #include "llvm/CodeGen/MachineBasicBlock.h"
  34 #include "llvm/CodeGen/MachineFunction.h"
  35 #include "llvm/CodeGen/MachineInstr.h"
  36 #include "llvm/CodeGen/MachineJumpTableInfo.h"
  37 #include "llvm/CodeGen/MachineModuleInfoImpls.h"
  38 #include "llvm/CodeGen/MachineOperand.h"
  39 #include "llvm/CodeGen/StackMaps.h"
  40 #include "llvm/CodeGen/TargetRegisterInfo.h"
  41 #include "llvm/IR/DataLayout.h"
  42 #include "llvm/IR/DebugInfoMetadata.h"
  43 #include "llvm/MC/MCAsmInfo.h"
  44 #include "llvm/MC/MCContext.h"
  45 #include "llvm/MC/MCInst.h"
  46 #include "llvm/MC/MCInstBuilder.h"
  47 #include "llvm/MC/MCSectionELF.h"
  48 #include "llvm/MC/MCStreamer.h"
  49 #include "llvm/MC/MCSymbol.h"
  50 #include "llvm/Support/Casting.h"
  51 #include "llvm/Support/ErrorHandling.h"
  52 #include "llvm/Support/TargetRegistry.h"
  53 #include "llvm/Support/raw_ostream.h"
  54 #include "llvm/Target/TargetMachine.h"
  55 #include <algorithm>
  56 #include <cassert>
  57 #include <cstdint>
  58 #include <map>
  59 #include <memory>
  60
  61 using namespace llvm;
  62
  63 #define DEBUG_TYPE "asm-printer"
  64
  65 namespace {
  66
  67 class AArch64AsmPrinter : public AsmPrinter {
  68   AArch64MCInstLower MCInstLowering;
  69   StackMaps SM;
  70   const AArch64Subtarget *STI;
  71
  72 public:
  73   AArch64AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer)
  74       : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this),
  75         SM(*this) {}
  76
  77   StringRef getPassName() const override { return "AArch64 Assembly Printer"; }
  78
  79   /// Wrapper for MCInstLowering.lowerOperand() for the
  80   /// tblgen'erated pseudo lowering.
  81   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const {
  82     return MCInstLowering.lowerOperand(MO, MCOp);
  83   }
  84
  85   void EmitJumpTableInfo() override;
  86   void emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
  87                           const MachineBasicBlock *MBB, unsigned JTI);
  88
  89   void LowerJumpTableDestSmall(MCStreamer &OutStreamer, const MachineInstr &MI);
  90
  91   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
  92                      const MachineInstr &MI);
  93   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  94                        const MachineInstr &MI);
  95
  96   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI);
  97   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI);
  98   void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI);
  99
 100   std::map<std::pair<unsigned, uint32_t>, MCSymbol *> HwasanMemaccessSymbols;
 101   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
 102   void EmitHwasanMemaccessSymbols(Module &M);
 103
 104   void EmitSled(const MachineInstr &MI, SledKind Kind);
 105
 106   /// tblgen'erated driver function for lowering simple MI->MC
 107   /// pseudo instructions.
 108   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
 109                                    const MachineInstr *MI);
 110
 111   void EmitInstruction(const MachineInstr *MI) override;
 112
 113   void getAnalysisUsage(AnalysisUsage &AU) const override {
 114     AsmPrinter::getAnalysisUsage(AU);
 115     AU.setPreservesAll();
 116   }
 117
 118   bool runOnMachineFunction(MachineFunction &MF) override {
 119     AArch64FI = MF.getInfo<AArch64FunctionInfo>();
 120     STI = static_cast<const AArch64Subtarget*>(&MF.getSubtarget());
 121
 122     SetupMachineFunction(MF);
 123
 124     if (STI->isTargetCOFF()) {
 125       bool Internal = MF.getFunction().hasInternalLinkage();
 126       COFF::SymbolStorageClass Scl = Internal ? COFF::IMAGE_SYM_CLASS_STATIC
 127                                               : COFF::IMAGE_SYM_CLASS_EXTERNAL;
 128       int Type =
 129         COFF::IMAGE_SYM_DTYPE_FUNCTION << COFF::SCT_COMPLEX_TYPE_SHIFT;
 130
 131       OutStreamer->BeginCOFFSymbolDef(CurrentFnSym);
 132       OutStreamer->EmitCOFFSymbolStorageClass(Scl);
 133       OutStreamer->EmitCOFFSymbolType(Type);
 134       OutStreamer->EndCOFFSymbolDef();
 135     }
 136
 137     // Emit the rest of the function body.
 138     EmitFunctionBody();
 139
 140     // Emit the XRay table for this function.
 141     emitXRayTable();
 142
 143     // We didn't modify anything.
 144     return false;
 145   }
 146
 147 private:
 148   void printOperand(const MachineInstr *MI, unsigned OpNum, raw_ostream &O);
 149   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
 150   bool printAsmRegInClass(const MachineOperand &MO,
 151                           const TargetRegisterClass *RC, bool isVector,
 152                           raw_ostream &O);
 153
 154   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 155                        unsigned AsmVariant, const char *ExtraCode,
 156                        raw_ostream &O) override;
 157   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
 158                              unsigned AsmVariant, const char *ExtraCode,
 159                              raw_ostream &O) override;
 160
 161   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
 162
 163   void EmitFunctionBodyEnd() override;
 164
 165   MCSymbol *GetCPISymbol(unsigned CPID) const override;
 166   void EmitEndOfAsmFile(Module &M) override;
 167
 168   AArch64FunctionInfo *AArch64FI = nullptr;
 169
 170   /// Emit the LOHs contained in AArch64FI.
 171   void EmitLOHs();
 172
 173   /// Emit instruction to set float register to zero.
 174   void EmitFMov0(const MachineInstr &MI);
 175
 176   using MInstToMCSymbol = std::map<const MachineInstr *, MCSymbol *>;
 177
 178   MInstToMCSymbol LOHInstToLabel;
 179 };
 180
 181 } // end anonymous namespace
 182
 183 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI)
 184 {
 185   EmitSled(MI, SledKind::FUNCTION_ENTER);
 186 }
 187
 188 void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI)
 189 {
 190   EmitSled(MI, SledKind::FUNCTION_EXIT);
 191 }
 192
 193 void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI)
 194 {
 195   EmitSled(MI, SledKind::TAIL_CALL);
 196 }
 197
 198 void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind)
 199 {
 200   static const int8_t NoopsInSledCount = 7;
 201   // We want to emit the following pattern:
 202   //
 203   // .Lxray_sled_N:
 204   //   ALIGN
 205   //   B #32
 206   //   ; 7 NOP instructions (28 bytes)
 207   // .tmpN
 208   //
 209   // We need the 28 bytes (7 instructions) because at runtime, we'd be patching
 210   // over the full 32 bytes (8 instructions) with the following pattern:
 211   //
 212   //   STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack
 213   //   LDR W0, #12 ; W0 := function ID
 214   //   LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit
 215   //   BLR X16 ; call the tracing trampoline
 216   //   ;DATA: 32 bits of function ID
 217   //   ;DATA: lower 32 bits of the address of the trampoline
 218   //   ;DATA: higher 32 bits of the address of the trampoline
 219   //   LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack
 220   //
 221   OutStreamer->EmitCodeAlignment(4);
 222   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
 223   OutStreamer->EmitLabel(CurSled);
 224   auto Target = OutContext.createTempSymbol();
 225
 226   // Emit "B #32" instruction, which jumps over the next 28 bytes.
 227   // The operand has to be the number of 4-byte instructions to jump over,
 228   // including the current instruction.
 229   EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8));
 230
 231   for (int8_t I = 0; I < NoopsInSledCount; I++)
 232     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 233
 234   OutStreamer->EmitLabel(Target);
 235   recordSled(CurSled, MI, Kind);
 236 }
 237
 238 void AArch64AsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
 239   unsigned Reg = MI.getOperand(0).getReg();
 240   uint32_t AccessInfo = MI.getOperand(1).getImm();
 241   MCSymbol *&Sym = HwasanMemaccessSymbols[{Reg, AccessInfo}];
 242   if (!Sym) {
 243     // FIXME: Make this work on non-ELF.
 244     if (!TM.getTargetTriple().isOSBinFormatELF())
 245       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
 246
 247     std::string SymName = "__hwasan_check_x" + utostr(Reg - AArch64::X0) + "_" +
 248                           utostr(AccessInfo);
 249     Sym = OutContext.getOrCreateSymbol(SymName);
 250   }
 251
 252   EmitToStreamer(*OutStreamer,
 253                  MCInstBuilder(AArch64::BL)
 254                      .addExpr(MCSymbolRefExpr::create(Sym, OutContext)));
 255 }
 256
 257 void AArch64AsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
 258   if (HwasanMemaccessSymbols.empty())
 259     return;
 260
 261   const Triple &TT = TM.getTargetTriple();
 262   assert(TT.isOSBinFormatELF());
 263   std::unique_ptr<MCSubtargetInfo> STI(
 264       TM.getTarget().createMCSubtargetInfo(TT.str(), "", ""));
 265
 266   MCSymbol *HwasanTagMismatchSym =
 267       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch");
 268
 269   for (auto &P : HwasanMemaccessSymbols) {
 270     unsigned Reg = P.first.first;
 271     uint32_t AccessInfo = P.first.second;
 272     MCSymbol *Sym = P.second;
 273
 274     OutStreamer->SwitchSection(OutContext.getELFSection(
 275         ".text.hot", ELF::SHT_PROGBITS,
 276         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0,
 277         Sym->getName()));
 278
 279     OutStreamer->EmitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
 280     OutStreamer->EmitSymbolAttribute(Sym, MCSA_Weak);
 281     OutStreamer->EmitSymbolAttribute(Sym, MCSA_Hidden);
 282     OutStreamer->EmitLabel(Sym);
 283
 284     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
 285                                      .addReg(AArch64::X16)
 286                                      .addReg(Reg)
 287                                      .addImm(4)
 288                                      .addImm(55),
 289                                  *STI);
 290     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::LDRBBroX)
 291                                      .addReg(AArch64::W16)
 292                                      .addReg(AArch64::X9)
 293                                      .addReg(AArch64::X16)
 294                                      .addImm(0)
 295                                      .addImm(0),
 296                                  *STI);
 297     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::UBFMXri)
 298                                      .addReg(AArch64::X17)
 299                                      .addReg(Reg)
 300                                      .addImm(56)
 301                                      .addImm(63),
 302                                  *STI);
 303     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::SUBSWrs)
 304                                      .addReg(AArch64::WZR)
 305                                      .addReg(AArch64::W16)
 306                                      .addReg(AArch64::W17)
 307                                      .addImm(0),
 308                                  *STI);
 309     MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
 310     OutStreamer->EmitInstruction(
 311         MCInstBuilder(AArch64::Bcc)
 312             .addImm(AArch64CC::NE)
 313             .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 314         *STI);
 315     OutStreamer->EmitInstruction(
 316         MCInstBuilder(AArch64::RET).addReg(AArch64::LR), *STI);
 317
 318     OutStreamer->EmitLabel(HandleMismatchSym);
 319     if (Reg != AArch64::X0)
 320       OutStreamer->EmitInstruction(MCInstBuilder(AArch64::ORRXrs)
 321                                        .addReg(AArch64::X0)
 322                                        .addReg(AArch64::XZR)
 323                                        .addReg(Reg)
 324                                        .addImm(0),
 325                                    *STI);
 326     OutStreamer->EmitInstruction(MCInstBuilder(AArch64::MOVZXi)
 327                                      .addReg(AArch64::X1)
 328                                      .addImm(AccessInfo)
 329                                      .addImm(0),
 330                                  *STI);
 331     OutStreamer->EmitInstruction(
 332         MCInstBuilder(AArch64::B)
 333             .addExpr(MCSymbolRefExpr::create(HwasanTagMismatchSym, OutContext)),
 334         *STI);
 335   }
 336 }
 337
 338 void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) {
 339   EmitHwasanMemaccessSymbols(M);
 340
 341   const Triple &TT = TM.getTargetTriple();
 342   if (TT.isOSBinFormatMachO()) {
 343     // Funny Darwin hack: This flag tells the linker that no global symbols
 344     // contain code that falls through to other global symbols (e.g. the obvious
 345     // implementation of multiple entry points).  If this doesn't occur, the
 346     // linker can safely perform dead code stripping.  Since LLVM never
 347     // generates code that does this, it is always safe to set.
 348     OutStreamer->EmitAssemblerFlag(MCAF_SubsectionsViaSymbols);
 349     emitStackMaps(SM);
 350   }
 351 }
 352
 353 void AArch64AsmPrinter::EmitLOHs() {
 354   SmallVector<MCSymbol *, 3> MCArgs;
 355
 356   for (const auto &D : AArch64FI->getLOHContainer()) {
 357     for (const MachineInstr *MI : D.getArgs()) {
 358       MInstToMCSymbol::iterator LabelIt = LOHInstToLabel.find(MI);
 359       assert(LabelIt != LOHInstToLabel.end() &&
 360              "Label hasn't been inserted for LOH related instruction");
 361       MCArgs.push_back(LabelIt->second);
 362     }
 363     OutStreamer->EmitLOHDirective(D.getKind(), MCArgs);
 364     MCArgs.clear();
 365   }
 366 }
 367
 368 void AArch64AsmPrinter::EmitFunctionBodyEnd() {
 369   if (!AArch64FI->getLOHRelated().empty())
 370     EmitLOHs();
 371 }
 372
 373 /// GetCPISymbol - Return the symbol for the specified constant pool entry.
 374 MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const {
 375   // Darwin uses a linker-private symbol name for constant-pools (to
 376   // avoid addends on the relocation?), ELF has no such concept and
 377   // uses a normal private symbol.
 378   if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty())
 379     return OutContext.getOrCreateSymbol(
 380         Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" +
 381         Twine(getFunctionNumber()) + "_" + Twine(CPID));
 382
 383   return AsmPrinter::GetCPISymbol(CPID);
 384 }
 385
 386 void AArch64AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNum,
 387                                      raw_ostream &O) {
 388   const MachineOperand &MO = MI->getOperand(OpNum);
 389   switch (MO.getType()) {
 390   default:
 391     llvm_unreachable("<unknown operand type>");
 392   case MachineOperand::MO_Register: {
 393     unsigned Reg = MO.getReg();
 394     assert(TargetRegisterInfo::isPhysicalRegister(Reg));
 395     assert(!MO.getSubReg() && "Subregs should be eliminated!");
 396     O << AArch64InstPrinter::getRegisterName(Reg);
 397     break;
 398   }
 399   case MachineOperand::MO_Immediate: {
 400     int64_t Imm = MO.getImm();
 401     O << '#' << Imm;
 402     break;
 403   }
 404   case MachineOperand::MO_GlobalAddress: {
 405     const GlobalValue *GV = MO.getGlobal();
 406     MCSymbol *Sym = getSymbol(GV);
 407
 408     // FIXME: Can we get anything other than a plain symbol here?
 409     assert(!MO.getTargetFlags() && "Unknown operand target flag!");
 410
 411     Sym->print(O, MAI);
 412     printOffset(MO.getOffset(), O);
 413     break;
 414   }
 415   case MachineOperand::MO_BlockAddress: {
 416     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
 417     Sym->print(O, MAI);
 418     break;
 419   }
 420   }
 421 }
 422
 423 bool AArch64AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
 424                                           raw_ostream &O) {
 425   unsigned Reg = MO.getReg();
 426   switch (Mode) {
 427   default:
 428     return true; // Unknown mode.
 429   case 'w':
 430     Reg = getWRegFromXReg(Reg);
 431     break;
 432   case 'x':
 433     Reg = getXRegFromWReg(Reg);
 434     break;
 435   }
 436
 437   O << AArch64InstPrinter::getRegisterName(Reg);
 438   return false;
 439 }
 440
 441 // Prints the register in MO using class RC using the offset in the
 442 // new register class. This should not be used for cross class
 443 // printing.
 444 bool AArch64AsmPrinter::printAsmRegInClass(const MachineOperand &MO,
 445                                            const TargetRegisterClass *RC,
 446                                            bool isVector, raw_ostream &O) {
 447   assert(MO.isReg() && "Should only get here with a register!");
 448   const TargetRegisterInfo *RI = STI->getRegisterInfo();
 449   unsigned Reg = MO.getReg();
 450   unsigned RegToPrint = RC->getRegister(RI->getEncodingValue(Reg));
 451   assert(RI->regsOverlap(RegToPrint, Reg));
 452   O << AArch64InstPrinter::getRegisterName(
 453            RegToPrint, isVector ? AArch64::vreg : AArch64::NoRegAltName);
 454   return false;
 455 }
 456
 457 bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
 458                                         unsigned AsmVariant,
 459                                         const char *ExtraCode, raw_ostream &O) {
 460   const MachineOperand &MO = MI->getOperand(OpNum);
 461
 462   // First try the generic code, which knows about modifiers like 'c' and 'n'.
 463   if (!AsmPrinter::PrintAsmOperand(MI, OpNum, AsmVariant, ExtraCode, O))
 464     return false;
 465
 466   // Does this asm operand have a single letter operand modifier?
 467   if (ExtraCode && ExtraCode[0]) {
 468     if (ExtraCode[1] != 0)
 469       return true; // Unknown modifier.
 470
 471     switch (ExtraCode[0]) {
 472     default:
 473       return true; // Unknown modifier.
 474     case 'a':      // Print 'a' modifier
 475       PrintAsmMemoryOperand(MI, OpNum, AsmVariant, ExtraCode, O);
 476       return false;
 477     case 'w':      // Print W register
 478     case 'x':      // Print X register
 479       if (MO.isReg())
 480         return printAsmMRegister(MO, ExtraCode[0], O);
 481       if (MO.isImm() && MO.getImm() == 0) {
 482         unsigned Reg = ExtraCode[0] == 'w' ? AArch64::WZR : AArch64::XZR;
 483         O << AArch64InstPrinter::getRegisterName(Reg);
 484         return false;
 485       }
 486       printOperand(MI, OpNum, O);
 487       return false;
 488     case 'b': // Print B register.
 489     case 'h': // Print H register.
 490     case 's': // Print S register.
 491     case 'd': // Print D register.
 492     case 'q': // Print Q register.
 493       if (MO.isReg()) {
 494         const TargetRegisterClass *RC;
 495         switch (ExtraCode[0]) {
 496         case 'b':
 497           RC = &AArch64::FPR8RegClass;
 498           break;
 499         case 'h':
 500           RC = &AArch64::FPR16RegClass;
 501           break;
 502         case 's':
 503           RC = &AArch64::FPR32RegClass;
 504           break;
 505         case 'd':
 506           RC = &AArch64::FPR64RegClass;
 507           break;
 508         case 'q':
 509           RC = &AArch64::FPR128RegClass;
 510           break;
 511         default:
 512           return true;
 513         }
 514         return printAsmRegInClass(MO, RC, false /* vector */, O);
 515       }
 516       printOperand(MI, OpNum, O);
 517       return false;
 518     }
 519   }
 520
 521   // According to ARM, we should emit x and v registers unless we have a
 522   // modifier.
 523   if (MO.isReg()) {
 524     unsigned Reg = MO.getReg();
 525
 526     // If this is a w or x register, print an x register.
 527     if (AArch64::GPR32allRegClass.contains(Reg) ||
 528         AArch64::GPR64allRegClass.contains(Reg))
 529       return printAsmMRegister(MO, 'x', O);
 530
 531     // If this is a b, h, s, d, or q register, print it as a v register.
 532     return printAsmRegInClass(MO, &AArch64::FPR128RegClass, true /* vector */,
 533                               O);
 534   }
 535
 536   printOperand(MI, OpNum, O);
 537   return false;
 538 }
 539
 540 bool AArch64AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 541                                               unsigned OpNum,
 542                                               unsigned AsmVariant,
 543                                               const char *ExtraCode,
 544                                               raw_ostream &O) {
 545   if (ExtraCode && ExtraCode[0] && ExtraCode[0] != 'a')
 546     return true; // Unknown modifier.
 547
 548   const MachineOperand &MO = MI->getOperand(OpNum);
 549   assert(MO.isReg() && "unexpected inline asm memory operand");
 550   O << "[" << AArch64InstPrinter::getRegisterName(MO.getReg()) << "]";
 551   return false;
 552 }
 553
 554 void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI,
 555                                                raw_ostream &OS) {
 556   unsigned NOps = MI->getNumOperands();
 557   assert(NOps == 4);
 558   OS << '\t' << MAI->getCommentString() << "DEBUG_VALUE: ";
 559   // cast away const; DIetc do not take const operands for some reason.
 560   OS << cast<DILocalVariable>(MI->getOperand(NOps - 2).getMetadata())
 561             ->getName();
 562   OS << " <- ";
 563   // Frame address.  Currently handles register +- offset only.
 564   assert(MI->getOperand(0).isReg() && MI->getOperand(1).isImm());
 565   OS << '[';
 566   printOperand(MI, 0, OS);
 567   OS << '+';
 568   printOperand(MI, 1, OS);
 569   OS << ']';
 570   OS << "+";
 571   printOperand(MI, NOps - 2, OS);
 572 }
 573
 574 void AArch64AsmPrinter::EmitJumpTableInfo() {
 575   const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
 576   if (!MJTI) return;
 577
 578   const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
 579   if (JT.empty()) return;
 580
 581   const Function &F = MF->getFunction();
 582   const TargetLoweringObjectFile &TLOF = getObjFileLowering();
 583   bool JTInDiffSection =
 584       !STI->isTargetCOFF() ||
 585       !TLOF.shouldPutJumpTableInFunctionSection(
 586           MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32,
 587           F);
 588   if (JTInDiffSection) {
 589       // Drop it in the readonly section.
 590       MCSection *ReadOnlySec = TLOF.getSectionForJumpTable(F, TM);
 591       OutStreamer->SwitchSection(ReadOnlySec);
 592   }
 593
 594   auto AFI = MF->getInfo<AArch64FunctionInfo>();
 595   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
 596     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
 597
 598     // If this jump table was deleted, ignore it.
 599     if (JTBBs.empty()) continue;
 600
 601     unsigned Size = AFI->getJumpTableEntrySize(JTI);
 602     EmitAlignment(Log2_32(Size));
 603     OutStreamer->EmitLabel(GetJTISymbol(JTI));
 604
 605     for (auto *JTBB : JTBBs)
 606       emitJumpTableEntry(MJTI, JTBB, JTI);
 607   }
 608 }
 609
 610 void AArch64AsmPrinter::emitJumpTableEntry(const MachineJumpTableInfo *MJTI,
 611                                            const MachineBasicBlock *MBB,
 612                                            unsigned JTI) {
 613   const MCExpr *Value = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
 614   auto AFI = MF->getInfo<AArch64FunctionInfo>();
 615   unsigned Size = AFI->getJumpTableEntrySize(JTI);
 616
 617   if (Size == 4) {
 618     // .word LBB - LJTI
 619     const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
 620     const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
 621     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
 622   } else {
 623     // .byte (LBB - LBB) >> 2 (or .hword)
 624     const MCSymbol *BaseSym = AFI->getJumpTableEntryPCRelSymbol(JTI);
 625     const MCExpr *Base = MCSymbolRefExpr::create(BaseSym, OutContext);
 626     Value = MCBinaryExpr::createSub(Value, Base, OutContext);
 627     Value = MCBinaryExpr::createLShr(
 628         Value, MCConstantExpr::create(2, OutContext), OutContext);
 629   }
 630
 631   OutStreamer->EmitValue(Value, Size);
 632 }
 633
 634 /// Small jump tables contain an unsigned byte or half, representing the offset
 635 /// from the lowest-addressed possible destination to the desired basic
 636 /// block. Since all instructions are 4-byte aligned, this is further compressed
 637 /// by counting in instructions rather than bytes (i.e. divided by 4). So, to
 638 /// materialize the correct destination we need:
 639 ///
 640 ///             adr xDest, .LBB0_0
 641 ///             ldrb wScratch, [xTable, xEntry]   (with "lsl #1" for ldrh).
 642 ///             add xDest, xDest, xScratch, lsl #2
 643 void AArch64AsmPrinter::LowerJumpTableDestSmall(llvm::MCStreamer &OutStreamer,
 644                                                 const llvm::MachineInstr &MI) {
 645   unsigned DestReg = MI.getOperand(0).getReg();
 646   unsigned ScratchReg = MI.getOperand(1).getReg();
 647   unsigned ScratchRegW =
 648       STI->getRegisterInfo()->getSubReg(ScratchReg, AArch64::sub_32);
 649   unsigned TableReg = MI.getOperand(2).getReg();
 650   unsigned EntryReg = MI.getOperand(3).getReg();
 651   int JTIdx = MI.getOperand(4).getIndex();
 652   bool IsByteEntry = MI.getOpcode() == AArch64::JumpTableDest8;
 653
 654   // This has to be first because the compression pass based its reachability
 655   // calculations on the start of the JumpTableDest instruction.
 656   auto Label =
 657       MF->getInfo<AArch64FunctionInfo>()->getJumpTableEntryPCRelSymbol(JTIdx);
 658   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADR)
 659                                   .addReg(DestReg)
 660                                   .addExpr(MCSymbolRefExpr::create(
 661                                       Label, MF->getContext())));
 662
 663   // Load the number of instruction-steps to offset from the label.
 664   unsigned LdrOpcode = IsByteEntry ? AArch64::LDRBBroX : AArch64::LDRHHroX;
 665   EmitToStreamer(OutStreamer, MCInstBuilder(LdrOpcode)
 666                                   .addReg(ScratchRegW)
 667                                   .addReg(TableReg)
 668                                   .addReg(EntryReg)
 669                                   .addImm(0)
 670                                   .addImm(IsByteEntry ? 0 : 1));
 671
 672   // Multiply the steps by 4 and add to the already materialized base label
 673   // address.
 674   EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::ADDXrs)
 675                                   .addReg(DestReg)
 676                                   .addReg(DestReg)
 677                                   .addReg(ScratchReg)
 678                                   .addImm(2));
 679 }
 680
 681 void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
 682                                       const MachineInstr &MI) {
 683   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
 684
 685   SM.recordStackMap(MI);
 686   assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
 687
 688   // Scan ahead to trim the shadow.
 689   const MachineBasicBlock &MBB = *MI.getParent();
 690   MachineBasicBlock::const_iterator MII(MI);
 691   ++MII;
 692   while (NumNOPBytes > 0) {
 693     if (MII == MBB.end() || MII->isCall() ||
 694         MII->getOpcode() == AArch64::DBG_VALUE ||
 695         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
 696         MII->getOpcode() == TargetOpcode::STACKMAP)
 697       break;
 698     ++MII;
 699     NumNOPBytes -= 4;
 700   }
 701
 702   // Emit nops.
 703   for (unsigned i = 0; i < NumNOPBytes; i += 4)
 704     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 705 }
 706
 707 // Lower a patchpoint of the form:
 708 // [<def>], <id>, <numBytes>, <target>, <numArgs>
 709 void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 710                                         const MachineInstr &MI) {
 711   SM.recordPatchPoint(MI);
 712
 713   PatchPointOpers Opers(&MI);
 714
 715   int64_t CallTarget = Opers.getCallTarget().getImm();
 716   unsigned EncodedBytes = 0;
 717   if (CallTarget) {
 718     assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
 719            "High 16 bits of call target should be zero.");
 720     unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
 721     EncodedBytes = 16;
 722     // Materialize the jump address:
 723     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
 724                                     .addReg(ScratchReg)
 725                                     .addImm((CallTarget >> 32) & 0xFFFF)
 726                                     .addImm(32));
 727     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
 728                                     .addReg(ScratchReg)
 729                                     .addReg(ScratchReg)
 730                                     .addImm((CallTarget >> 16) & 0xFFFF)
 731                                     .addImm(16));
 732     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
 733                                     .addReg(ScratchReg)
 734                                     .addReg(ScratchReg)
 735                                     .addImm(CallTarget & 0xFFFF)
 736                                     .addImm(0));
 737     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
 738   }
 739   // Emit padding.
 740   unsigned NumBytes = Opers.getNumPatchBytes();
 741   assert(NumBytes >= EncodedBytes &&
 742          "Patchpoint can't request size less than the length of a call.");
 743   assert((NumBytes - EncodedBytes) % 4 == 0 &&
 744          "Invalid number of NOP bytes requested!");
 745   for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
 746     EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
 747 }
 748
 749 void AArch64AsmPrinter::EmitFMov0(const MachineInstr &MI) {
 750   unsigned DestReg = MI.getOperand(0).getReg();
 751   if (STI->hasZeroCycleZeroingFP() && !STI->hasZeroCycleZeroingFPWorkaround()) {
 752     // Convert H/S/D register to corresponding Q register
 753     if (AArch64::H0 <= DestReg && DestReg <= AArch64::H31)
 754       DestReg = AArch64::Q0 + (DestReg - AArch64::H0);
 755     else if (AArch64::S0 <= DestReg && DestReg <= AArch64::S31)
 756       DestReg = AArch64::Q0 + (DestReg - AArch64::S0);
 757     else {
 758       assert(AArch64::D0 <= DestReg && DestReg <= AArch64::D31);
 759       DestReg = AArch64::Q0 + (DestReg - AArch64::D0);
 760     }
 761     MCInst MOVI;
 762     MOVI.setOpcode(AArch64::MOVIv2d_ns);
 763     MOVI.addOperand(MCOperand::createReg(DestReg));
 764     MOVI.addOperand(MCOperand::createImm(0));
 765     EmitToStreamer(*OutStreamer, MOVI);
 766   } else {
 767     MCInst FMov;
 768     switch (MI.getOpcode()) {
 769     default: llvm_unreachable("Unexpected opcode");
 770     case AArch64::FMOVH0:
 771       FMov.setOpcode(AArch64::FMOVWHr);
 772       FMov.addOperand(MCOperand::createReg(DestReg));
 773       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
 774       break;
 775     case AArch64::FMOVS0:
 776       FMov.setOpcode(AArch64::FMOVWSr);
 777       FMov.addOperand(MCOperand::createReg(DestReg));
 778       FMov.addOperand(MCOperand::createReg(AArch64::WZR));
 779       break;
 780     case AArch64::FMOVD0:
 781       FMov.setOpcode(AArch64::FMOVXDr);
 782       FMov.addOperand(MCOperand::createReg(DestReg));
 783       FMov.addOperand(MCOperand::createReg(AArch64::XZR));
 784       break;
 785     }
 786     EmitToStreamer(*OutStreamer, FMov);
 787   }
 788 }
 789
 790 // Simple pseudo-instructions have their lowering (with expansion to real
 791 // instructions) auto-generated.
 792 #include "AArch64GenMCPseudoLowering.inc"
 793
 794 void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) {
 795   // Do any auto-generated pseudo lowerings.
 796   if (emitPseudoExpansionLowering(*OutStreamer, MI))
 797     return;
 798
 799   if (AArch64FI->getLOHRelated().count(MI)) {
 800     // Generate a label for LOH related instruction
 801     MCSymbol *LOHLabel = createTempSymbol("loh");
 802     // Associate the instruction with the label
 803     LOHInstToLabel[MI] = LOHLabel;
 804     OutStreamer->EmitLabel(LOHLabel);
 805   }
 806
 807   AArch64TargetStreamer *TS =
 808     static_cast<AArch64TargetStreamer *>(OutStreamer->getTargetStreamer());
 809   // Do any manual lowerings.
 810   switch (MI->getOpcode()) {
 811   default:
 812     break;
 813     case AArch64::MOVMCSym: {
 814     unsigned DestReg = MI->getOperand(0).getReg();
 815     const MachineOperand &MO_Sym = MI->getOperand(1);
 816     MachineOperand Hi_MOSym(MO_Sym), Lo_MOSym(MO_Sym);
 817     MCOperand Hi_MCSym, Lo_MCSym;
 818
 819     Hi_MOSym.setTargetFlags(AArch64II::MO_G1 | AArch64II::MO_S);
 820     Lo_MOSym.setTargetFlags(AArch64II::MO_G0 | AArch64II::MO_NC);
 821
 822     MCInstLowering.lowerOperand(Hi_MOSym, Hi_MCSym);
 823     MCInstLowering.lowerOperand(Lo_MOSym, Lo_MCSym);
 824
 825     MCInst MovZ;
 826     MovZ.setOpcode(AArch64::MOVZXi);
 827     MovZ.addOperand(MCOperand::createReg(DestReg));
 828     MovZ.addOperand(Hi_MCSym);
 829     MovZ.addOperand(MCOperand::createImm(16));
 830     EmitToStreamer(*OutStreamer, MovZ);
 831
 832     MCInst MovK;
 833     MovK.setOpcode(AArch64::MOVKXi);
 834     MovK.addOperand(MCOperand::createReg(DestReg));
 835     MovK.addOperand(MCOperand::createReg(DestReg));
 836     MovK.addOperand(Lo_MCSym);
 837     MovK.addOperand(MCOperand::createImm(0));
 838     EmitToStreamer(*OutStreamer, MovK);
 839     return;
 840   }
 841   case AArch64::MOVIv2d_ns:
 842     // If the target has <rdar://problem/16473581>, lower this
 843     // instruction to movi.16b instead.
 844     if (STI->hasZeroCycleZeroingFPWorkaround() &&
 845         MI->getOperand(1).getImm() == 0) {
 846       MCInst TmpInst;
 847       TmpInst.setOpcode(AArch64::MOVIv16b_ns);
 848       TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
 849       TmpInst.addOperand(MCOperand::createImm(MI->getOperand(1).getImm()));
 850       EmitToStreamer(*OutStreamer, TmpInst);
 851       return;
 852     }
 853     break;
 854
 855   case AArch64::DBG_VALUE: {
 856     if (isVerbose() && OutStreamer->hasRawTextSupport()) {
 857       SmallString<128> TmpStr;
 858       raw_svector_ostream OS(TmpStr);
 859       PrintDebugValueComment(MI, OS);
 860       OutStreamer->EmitRawText(StringRef(OS.str()));
 861     }
 862     return;
 863
 864   case AArch64::EMITBKEY: {
 865       ExceptionHandling ExceptionHandlingType = MAI->getExceptionHandlingType();
 866       if (ExceptionHandlingType != ExceptionHandling::DwarfCFI &&
 867           ExceptionHandlingType != ExceptionHandling::ARM)
 868         return;
 869
 870       if (needsCFIMoves() == CFI_M_None)
 871         return;
 872
 873       OutStreamer->EmitCFIBKeyFrame();
 874       return;
 875     }
 876   }
 877
 878   // Tail calls use pseudo instructions so they have the proper code-gen
 879   // attributes (isCall, isReturn, etc.). We lower them to the real
 880   // instruction here.
 881   case AArch64::TCRETURNri:
 882   case AArch64::TCRETURNriBTI:
 883   case AArch64::TCRETURNriALL: {
 884     MCInst TmpInst;
 885     TmpInst.setOpcode(AArch64::BR);
 886     TmpInst.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
 887     EmitToStreamer(*OutStreamer, TmpInst);
 888     return;
 889   }
 890   case AArch64::TCRETURNdi: {
 891     MCOperand Dest;
 892     MCInstLowering.lowerOperand(MI->getOperand(0), Dest);
 893     MCInst TmpInst;
 894     TmpInst.setOpcode(AArch64::B);
 895     TmpInst.addOperand(Dest);
 896     EmitToStreamer(*OutStreamer, TmpInst);
 897     return;
 898   }
 899   case AArch64::TLSDESC_CALLSEQ: {
 900     /// lower this to:
 901     ///    adrp  x0, :tlsdesc:var
 902     ///    ldr   x1, [x0, #:tlsdesc_lo12:var]
 903     ///    add   x0, x0, #:tlsdesc_lo12:var
 904     ///    .tlsdesccall var
 905     ///    blr   x1
 906     ///    (TPIDR_EL0 offset now in x0)
 907     const MachineOperand &MO_Sym = MI->getOperand(0);
 908     MachineOperand MO_TLSDESC_LO12(MO_Sym), MO_TLSDESC(MO_Sym);
 909     MCOperand Sym, SymTLSDescLo12, SymTLSDesc;
 910     MO_TLSDESC_LO12.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGEOFF);
 911     MO_TLSDESC.setTargetFlags(AArch64II::MO_TLS | AArch64II::MO_PAGE);
 912     MCInstLowering.lowerOperand(MO_Sym, Sym);
 913     MCInstLowering.lowerOperand(MO_TLSDESC_LO12, SymTLSDescLo12);
 914     MCInstLowering.lowerOperand(MO_TLSDESC, SymTLSDesc);
 915
 916     MCInst Adrp;
 917     Adrp.setOpcode(AArch64::ADRP);
 918     Adrp.addOperand(MCOperand::createReg(AArch64::X0));
 919     Adrp.addOperand(SymTLSDesc);
 920     EmitToStreamer(*OutStreamer, Adrp);
 921
 922     MCInst Ldr;
 923     Ldr.setOpcode(AArch64::LDRXui);
 924     Ldr.addOperand(MCOperand::createReg(AArch64::X1));
 925     Ldr.addOperand(MCOperand::createReg(AArch64::X0));
 926     Ldr.addOperand(SymTLSDescLo12);
 927     Ldr.addOperand(MCOperand::createImm(0));
 928     EmitToStreamer(*OutStreamer, Ldr);
 929
 930     MCInst Add;
 931     Add.setOpcode(AArch64::ADDXri);
 932     Add.addOperand(MCOperand::createReg(AArch64::X0));
 933     Add.addOperand(MCOperand::createReg(AArch64::X0));
 934     Add.addOperand(SymTLSDescLo12);
 935     Add.addOperand(MCOperand::createImm(AArch64_AM::getShiftValue(0)));
 936     EmitToStreamer(*OutStreamer, Add);
 937
 938     // Emit a relocation-annotation. This expands to no code, but requests
 939     // the following instruction gets an R_AARCH64_TLSDESC_CALL.
 940     MCInst TLSDescCall;
 941     TLSDescCall.setOpcode(AArch64::TLSDESCCALL);
 942     TLSDescCall.addOperand(Sym);
 943     EmitToStreamer(*OutStreamer, TLSDescCall);
 944
 945     MCInst Blr;
 946     Blr.setOpcode(AArch64::BLR);
 947     Blr.addOperand(MCOperand::createReg(AArch64::X1));
 948     EmitToStreamer(*OutStreamer, Blr);
 949
 950     return;
 951   }
 952
 953   case AArch64::JumpTableDest32: {
 954     // We want:
 955     //     ldrsw xScratch, [xTable, xEntry, lsl #2]
 956     //     add xDest, xTable, xScratch
 957     unsigned DestReg = MI->getOperand(0).getReg(),
 958              ScratchReg = MI->getOperand(1).getReg(),
 959              TableReg = MI->getOperand(2).getReg(),
 960              EntryReg = MI->getOperand(3).getReg();
 961     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::LDRSWroX)
 962                                      .addReg(ScratchReg)
 963                                      .addReg(TableReg)
 964                                      .addReg(EntryReg)
 965                                      .addImm(0)
 966                                      .addImm(1));
 967     EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::ADDXrs)
 968                                      .addReg(DestReg)
 969                                      .addReg(TableReg)
 970                                      .addReg(ScratchReg)
 971                                      .addImm(0));
 972     return;
 973   }
 974   case AArch64::JumpTableDest16:
 975   case AArch64::JumpTableDest8:
 976     LowerJumpTableDestSmall(*OutStreamer, *MI);
 977     return;
 978
 979   case AArch64::FMOVH0:
 980   case AArch64::FMOVS0:
 981   case AArch64::FMOVD0:
 982     EmitFMov0(*MI);
 983     return;
 984
 985   case TargetOpcode::STACKMAP:
 986     return LowerSTACKMAP(*OutStreamer, SM, *MI);
 987
 988   case TargetOpcode::PATCHPOINT:
 989     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
 990
 991   case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
 992     LowerPATCHABLE_FUNCTION_ENTER(*MI);
 993     return;
 994
 995   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
 996     LowerPATCHABLE_FUNCTION_EXIT(*MI);
 997     return;
 998
 999   case TargetOpcode::PATCHABLE_TAIL_CALL:
1000     LowerPATCHABLE_TAIL_CALL(*MI);
1001     return;
1002
1003   case AArch64::HWASAN_CHECK_MEMACCESS:
1004     LowerHWASAN_CHECK_MEMACCESS(*MI);
1005     return;
1006
1007   case AArch64::SEH_StackAlloc:
1008     TS->EmitARM64WinCFIAllocStack(MI->getOperand(0).getImm());
1009     return;
1010
1011   case AArch64::SEH_SaveFPLR:
1012     TS->EmitARM64WinCFISaveFPLR(MI->getOperand(0).getImm());
1013     return;
1014
1015   case AArch64::SEH_SaveFPLR_X:
1016     assert(MI->getOperand(0).getImm() < 0 &&
1017            "Pre increment SEH opcode must have a negative offset");
1018     TS->EmitARM64WinCFISaveFPLRX(-MI->getOperand(0).getImm());
1019     return;
1020
1021   case AArch64::SEH_SaveReg:
1022     TS->EmitARM64WinCFISaveReg(MI->getOperand(0).getImm(),
1023                                MI->getOperand(1).getImm());
1024     return;
1025
1026   case AArch64::SEH_SaveReg_X:
1027     assert(MI->getOperand(1).getImm() < 0 &&
1028            "Pre increment SEH opcode must have a negative offset");
1029     TS->EmitARM64WinCFISaveRegX(MI->getOperand(0).getImm(),
1030                                 -MI->getOperand(1).getImm());
1031     return;
1032
1033   case AArch64::SEH_SaveRegP:
1034     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1035             "Non-consecutive registers not allowed for save_regp");
1036     TS->EmitARM64WinCFISaveRegP(MI->getOperand(0).getImm(),
1037                                 MI->getOperand(2).getImm());
1038     return;
1039
1040   case AArch64::SEH_SaveRegP_X:
1041     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1042             "Non-consecutive registers not allowed for save_regp_x");
1043     assert(MI->getOperand(2).getImm() < 0 &&
1044            "Pre increment SEH opcode must have a negative offset");
1045     TS->EmitARM64WinCFISaveRegPX(MI->getOperand(0).getImm(),
1046                                  -MI->getOperand(2).getImm());
1047     return;
1048
1049   case AArch64::SEH_SaveFReg:
1050     TS->EmitARM64WinCFISaveFReg(MI->getOperand(0).getImm(),
1051                                 MI->getOperand(1).getImm());
1052     return;
1053
1054   case AArch64::SEH_SaveFReg_X:
1055     assert(MI->getOperand(1).getImm() < 0 &&
1056            "Pre increment SEH opcode must have a negative offset");
1057     TS->EmitARM64WinCFISaveFRegX(MI->getOperand(0).getImm(),
1058                                  -MI->getOperand(1).getImm());
1059     return;
1060
1061   case AArch64::SEH_SaveFRegP:
1062     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1063             "Non-consecutive registers not allowed for save_regp");
1064     TS->EmitARM64WinCFISaveFRegP(MI->getOperand(0).getImm(),
1065                                  MI->getOperand(2).getImm());
1066     return;
1067
1068   case AArch64::SEH_SaveFRegP_X:
1069     assert((MI->getOperand(1).getImm() - MI->getOperand(0).getImm() == 1) &&
1070             "Non-consecutive registers not allowed for save_regp_x");
1071     assert(MI->getOperand(2).getImm() < 0 &&
1072            "Pre increment SEH opcode must have a negative offset");
1073     TS->EmitARM64WinCFISaveFRegPX(MI->getOperand(0).getImm(),
1074                                   -MI->getOperand(2).getImm());
1075     return;
1076
1077   case AArch64::SEH_SetFP:
1078     TS->EmitARM64WinCFISetFP();
1079     return;
1080
1081   case AArch64::SEH_AddFP:
1082     TS->EmitARM64WinCFIAddFP(MI->getOperand(0).getImm());
1083     return;
1084
1085   case AArch64::SEH_Nop:
1086     TS->EmitARM64WinCFINop();
1087     return;
1088
1089   case AArch64::SEH_PrologEnd:
1090     TS->EmitARM64WinCFIPrologEnd();
1091     return;
1092
1093   case AArch64::SEH_EpilogStart:
1094     TS->EmitARM64WinCFIEpilogStart();
1095     return;
1096
1097   case AArch64::SEH_EpilogEnd:
1098     TS->EmitARM64WinCFIEpilogEnd();
1099     return;
1100   }
1101
1102   // Finally, do the automated lowerings for everything else.
1103   MCInst TmpInst;
1104   MCInstLowering.Lower(MI, TmpInst);
1105   EmitToStreamer(*OutStreamer, TmpInst);
1106 }
1107
1108 // Force static initialization.
1109 extern "C" void LLVMInitializeAArch64AsmPrinter() {
1110   RegisterAsmPrinter<AArch64AsmPrinter> X(getTheAArch64leTarget());
1111   RegisterAsmPrinter<AArch64AsmPrinter> Y(getTheAArch64beTarget());
1112   RegisterAsmPrinter<AArch64AsmPrinter> Z(getTheARM64Target());
1113 }