llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

   1 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a printer that converts from our internal representation
  10 // of machine-dependent LLVM code to the RISC-V assembly language.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "MCTargetDesc/RISCVBaseInfo.h"
  15 #include "MCTargetDesc/RISCVInstPrinter.h"
  16 #include "MCTargetDesc/RISCVMCExpr.h"
  17 #include "MCTargetDesc/RISCVMatInt.h"
  18 #include "MCTargetDesc/RISCVTargetStreamer.h"
  19 #include "RISCV.h"
  20 #include "RISCVConstantPoolValue.h"
  21 #include "RISCVMachineFunctionInfo.h"
  22 #include "RISCVRegisterInfo.h"
  23 #include "TargetInfo/RISCVTargetInfo.h"
  24 #include "llvm/ADT/APInt.h"
  25 #include "llvm/ADT/Statistic.h"
  26 #include "llvm/BinaryFormat/ELF.h"
  27 #include "llvm/CodeGen/AsmPrinter.h"
  28 #include "llvm/CodeGen/MachineConstantPool.h"
  29 #include "llvm/CodeGen/MachineInstr.h"
  30 #include "llvm/CodeGen/MachineModuleInfo.h"
  31 #include "llvm/IR/Module.h"
  32 #include "llvm/MC/MCAsmInfo.h"
  33 #include "llvm/MC/MCContext.h"
  34 #include "llvm/MC/MCInst.h"
  35 #include "llvm/MC/MCInstBuilder.h"
  36 #include "llvm/MC/MCObjectFileInfo.h"
  37 #include "llvm/MC/MCSectionELF.h"
  38 #include "llvm/MC/MCStreamer.h"
  39 #include "llvm/MC/MCSymbol.h"
  40 #include "llvm/MC/TargetRegistry.h"
  41 #include "llvm/Support/raw_ostream.h"
  42 #include "llvm/TargetParser/RISCVISAInfo.h"
  43 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
  44
  45 using namespace llvm;
  46
  47 #define DEBUG_TYPE "asm-printer"
  48
  49 STATISTIC(RISCVNumInstrsCompressed,
  50           "Number of RISC-V Compressed instructions emitted");
  51
  52 namespace llvm {
  53 extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures];
  54 } // namespace llvm
  55
  56 namespace {
  57 class RISCVAsmPrinter : public AsmPrinter {
  58   const RISCVSubtarget *STI;
  59
  60 public:
  61   explicit RISCVAsmPrinter(TargetMachine &TM,
  62                            std::unique_ptr<MCStreamer> Streamer)
  63       : AsmPrinter(TM, std::move(Streamer)) {}
  64
  65   StringRef getPassName() const override { return "RISC-V Assembly Printer"; }
  66
  67   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
  68                      const MachineInstr &MI);
  69
  70   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  71                        const MachineInstr &MI);
  72
  73   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  74                        const MachineInstr &MI);
  75
  76   bool runOnMachineFunction(MachineFunction &MF) override;
  77
  78   void emitInstruction(const MachineInstr *MI) override;
  79
  80   void emitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) override;
  81
  82   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
  83                        const char *ExtraCode, raw_ostream &OS) override;
  84   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
  85                              const char *ExtraCode, raw_ostream &OS) override;
  86
  87   // Returns whether Inst is compressed.
  88   bool EmitToStreamer(MCStreamer &S, const MCInst &Inst,
  89                       const MCSubtargetInfo &SubtargetInfo);
  90   bool EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
  91     return EmitToStreamer(S, Inst, *STI);
  92   }
  93
  94   bool lowerPseudoInstExpansion(const MachineInstr *MI, MCInst &Inst);
  95
  96   typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
  97   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
  98   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
  99   void LowerKCFI_CHECK(const MachineInstr &MI);
 100   void EmitHwasanMemaccessSymbols(Module &M);
 101
 102   // Wrapper needed for tblgenned pseudo lowering.
 103   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const;
 104
 105   void emitStartOfAsmFile(Module &M) override;
 106   void emitEndOfAsmFile(Module &M) override;
 107
 108   void emitFunctionEntryLabel() override;
 109   bool emitDirectiveOptionArch();
 110
 111 private:
 112   void emitAttributes(const MCSubtargetInfo &SubtargetInfo);
 113
 114   void emitNTLHint(const MachineInstr *MI);
 115
 116   // XRay Support
 117   void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr *MI);
 118   void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr *MI);
 119   void LowerPATCHABLE_TAIL_CALL(const MachineInstr *MI);
 120   void emitSled(const MachineInstr *MI, SledKind Kind);
 121
 122   bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
 123 };
 124 }
 125
 126 void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
 127                                     const MachineInstr &MI) {
 128   unsigned NOPBytes = STI->hasStdExtCOrZca() ? 2 : 4;
 129   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
 130
 131   auto &Ctx = OutStreamer.getContext();
 132   MCSymbol *MILabel = Ctx.createTempSymbol();
 133   OutStreamer.emitLabel(MILabel);
 134
 135   SM.recordStackMap(*MILabel, MI);
 136   assert(NumNOPBytes % NOPBytes == 0 &&
 137          "Invalid number of NOP bytes requested!");
 138
 139   // Scan ahead to trim the shadow.
 140   const MachineBasicBlock &MBB = *MI.getParent();
 141   MachineBasicBlock::const_iterator MII(MI);
 142   ++MII;
 143   while (NumNOPBytes > 0) {
 144     if (MII == MBB.end() || MII->isCall() ||
 145         MII->getOpcode() == RISCV::DBG_VALUE ||
 146         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
 147         MII->getOpcode() == TargetOpcode::STACKMAP)
 148       break;
 149     ++MII;
 150     NumNOPBytes -= 4;
 151   }
 152
 153   // Emit nops.
 154   emitNops(NumNOPBytes / NOPBytes);
 155 }
 156
 157 // Lower a patchpoint of the form:
 158 // [<def>], <id>, <numBytes>, <target>, <numArgs>
 159 void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 160                                       const MachineInstr &MI) {
 161   unsigned NOPBytes = STI->hasStdExtCOrZca() ? 2 : 4;
 162
 163   auto &Ctx = OutStreamer.getContext();
 164   MCSymbol *MILabel = Ctx.createTempSymbol();
 165   OutStreamer.emitLabel(MILabel);
 166   SM.recordPatchPoint(*MILabel, MI);
 167
 168   PatchPointOpers Opers(&MI);
 169
 170   const MachineOperand &CalleeMO = Opers.getCallTarget();
 171   unsigned EncodedBytes = 0;
 172
 173   if (CalleeMO.isImm()) {
 174     uint64_t CallTarget = CalleeMO.getImm();
 175     if (CallTarget) {
 176       assert((CallTarget & 0xFFFF'FFFF'FFFF) == CallTarget &&
 177              "High 16 bits of call target should be zero.");
 178       // Materialize the jump address:
 179       SmallVector<MCInst, 8> Seq;
 180       RISCVMatInt::generateMCInstSeq(CallTarget, *STI, RISCV::X1, Seq);
 181       for (MCInst &Inst : Seq) {
 182         bool Compressed = EmitToStreamer(OutStreamer, Inst);
 183         EncodedBytes += Compressed ? 2 : 4;
 184       }
 185       bool Compressed = EmitToStreamer(OutStreamer, MCInstBuilder(RISCV::JALR)
 186                                                         .addReg(RISCV::X1)
 187                                                         .addReg(RISCV::X1)
 188                                                         .addImm(0));
 189       EncodedBytes += Compressed ? 2 : 4;
 190     }
 191   } else if (CalleeMO.isGlobal()) {
 192     MCOperand CallTargetMCOp;
 193     lowerOperand(CalleeMO, CallTargetMCOp);
 194     EmitToStreamer(OutStreamer,
 195                    MCInstBuilder(RISCV::PseudoCALL).addOperand(CallTargetMCOp));
 196     EncodedBytes += 8;
 197   }
 198
 199   // Emit padding.
 200   unsigned NumBytes = Opers.getNumPatchBytes();
 201   assert(NumBytes >= EncodedBytes &&
 202          "Patchpoint can't request size less than the length of a call.");
 203   assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
 204          "Invalid number of NOP bytes requested!");
 205   emitNops((NumBytes - EncodedBytes) / NOPBytes);
 206 }
 207
 208 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 209                                       const MachineInstr &MI) {
 210   unsigned NOPBytes = STI->hasStdExtCOrZca() ? 2 : 4;
 211
 212   StatepointOpers SOpers(&MI);
 213   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
 214     assert(PatchBytes % NOPBytes == 0 &&
 215            "Invalid number of NOP bytes requested!");
 216     emitNops(PatchBytes / NOPBytes);
 217   } else {
 218     // Lower call target and choose correct opcode
 219     const MachineOperand &CallTarget = SOpers.getCallTarget();
 220     MCOperand CallTargetMCOp;
 221     switch (CallTarget.getType()) {
 222     case MachineOperand::MO_GlobalAddress:
 223     case MachineOperand::MO_ExternalSymbol:
 224       lowerOperand(CallTarget, CallTargetMCOp);
 225       EmitToStreamer(
 226           OutStreamer,
 227           MCInstBuilder(RISCV::PseudoCALL).addOperand(CallTargetMCOp));
 228       break;
 229     case MachineOperand::MO_Immediate:
 230       CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
 231       EmitToStreamer(OutStreamer, MCInstBuilder(RISCV::JAL)
 232                                       .addReg(RISCV::X1)
 233                                       .addOperand(CallTargetMCOp));
 234       break;
 235     case MachineOperand::MO_Register:
 236       CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
 237       EmitToStreamer(OutStreamer, MCInstBuilder(RISCV::JALR)
 238                                       .addReg(RISCV::X1)
 239                                       .addOperand(CallTargetMCOp)
 240                                       .addImm(0));
 241       break;
 242     default:
 243       llvm_unreachable("Unsupported operand type in statepoint call target");
 244       break;
 245     }
 246   }
 247
 248   auto &Ctx = OutStreamer.getContext();
 249   MCSymbol *MILabel = Ctx.createTempSymbol();
 250   OutStreamer.emitLabel(MILabel);
 251   SM.recordStatepoint(*MILabel, MI);
 252 }
 253
 254 bool RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst,
 255                                      const MCSubtargetInfo &SubtargetInfo) {
 256   MCInst CInst;
 257   bool Res = RISCVRVC::compress(CInst, Inst, SubtargetInfo);
 258   if (Res)
 259     ++RISCVNumInstrsCompressed;
 260   S.emitInstruction(Res ? CInst : Inst, SubtargetInfo);
 261   return Res;
 262 }
 263
 264 // Simple pseudo-instructions have their lowering (with expansion to real
 265 // instructions) auto-generated.
 266 #include "RISCVGenMCPseudoLowering.inc"
 267
 268 // If the target supports Zihintntl and the instruction has a nontemporal
 269 // MachineMemOperand, emit an NTLH hint instruction before it.
 270 void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
 271   if (!STI->hasStdExtZihintntl())
 272     return;
 273
 274   if (MI->memoperands_empty())
 275     return;
 276
 277   MachineMemOperand *MMO = *(MI->memoperands_begin());
 278   if (!MMO->isNonTemporal())
 279     return;
 280
 281   unsigned NontemporalMode = 0;
 282   if (MMO->getFlags() & MONontemporalBit0)
 283     NontemporalMode += 0b1;
 284   if (MMO->getFlags() & MONontemporalBit1)
 285     NontemporalMode += 0b10;
 286
 287   MCInst Hint;
 288   if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs())
 289     Hint.setOpcode(RISCV::C_ADD_HINT);
 290   else
 291     Hint.setOpcode(RISCV::ADD);
 292
 293   Hint.addOperand(MCOperand::createReg(RISCV::X0));
 294   Hint.addOperand(MCOperand::createReg(RISCV::X0));
 295   Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode));
 296
 297   EmitToStreamer(*OutStreamer, Hint);
 298 }
 299
 300 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
 301   RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
 302                                         getSubtargetInfo().getFeatureBits());
 303
 304   emitNTLHint(MI);
 305
 306   // Do any auto-generated pseudo lowerings.
 307   if (MCInst OutInst; lowerPseudoInstExpansion(MI, OutInst)) {
 308     EmitToStreamer(*OutStreamer, OutInst);
 309     return;
 310   }
 311
 312   switch (MI->getOpcode()) {
 313   case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
 314     LowerHWASAN_CHECK_MEMACCESS(*MI);
 315     return;
 316   case RISCV::KCFI_CHECK:
 317     LowerKCFI_CHECK(*MI);
 318     return;
 319   case TargetOpcode::STACKMAP:
 320     return LowerSTACKMAP(*OutStreamer, SM, *MI);
 321   case TargetOpcode::PATCHPOINT:
 322     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
 323   case TargetOpcode::STATEPOINT:
 324     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
 325   case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
 326     // patchable-function-entry is handled in lowerToMCInst
 327     // Therefore, we break out of the switch statement if we encounter it here.
 328     const Function &F = MI->getParent()->getParent()->getFunction();
 329     if (F.hasFnAttribute("patchable-function-entry"))
 330       break;
 331
 332     LowerPATCHABLE_FUNCTION_ENTER(MI);
 333     return;
 334   }
 335   case TargetOpcode::PATCHABLE_FUNCTION_EXIT:
 336     LowerPATCHABLE_FUNCTION_EXIT(MI);
 337     return;
 338   case TargetOpcode::PATCHABLE_TAIL_CALL:
 339     LowerPATCHABLE_TAIL_CALL(MI);
 340     return;
 341   }
 342
 343   MCInst OutInst;
 344   if (!lowerToMCInst(MI, OutInst))
 345     EmitToStreamer(*OutStreamer, OutInst);
 346 }
 347
 348 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
 349                                       const char *ExtraCode, raw_ostream &OS) {
 350   // First try the generic code, which knows about modifiers like 'c' and 'n'.
 351   if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS))
 352     return false;
 353
 354   const MachineOperand &MO = MI->getOperand(OpNo);
 355   if (ExtraCode && ExtraCode[0]) {
 356     if (ExtraCode[1] != 0)
 357       return true; // Unknown modifier.
 358
 359     switch (ExtraCode[0]) {
 360     default:
 361       return true; // Unknown modifier.
 362     case 'z':      // Print zero register if zero, regular printing otherwise.
 363       if (MO.isImm() && MO.getImm() == 0) {
 364         OS << RISCVInstPrinter::getRegisterName(RISCV::X0);
 365         return false;
 366       }
 367       break;
 368     case 'i': // Literal 'i' if operand is not a register.
 369       if (!MO.isReg())
 370         OS << 'i';
 371       return false;
 372     case 'N': // Print the register encoding as an integer (0-31)
 373       if (!MO.isReg())
 374         return true;
 375
 376       const RISCVRegisterInfo *TRI = STI->getRegisterInfo();
 377       OS << TRI->getEncodingValue(MO.getReg());
 378       return false;
 379     }
 380   }
 381
 382   switch (MO.getType()) {
 383   case MachineOperand::MO_Immediate:
 384     OS << MO.getImm();
 385     return false;
 386   case MachineOperand::MO_Register:
 387     OS << RISCVInstPrinter::getRegisterName(MO.getReg());
 388     return false;
 389   case MachineOperand::MO_GlobalAddress:
 390     PrintSymbolOperand(MO, OS);
 391     return false;
 392   case MachineOperand::MO_BlockAddress: {
 393     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
 394     Sym->print(OS, MAI);
 395     return false;
 396   }
 397   default:
 398     break;
 399   }
 400
 401   return true;
 402 }
 403
 404 bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 405                                             unsigned OpNo,
 406                                             const char *ExtraCode,
 407                                             raw_ostream &OS) {
 408   if (ExtraCode)
 409     return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS);
 410
 411   const MachineOperand &AddrReg = MI->getOperand(OpNo);
 412   assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand");
 413   const MachineOperand &Offset = MI->getOperand(OpNo + 1);
 414   // All memory operands should have a register and an immediate operand (see
 415   // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand).
 416   if (!AddrReg.isReg())
 417     return true;
 418   if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() &&
 419       !Offset.isMCSymbol())
 420     return true;
 421
 422   MCOperand MCO;
 423   if (!lowerOperand(Offset, MCO))
 424     return true;
 425
 426   if (Offset.isImm())
 427     OS << MCO.getImm();
 428   else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol())
 429     OS << *MCO.getExpr();
 430
 431   if (Offset.isMCSymbol())
 432     MMI->getContext().registerInlineAsmLabel(Offset.getMCSymbol());
 433   if (Offset.isBlockAddress()) {
 434     const BlockAddress *BA = Offset.getBlockAddress();
 435     MCSymbol *Sym = GetBlockAddressSymbol(BA);
 436     MMI->getContext().registerInlineAsmLabel(Sym);
 437   }
 438
 439   OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")";
 440   return false;
 441 }
 442
 443 bool RISCVAsmPrinter::emitDirectiveOptionArch() {
 444   RISCVTargetStreamer &RTS =
 445       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 446   SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs;
 447   const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
 448   for (const auto &Feature : RISCVFeatureKV) {
 449     if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value))
 450       continue;
 451
 452     if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key))
 453       continue;
 454
 455     auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus
 456                                                 : RISCVOptionArchArgType::Minus;
 457     NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key);
 458   }
 459   if (!NeedEmitStdOptionArgs.empty()) {
 460     RTS.emitDirectiveOptionPush();
 461     RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs);
 462     return true;
 463   }
 464
 465   return false;
 466 }
 467
 468 bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 469   STI = &MF.getSubtarget<RISCVSubtarget>();
 470   RISCVTargetStreamer &RTS =
 471       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 472
 473   bool EmittedOptionArch = emitDirectiveOptionArch();
 474
 475   SetupMachineFunction(MF);
 476   emitFunctionBody();
 477
 478   // Emit the XRay table
 479   emitXRayTable();
 480
 481   if (EmittedOptionArch)
 482     RTS.emitDirectiveOptionPop();
 483   return false;
 484 }
 485
 486 void RISCVAsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr *MI) {
 487   emitSled(MI, SledKind::FUNCTION_ENTER);
 488 }
 489
 490 void RISCVAsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr *MI) {
 491   emitSled(MI, SledKind::FUNCTION_EXIT);
 492 }
 493
 494 void RISCVAsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr *MI) {
 495   emitSled(MI, SledKind::TAIL_CALL);
 496 }
 497
 498 void RISCVAsmPrinter::emitSled(const MachineInstr *MI, SledKind Kind) {
 499   // We want to emit the jump instruction and the nops constituting the sled.
 500   // The format is as follows:
 501   // .Lxray_sled_N
 502   //   ALIGN
 503   //   J .tmpN
 504   //   21 or 33 C.NOP instructions
 505   // .tmpN
 506
 507   // The following variable holds the count of the number of NOPs to be patched
 508   // in for XRay instrumentation during compilation.
 509   // Note that RV64 and RV32 each has a sled of 68 and 44 bytes, respectively.
 510   // Assuming we're using JAL to jump to .tmpN, then we only need
 511   // (68 - 4)/2 = 32 NOPs for RV64 and (44 - 4)/2 = 20 for RV32. However, there
 512   // is a chance that we'll use C.JAL instead, so an additional NOP is needed.
 513   const uint8_t NoopsInSledCount =
 514       MI->getParent()->getParent()->getSubtarget<RISCVSubtarget>().is64Bit()
 515           ? 33
 516           : 21;
 517
 518   OutStreamer->emitCodeAlignment(Align(4), &getSubtargetInfo());
 519   auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
 520   OutStreamer->emitLabel(CurSled);
 521   auto Target = OutContext.createTempSymbol();
 522
 523   const MCExpr *TargetExpr = MCSymbolRefExpr::create(
 524       Target, MCSymbolRefExpr::VariantKind::VK_None, OutContext);
 525
 526   // Emit "J bytes" instruction, which jumps over the nop sled to the actual
 527   // start of function.
 528   EmitToStreamer(
 529       *OutStreamer,
 530       MCInstBuilder(RISCV::JAL).addReg(RISCV::X0).addExpr(TargetExpr));
 531
 532   // Emit NOP instructions
 533   for (int8_t I = 0; I < NoopsInSledCount; ++I)
 534     EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
 535                                      .addReg(RISCV::X0)
 536                                      .addReg(RISCV::X0)
 537                                      .addImm(0));
 538
 539   OutStreamer->emitLabel(Target);
 540   recordSled(CurSled, *MI, Kind, 2);
 541 }
 542
 543 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
 544   RISCVTargetStreamer &RTS =
 545       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 546   if (const MDString *ModuleTargetABI =
 547           dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi")))
 548     RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString()));
 549
 550   MCSubtargetInfo SubtargetInfo = *TM.getMCSubtargetInfo();
 551
 552   // Use module flag to update feature bits.
 553   if (auto *MD = dyn_cast_or_null<MDNode>(M.getModuleFlag("riscv-isa"))) {
 554     for (auto &ISA : MD->operands()) {
 555       if (auto *ISAString = dyn_cast_or_null<MDString>(ISA)) {
 556         auto ParseResult = llvm::RISCVISAInfo::parseArchString(
 557             ISAString->getString(), /*EnableExperimentalExtension=*/true,
 558             /*ExperimentalExtensionVersionCheck=*/true);
 559         if (!errorToBool(ParseResult.takeError())) {
 560           auto &ISAInfo = *ParseResult;
 561           for (const auto &Feature : RISCVFeatureKV) {
 562             if (ISAInfo->hasExtension(Feature.Key) &&
 563                 !SubtargetInfo.hasFeature(Feature.Value))
 564               SubtargetInfo.ToggleFeature(Feature.Key);
 565           }
 566         }
 567       }
 568     }
 569
 570     RTS.setFlagsFromFeatures(SubtargetInfo);
 571   }
 572
 573   if (TM.getTargetTriple().isOSBinFormatELF())
 574     emitAttributes(SubtargetInfo);
 575 }
 576
 577 void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) {
 578   RISCVTargetStreamer &RTS =
 579       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 580
 581   if (TM.getTargetTriple().isOSBinFormatELF())
 582     RTS.finishAttributeSection();
 583   EmitHwasanMemaccessSymbols(M);
 584 }
 585
 586 void RISCVAsmPrinter::emitAttributes(const MCSubtargetInfo &SubtargetInfo) {
 587   RISCVTargetStreamer &RTS =
 588       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 589   // Use MCSubtargetInfo from TargetMachine. Individual functions may have
 590   // attributes that differ from other functions in the module and we have no
 591   // way to know which function is correct.
 592   RTS.emitTargetAttributes(SubtargetInfo, /*EmitStackAlign*/ true);
 593 }
 594
 595 void RISCVAsmPrinter::emitFunctionEntryLabel() {
 596   const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>();
 597   if (RMFI->isVectorCall()) {
 598     auto &RTS =
 599         static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 600     RTS.emitDirectiveVariantCC(*CurrentFnSym);
 601   }
 602   return AsmPrinter::emitFunctionEntryLabel();
 603 }
 604
 605 // Force static initialization.
 606 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() {
 607   RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
 608   RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target());
 609 }
 610
 611 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
 612   Register Reg = MI.getOperand(0).getReg();
 613   uint32_t AccessInfo = MI.getOperand(1).getImm();
 614   MCSymbol *&Sym =
 615       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)];
 616   if (!Sym) {
 617     // FIXME: Make this work on non-ELF.
 618     if (!TM.getTargetTriple().isOSBinFormatELF())
 619       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
 620
 621     std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" +
 622                           utostr(AccessInfo) + "_short";
 623     Sym = OutContext.getOrCreateSymbol(SymName);
 624   }
 625   auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext);
 626   auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext);
 627
 628   EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
 629 }
 630
 631 void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
 632   Register AddrReg = MI.getOperand(0).getReg();
 633   assert(std::next(MI.getIterator())->isCall() &&
 634          "KCFI_CHECK not followed by a call instruction");
 635   assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
 636          "KCFI_CHECK call target doesn't match call operand");
 637
 638   // Temporary registers for comparing the hashes. If a register is used
 639   // for the call target, or reserved by the user, we can clobber another
 640   // temporary register as the check is immediately followed by the
 641   // call. The check defaults to X6/X7, but can fall back to X28-X31 if
 642   // needed.
 643   unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
 644   unsigned NextReg = RISCV::X28;
 645   auto isRegAvailable = [&](unsigned Reg) {
 646     return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg);
 647   };
 648   for (auto &Reg : ScratchRegs) {
 649     if (isRegAvailable(Reg))
 650       continue;
 651     while (!isRegAvailable(NextReg))
 652       ++NextReg;
 653     Reg = NextReg++;
 654     if (Reg > RISCV::X31)
 655       report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
 656   }
 657
 658   if (AddrReg == RISCV::X0) {
 659     // Checking X0 makes no sense. Instead of emitting a load, zero
 660     // ScratchRegs[0].
 661     EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
 662                                      .addReg(ScratchRegs[0])
 663                                      .addReg(RISCV::X0)
 664                                      .addImm(0));
 665   } else {
 666     // Adjust the offset for patchable-function-prefix. This assumes that
 667     // patchable-function-prefix is the same for all functions.
 668     int NopSize = STI->hasStdExtCOrZca() ? 2 : 4;
 669     int64_t PrefixNops = 0;
 670     (void)MI.getMF()
 671         ->getFunction()
 672         .getFnAttribute("patchable-function-prefix")
 673         .getValueAsString()
 674         .getAsInteger(10, PrefixNops);
 675
 676     // Load the target function type hash.
 677     EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
 678                                      .addReg(ScratchRegs[0])
 679                                      .addReg(AddrReg)
 680                                      .addImm(-(PrefixNops * NopSize + 4)));
 681   }
 682
 683   // Load the expected 32-bit type hash.
 684   const int64_t Type = MI.getOperand(1).getImm();
 685   const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
 686   const int64_t Lo12 = SignExtend64<12>(Type);
 687   if (Hi20) {
 688     EmitToStreamer(
 689         *OutStreamer,
 690         MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
 691   }
 692   if (Lo12 || Hi20 == 0) {
 693     EmitToStreamer(*OutStreamer,
 694                    MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20)
 695                                      ? RISCV::ADDIW
 696                                      : RISCV::ADDI)
 697                        .addReg(ScratchRegs[1])
 698                        .addReg(ScratchRegs[1])
 699                        .addImm(Lo12));
 700   }
 701
 702   // Compare the hashes and trap if there's a mismatch.
 703   MCSymbol *Pass = OutContext.createTempSymbol();
 704   EmitToStreamer(*OutStreamer,
 705                  MCInstBuilder(RISCV::BEQ)
 706                      .addReg(ScratchRegs[0])
 707                      .addReg(ScratchRegs[1])
 708                      .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
 709
 710   MCSymbol *Trap = OutContext.createTempSymbol();
 711   OutStreamer->emitLabel(Trap);
 712   EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
 713   emitKCFITrapEntry(*MI.getMF(), Trap);
 714   OutStreamer->emitLabel(Pass);
 715 }
 716
 717 void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
 718   if (HwasanMemaccessSymbols.empty())
 719     return;
 720
 721   assert(TM.getTargetTriple().isOSBinFormatELF());
 722   // Use MCSubtargetInfo from TargetMachine. Individual functions may have
 723   // attributes that differ from other functions in the module and we have no
 724   // way to know which function is correct.
 725   const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
 726
 727   MCSymbol *HwasanTagMismatchV2Sym =
 728       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
 729   // Annotate symbol as one having incompatible calling convention, so
 730   // run-time linkers can instead eagerly bind this function.
 731   auto &RTS =
 732       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 733   RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym);
 734
 735   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
 736       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
 737   auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref,
 738                                   RISCVMCExpr::VK_RISCV_CALL, OutContext);
 739
 740   for (auto &P : HwasanMemaccessSymbols) {
 741     unsigned Reg = std::get<0>(P.first);
 742     uint32_t AccessInfo = std::get<1>(P.first);
 743     MCSymbol *Sym = P.second;
 744
 745     unsigned Size =
 746         1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
 747     OutStreamer->switchSection(OutContext.getELFSection(
 748         ".text.hot", ELF::SHT_PROGBITS,
 749         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
 750         /*IsComdat=*/true));
 751
 752     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
 753     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
 754     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
 755     OutStreamer->emitLabel(Sym);
 756
 757     // Extract shadow offset from ptr
 758     EmitToStreamer(
 759         *OutStreamer,
 760         MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8),
 761         MCSTI);
 762     EmitToStreamer(*OutStreamer,
 763                    MCInstBuilder(RISCV::SRLI)
 764                        .addReg(RISCV::X6)
 765                        .addReg(RISCV::X6)
 766                        .addImm(12),
 767                    MCSTI);
 768     // load shadow tag in X6, X5 contains shadow base
 769     EmitToStreamer(*OutStreamer,
 770                    MCInstBuilder(RISCV::ADD)
 771                        .addReg(RISCV::X6)
 772                        .addReg(RISCV::X5)
 773                        .addReg(RISCV::X6),
 774                    MCSTI);
 775     EmitToStreamer(
 776         *OutStreamer,
 777         MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
 778         MCSTI);
 779     // Extract tag from pointer and compare it with loaded tag from shadow
 780     EmitToStreamer(
 781         *OutStreamer,
 782         MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56),
 783         MCSTI);
 784     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
 785     // X7 contains tag from the pointer, while X6 contains tag from memory
 786     EmitToStreamer(*OutStreamer,
 787                    MCInstBuilder(RISCV::BNE)
 788                        .addReg(RISCV::X7)
 789                        .addReg(RISCV::X6)
 790                        .addExpr(MCSymbolRefExpr::create(
 791                            HandleMismatchOrPartialSym, OutContext)),
 792                    MCSTI);
 793     MCSymbol *ReturnSym = OutContext.createTempSymbol();
 794     OutStreamer->emitLabel(ReturnSym);
 795     EmitToStreamer(*OutStreamer,
 796                    MCInstBuilder(RISCV::JALR)
 797                        .addReg(RISCV::X0)
 798                        .addReg(RISCV::X1)
 799                        .addImm(0),
 800                    MCSTI);
 801     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
 802
 803     EmitToStreamer(*OutStreamer,
 804                    MCInstBuilder(RISCV::ADDI)
 805                        .addReg(RISCV::X28)
 806                        .addReg(RISCV::X0)
 807                        .addImm(16),
 808                    MCSTI);
 809     MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
 810     EmitToStreamer(
 811         *OutStreamer,
 812         MCInstBuilder(RISCV::BGEU)
 813             .addReg(RISCV::X6)
 814             .addReg(RISCV::X28)
 815             .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 816         MCSTI);
 817
 818     EmitToStreamer(
 819         *OutStreamer,
 820         MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF),
 821         MCSTI);
 822
 823     if (Size != 1)
 824       EmitToStreamer(*OutStreamer,
 825                      MCInstBuilder(RISCV::ADDI)
 826                          .addReg(RISCV::X28)
 827                          .addReg(RISCV::X28)
 828                          .addImm(Size - 1),
 829                      MCSTI);
 830     EmitToStreamer(
 831         *OutStreamer,
 832         MCInstBuilder(RISCV::BGE)
 833             .addReg(RISCV::X28)
 834             .addReg(RISCV::X6)
 835             .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 836         MCSTI);
 837
 838     EmitToStreamer(
 839         *OutStreamer,
 840         MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF),
 841         MCSTI);
 842     EmitToStreamer(
 843         *OutStreamer,
 844         MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
 845         MCSTI);
 846     EmitToStreamer(*OutStreamer,
 847                    MCInstBuilder(RISCV::BEQ)
 848                        .addReg(RISCV::X6)
 849                        .addReg(RISCV::X7)
 850                        .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
 851                    MCSTI);
 852
 853     OutStreamer->emitLabel(HandleMismatchSym);
 854
 855     // | Previous stack frames...        |
 856     // +=================================+ <-- [SP + 256]
 857     // |              ...                |
 858     // |                                 |
 859     // | Stack frame space for x12 - x31.|
 860     // |                                 |
 861     // |              ...                |
 862     // +---------------------------------+ <-- [SP + 96]
 863     // | Saved x11(arg1), as             |
 864     // | __hwasan_check_* clobbers it.   |
 865     // +---------------------------------+ <-- [SP + 88]
 866     // | Saved x10(arg0), as             |
 867     // | __hwasan_check_* clobbers it.   |
 868     // +---------------------------------+ <-- [SP + 80]
 869     // |                                 |
 870     // | Stack frame space for x9.       |
 871     // +---------------------------------+ <-- [SP + 72]
 872     // |                                 |
 873     // | Saved x8(fp), as                |
 874     // | __hwasan_check_* clobbers it.   |
 875     // +---------------------------------+ <-- [SP + 64]
 876     // |              ...                |
 877     // |                                 |
 878     // | Stack frame space for x2 - x7.  |
 879     // |                                 |
 880     // |              ...                |
 881     // +---------------------------------+ <-- [SP + 16]
 882     // | Return address (x1) for caller  |
 883     // | of __hwasan_check_*.            |
 884     // +---------------------------------+ <-- [SP + 8]
 885     // | Reserved place for x0, possibly |
 886     // | junk, since we don't save it.   |
 887     // +---------------------------------+ <-- [x2 / SP]
 888
 889     // Adjust sp
 890     EmitToStreamer(*OutStreamer,
 891                    MCInstBuilder(RISCV::ADDI)
 892                        .addReg(RISCV::X2)
 893                        .addReg(RISCV::X2)
 894                        .addImm(-256),
 895                    MCSTI);
 896
 897     // store x10(arg0) by new sp
 898     EmitToStreamer(*OutStreamer,
 899                    MCInstBuilder(RISCV::SD)
 900                        .addReg(RISCV::X10)
 901                        .addReg(RISCV::X2)
 902                        .addImm(8 * 10),
 903                    MCSTI);
 904     // store x11(arg1) by new sp
 905     EmitToStreamer(*OutStreamer,
 906                    MCInstBuilder(RISCV::SD)
 907                        .addReg(RISCV::X11)
 908                        .addReg(RISCV::X2)
 909                        .addImm(8 * 11),
 910                    MCSTI);
 911
 912     // store x8(fp) by new sp
 913     EmitToStreamer(
 914         *OutStreamer,
 915         MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 *
 916                                                                             8),
 917         MCSTI);
 918     // store x1(ra) by new sp
 919     EmitToStreamer(
 920         *OutStreamer,
 921         MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 *
 922                                                                             8),
 923         MCSTI);
 924     if (Reg != RISCV::X10)
 925       EmitToStreamer(
 926           *OutStreamer,
 927           MCInstBuilder(RISCV::ADDI).addReg(RISCV::X10).addReg(Reg).addImm(0),
 928           MCSTI);
 929     EmitToStreamer(*OutStreamer,
 930                    MCInstBuilder(RISCV::ADDI)
 931                        .addReg(RISCV::X11)
 932                        .addReg(RISCV::X0)
 933                        .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask),
 934                    MCSTI);
 935
 936     EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr),
 937                    MCSTI);
 938   }
 939 }
 940
 941 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
 942                                     const AsmPrinter &AP) {
 943   MCContext &Ctx = AP.OutContext;
 944   RISCVMCExpr::VariantKind Kind;
 945
 946   switch (MO.getTargetFlags()) {
 947   default:
 948     llvm_unreachable("Unknown target flag on GV operand");
 949   case RISCVII::MO_None:
 950     Kind = RISCVMCExpr::VK_RISCV_None;
 951     break;
 952   case RISCVII::MO_CALL:
 953     Kind = RISCVMCExpr::VK_RISCV_CALL_PLT;
 954     break;
 955   case RISCVII::MO_LO:
 956     Kind = RISCVMCExpr::VK_RISCV_LO;
 957     break;
 958   case RISCVII::MO_HI:
 959     Kind = RISCVMCExpr::VK_RISCV_HI;
 960     break;
 961   case RISCVII::MO_PCREL_LO:
 962     Kind = RISCVMCExpr::VK_RISCV_PCREL_LO;
 963     break;
 964   case RISCVII::MO_PCREL_HI:
 965     Kind = RISCVMCExpr::VK_RISCV_PCREL_HI;
 966     break;
 967   case RISCVII::MO_GOT_HI:
 968     Kind = RISCVMCExpr::VK_RISCV_GOT_HI;
 969     break;
 970   case RISCVII::MO_TPREL_LO:
 971     Kind = RISCVMCExpr::VK_RISCV_TPREL_LO;
 972     break;
 973   case RISCVII::MO_TPREL_HI:
 974     Kind = RISCVMCExpr::VK_RISCV_TPREL_HI;
 975     break;
 976   case RISCVII::MO_TPREL_ADD:
 977     Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD;
 978     break;
 979   case RISCVII::MO_TLS_GOT_HI:
 980     Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI;
 981     break;
 982   case RISCVII::MO_TLS_GD_HI:
 983     Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI;
 984     break;
 985   case RISCVII::MO_TLSDESC_HI:
 986     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI;
 987     break;
 988   case RISCVII::MO_TLSDESC_LOAD_LO:
 989     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO;
 990     break;
 991   case RISCVII::MO_TLSDESC_ADD_LO:
 992     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO;
 993     break;
 994   case RISCVII::MO_TLSDESC_CALL:
 995     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL;
 996     break;
 997   }
 998
 999   const MCExpr *ME =
1000       MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
1001
1002   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
1003     ME = MCBinaryExpr::createAdd(
1004         ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
1005
1006   if (Kind != RISCVMCExpr::VK_RISCV_None)
1007     ME = RISCVMCExpr::create(ME, Kind, Ctx);
1008   return MCOperand::createExpr(ME);
1009 }
1010
1011 bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
1012                                    MCOperand &MCOp) const {
1013   switch (MO.getType()) {
1014   default:
1015     report_fatal_error("lowerOperand: unknown operand type");
1016   case MachineOperand::MO_Register:
1017     // Ignore all implicit register operands.
1018     if (MO.isImplicit())
1019       return false;
1020     MCOp = MCOperand::createReg(MO.getReg());
1021     break;
1022   case MachineOperand::MO_RegisterMask:
1023     // Regmasks are like implicit defs.
1024     return false;
1025   case MachineOperand::MO_Immediate:
1026     MCOp = MCOperand::createImm(MO.getImm());
1027     break;
1028   case MachineOperand::MO_MachineBasicBlock:
1029     MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this);
1030     break;
1031   case MachineOperand::MO_GlobalAddress:
1032     MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this);
1033     break;
1034   case MachineOperand::MO_BlockAddress:
1035     MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()),
1036                               *this);
1037     break;
1038   case MachineOperand::MO_ExternalSymbol:
1039     MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()),
1040                               *this);
1041     break;
1042   case MachineOperand::MO_ConstantPoolIndex:
1043     MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this);
1044     break;
1045   case MachineOperand::MO_JumpTableIndex:
1046     MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this);
1047     break;
1048   case MachineOperand::MO_MCSymbol:
1049     MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this);
1050     break;
1051   }
1052   return true;
1053 }
1054
1055 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
1056                                             MCInst &OutMI) {
1057   const RISCVVPseudosTable::PseudoInfo *RVV =
1058       RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
1059   if (!RVV)
1060     return false;
1061
1062   OutMI.setOpcode(RVV->BaseInstr);
1063
1064   const MachineBasicBlock *MBB = MI->getParent();
1065   assert(MBB && "MI expected to be in a basic block");
1066   const MachineFunction *MF = MBB->getParent();
1067   assert(MF && "MBB expected to be in a machine function");
1068
1069   const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
1070   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
1071   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
1072   assert(TRI && "TargetRegisterInfo expected");
1073
1074   const MCInstrDesc &MCID = MI->getDesc();
1075   uint64_t TSFlags = MCID.TSFlags;
1076   unsigned NumOps = MI->getNumExplicitOperands();
1077
1078   // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if
1079   // present.
1080   if (RISCVII::hasVecPolicyOp(TSFlags))
1081     --NumOps;
1082   if (RISCVII::hasSEWOp(TSFlags))
1083     --NumOps;
1084   if (RISCVII::hasVLOp(TSFlags))
1085     --NumOps;
1086   if (RISCVII::hasRoundModeOp(TSFlags))
1087     --NumOps;
1088
1089   bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
1090   for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
1091     const MachineOperand &MO = MI->getOperand(OpNo);
1092     // Skip vl ouput. It should be the second output.
1093     if (hasVLOutput && OpNo == 1)
1094       continue;
1095
1096     // Skip passthru op. It should be the first operand after the defs.
1097     if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) {
1098       assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 &&
1099              "Expected tied to first def.");
1100       const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
1101       // Skip if the next operand in OutMI is not supposed to be tied. Unless it
1102       // is a _TIED instruction.
1103       if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) <
1104               0 &&
1105           !RISCVII::isTiedPseudo(TSFlags))
1106         continue;
1107     }
1108
1109     MCOperand MCOp;
1110     switch (MO.getType()) {
1111     default:
1112       llvm_unreachable("Unknown operand type");
1113     case MachineOperand::MO_Register: {
1114       Register Reg = MO.getReg();
1115
1116       if (RISCV::VRM2RegClass.contains(Reg) ||
1117           RISCV::VRM4RegClass.contains(Reg) ||
1118           RISCV::VRM8RegClass.contains(Reg)) {
1119         Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
1120         assert(Reg && "Subregister does not exist");
1121       } else if (RISCV::FPR16RegClass.contains(Reg)) {
1122         Reg =
1123             TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass);
1124         assert(Reg && "Subregister does not exist");
1125       } else if (RISCV::FPR64RegClass.contains(Reg)) {
1126         Reg = TRI->getSubReg(Reg, RISCV::sub_32);
1127         assert(Reg && "Superregister does not exist");
1128       } else if (RISCV::VRN2M1RegClass.contains(Reg) ||
1129                  RISCV::VRN2M2RegClass.contains(Reg) ||
1130                  RISCV::VRN2M4RegClass.contains(Reg) ||
1131                  RISCV::VRN3M1RegClass.contains(Reg) ||
1132                  RISCV::VRN3M2RegClass.contains(Reg) ||
1133                  RISCV::VRN4M1RegClass.contains(Reg) ||
1134                  RISCV::VRN4M2RegClass.contains(Reg) ||
1135                  RISCV::VRN5M1RegClass.contains(Reg) ||
1136                  RISCV::VRN6M1RegClass.contains(Reg) ||
1137                  RISCV::VRN7M1RegClass.contains(Reg) ||
1138                  RISCV::VRN8M1RegClass.contains(Reg)) {
1139         Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
1140         assert(Reg && "Subregister does not exist");
1141       }
1142
1143       MCOp = MCOperand::createReg(Reg);
1144       break;
1145     }
1146     case MachineOperand::MO_Immediate:
1147       MCOp = MCOperand::createImm(MO.getImm());
1148       break;
1149     }
1150     OutMI.addOperand(MCOp);
1151   }
1152
1153   // Unmasked pseudo instructions need to append dummy mask operand to
1154   // V instructions. All V instructions are modeled as the masked version.
1155   const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
1156   if (OutMI.getNumOperands() < OutMCID.getNumOperands()) {
1157     assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass ==
1158                RISCV::VMV0RegClassID &&
1159            "Expected only mask operand to be missing");
1160     OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister));
1161   }
1162
1163   assert(OutMI.getNumOperands() == OutMCID.getNumOperands());
1164   return true;
1165 }
1166
1167 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
1168   if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
1169     return false;
1170
1171   OutMI.setOpcode(MI->getOpcode());
1172
1173   for (const MachineOperand &MO : MI->operands()) {
1174     MCOperand MCOp;
1175     if (lowerOperand(MO, MCOp))
1176       OutMI.addOperand(MCOp);
1177   }
1178
1179   switch (OutMI.getOpcode()) {
1180   case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
1181     const Function &F = MI->getParent()->getParent()->getFunction();
1182     if (F.hasFnAttribute("patchable-function-entry")) {
1183       unsigned Num;
1184       if (F.getFnAttribute("patchable-function-entry")
1185               .getValueAsString()
1186               .getAsInteger(10, Num))
1187         return false;
1188       emitNops(Num);
1189       return true;
1190     }
1191     break;
1192   }
1193   }
1194   return false;
1195 }
1196
1197 void RISCVAsmPrinter::emitMachineConstantPoolValue(
1198     MachineConstantPoolValue *MCPV) {
1199   auto *RCPV = static_cast<RISCVConstantPoolValue *>(MCPV);
1200   MCSymbol *MCSym;
1201
1202   if (RCPV->isGlobalValue()) {
1203     auto *GV = RCPV->getGlobalValue();
1204     MCSym = getSymbol(GV);
1205   } else {
1206     assert(RCPV->isExtSymbol() && "unrecognized constant pool type");
1207     auto Sym = RCPV->getSymbol();
1208     MCSym = GetExternalSymbolSymbol(Sym);
1209   }
1210
1211   const MCExpr *Expr =
1212       MCSymbolRefExpr::create(MCSym, MCSymbolRefExpr::VK_None, OutContext);
1213   uint64_t Size = getDataLayout().getTypeAllocSize(RCPV->getType());
1214   OutStreamer->emitValue(Expr, Size);
1215 }