llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

   1 //===-- RISCVAsmPrinter.cpp - RISC-V LLVM assembly writer -----------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains a printer that converts from our internal representation
  10 // of machine-dependent LLVM code to the RISC-V assembly language.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "MCTargetDesc/RISCVBaseInfo.h"
  15 #include "MCTargetDesc/RISCVInstPrinter.h"
  16 #include "MCTargetDesc/RISCVMCExpr.h"
  17 #include "MCTargetDesc/RISCVTargetStreamer.h"
  18 #include "RISCV.h"
  19 #include "RISCVMachineFunctionInfo.h"
  20 #include "RISCVTargetMachine.h"
  21 #include "TargetInfo/RISCVTargetInfo.h"
  22 #include "llvm/ADT/APInt.h"
  23 #include "llvm/ADT/Statistic.h"
  24 #include "llvm/BinaryFormat/ELF.h"
  25 #include "llvm/CodeGen/AsmPrinter.h"
  26 #include "llvm/CodeGen/MachineConstantPool.h"
  27 #include "llvm/CodeGen/MachineFunctionPass.h"
  28 #include "llvm/CodeGen/MachineInstr.h"
  29 #include "llvm/CodeGen/MachineModuleInfo.h"
  30 #include "llvm/MC/MCAsmInfo.h"
  31 #include "llvm/MC/MCContext.h"
  32 #include "llvm/MC/MCInst.h"
  33 #include "llvm/MC/MCInstBuilder.h"
  34 #include "llvm/MC/MCObjectFileInfo.h"
  35 #include "llvm/MC/MCSectionELF.h"
  36 #include "llvm/MC/MCStreamer.h"
  37 #include "llvm/MC/MCSymbol.h"
  38 #include "llvm/MC/TargetRegistry.h"
  39 #include "llvm/Support/RISCVISAInfo.h"
  40 #include "llvm/Support/raw_ostream.h"
  41 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
  42
  43 using namespace llvm;
  44
  45 #define DEBUG_TYPE "asm-printer"
  46
  47 STATISTIC(RISCVNumInstrsCompressed,
  48           "Number of RISC-V Compressed instructions emitted");
  49
  50 namespace llvm {
  51 extern const SubtargetFeatureKV RISCVFeatureKV[RISCV::NumSubtargetFeatures];
  52 } // namespace llvm
  53
  54 namespace {
  55 class RISCVAsmPrinter : public AsmPrinter {
  56   const RISCVSubtarget *STI;
  57
  58 public:
  59   explicit RISCVAsmPrinter(TargetMachine &TM,
  60                            std::unique_ptr<MCStreamer> Streamer)
  61       : AsmPrinter(TM, std::move(Streamer)) {}
  62
  63   StringRef getPassName() const override { return "RISC-V Assembly Printer"; }
  64
  65   void LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
  66                      const MachineInstr &MI);
  67
  68   void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  69                        const MachineInstr &MI);
  70
  71   void LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
  72                        const MachineInstr &MI);
  73
  74   bool runOnMachineFunction(MachineFunction &MF) override;
  75
  76   void emitInstruction(const MachineInstr *MI) override;
  77
  78   bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
  79                        const char *ExtraCode, raw_ostream &OS) override;
  80   bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
  81                              const char *ExtraCode, raw_ostream &OS) override;
  82
  83   void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
  84   bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
  85                                    const MachineInstr *MI);
  86
  87   typedef std::tuple<unsigned, uint32_t> HwasanMemaccessTuple;
  88   std::map<HwasanMemaccessTuple, MCSymbol *> HwasanMemaccessSymbols;
  89   void LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI);
  90   void LowerKCFI_CHECK(const MachineInstr &MI);
  91   void EmitHwasanMemaccessSymbols(Module &M);
  92
  93   // Wrapper needed for tblgenned pseudo lowering.
  94   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const;
  95
  96   void emitStartOfAsmFile(Module &M) override;
  97   void emitEndOfAsmFile(Module &M) override;
  98
  99   void emitFunctionEntryLabel() override;
 100   bool emitDirectiveOptionArch();
 101
 102 private:
 103   void emitAttributes();
 104
 105   void emitNTLHint(const MachineInstr *MI);
 106
 107   bool lowerToMCInst(const MachineInstr *MI, MCInst &OutMI);
 108 };
 109 }
 110
 111 void RISCVAsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
 112                                     const MachineInstr &MI) {
 113   unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
 114   unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes();
 115
 116   auto &Ctx = OutStreamer.getContext();
 117   MCSymbol *MILabel = Ctx.createTempSymbol();
 118   OutStreamer.emitLabel(MILabel);
 119
 120   SM.recordStackMap(*MILabel, MI);
 121   assert(NumNOPBytes % NOPBytes == 0 &&
 122          "Invalid number of NOP bytes requested!");
 123
 124   // Scan ahead to trim the shadow.
 125   const MachineBasicBlock &MBB = *MI.getParent();
 126   MachineBasicBlock::const_iterator MII(MI);
 127   ++MII;
 128   while (NumNOPBytes > 0) {
 129     if (MII == MBB.end() || MII->isCall() ||
 130         MII->getOpcode() == RISCV::DBG_VALUE ||
 131         MII->getOpcode() == TargetOpcode::PATCHPOINT ||
 132         MII->getOpcode() == TargetOpcode::STACKMAP)
 133       break;
 134     ++MII;
 135     NumNOPBytes -= 4;
 136   }
 137
 138   // Emit nops.
 139   emitNops(NumNOPBytes / NOPBytes);
 140 }
 141
 142 // Lower a patchpoint of the form:
 143 // [<def>], <id>, <numBytes>, <target>, <numArgs>
 144 void RISCVAsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 145                                       const MachineInstr &MI) {
 146   unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
 147
 148   auto &Ctx = OutStreamer.getContext();
 149   MCSymbol *MILabel = Ctx.createTempSymbol();
 150   OutStreamer.emitLabel(MILabel);
 151   SM.recordPatchPoint(*MILabel, MI);
 152
 153   PatchPointOpers Opers(&MI);
 154
 155   unsigned EncodedBytes = 0;
 156
 157   // Emit padding.
 158   unsigned NumBytes = Opers.getNumPatchBytes();
 159   assert(NumBytes >= EncodedBytes &&
 160          "Patchpoint can't request size less than the length of a call.");
 161   assert((NumBytes - EncodedBytes) % NOPBytes == 0 &&
 162          "Invalid number of NOP bytes requested!");
 163   emitNops((NumBytes - EncodedBytes) / NOPBytes);
 164 }
 165
 166 void RISCVAsmPrinter::LowerSTATEPOINT(MCStreamer &OutStreamer, StackMaps &SM,
 167                                       const MachineInstr &MI) {
 168   unsigned NOPBytes = STI->getFeatureBits()[RISCV::FeatureStdExtC] ? 2 : 4;
 169
 170   StatepointOpers SOpers(&MI);
 171   if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
 172     assert(PatchBytes % NOPBytes == 0 &&
 173            "Invalid number of NOP bytes requested!");
 174     emitNops(PatchBytes / NOPBytes);
 175   }
 176
 177   auto &Ctx = OutStreamer.getContext();
 178   MCSymbol *MILabel = Ctx.createTempSymbol();
 179   OutStreamer.emitLabel(MILabel);
 180   SM.recordStatepoint(*MILabel, MI);
 181 }
 182
 183 void RISCVAsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) {
 184   MCInst CInst;
 185   bool Res = RISCVRVC::compress(CInst, Inst, *STI);
 186   if (Res)
 187     ++RISCVNumInstrsCompressed;
 188   AsmPrinter::EmitToStreamer(*OutStreamer, Res ? CInst : Inst);
 189 }
 190
 191 // Simple pseudo-instructions have their lowering (with expansion to real
 192 // instructions) auto-generated.
 193 #include "RISCVGenMCPseudoLowering.inc"
 194
 195 // If the target supports Zihintntl and the instruction has a nontemporal
 196 // MachineMemOperand, emit an NTLH hint instruction before it.
 197 void RISCVAsmPrinter::emitNTLHint(const MachineInstr *MI) {
 198   if (!STI->hasStdExtZihintntl())
 199     return;
 200
 201   if (MI->memoperands_empty())
 202     return;
 203
 204   MachineMemOperand *MMO = *(MI->memoperands_begin());
 205   if (!MMO->isNonTemporal())
 206     return;
 207
 208   unsigned NontemporalMode = 0;
 209   if (MMO->getFlags() & MONontemporalBit0)
 210     NontemporalMode += 0b1;
 211   if (MMO->getFlags() & MONontemporalBit1)
 212     NontemporalMode += 0b10;
 213
 214   MCInst Hint;
 215   if (STI->hasStdExtCOrZca() && STI->enableRVCHintInstrs())
 216     Hint.setOpcode(RISCV::C_ADD_HINT);
 217   else
 218     Hint.setOpcode(RISCV::ADD);
 219
 220   Hint.addOperand(MCOperand::createReg(RISCV::X0));
 221   Hint.addOperand(MCOperand::createReg(RISCV::X0));
 222   Hint.addOperand(MCOperand::createReg(RISCV::X2 + NontemporalMode));
 223
 224   EmitToStreamer(*OutStreamer, Hint);
 225 }
 226
 227 void RISCVAsmPrinter::emitInstruction(const MachineInstr *MI) {
 228   RISCV_MC::verifyInstructionPredicates(MI->getOpcode(),
 229                                         getSubtargetInfo().getFeatureBits());
 230
 231   emitNTLHint(MI);
 232
 233   // Do any auto-generated pseudo lowerings.
 234   if (emitPseudoExpansionLowering(*OutStreamer, MI))
 235     return;
 236
 237
 238   switch (MI->getOpcode()) {
 239   case RISCV::HWASAN_CHECK_MEMACCESS_SHORTGRANULES:
 240     LowerHWASAN_CHECK_MEMACCESS(*MI);
 241     return;
 242   case RISCV::KCFI_CHECK:
 243     LowerKCFI_CHECK(*MI);
 244     return;
 245   case RISCV::PseudoRVVInitUndefM1:
 246   case RISCV::PseudoRVVInitUndefM2:
 247   case RISCV::PseudoRVVInitUndefM4:
 248   case RISCV::PseudoRVVInitUndefM8:
 249     return;
 250   case TargetOpcode::STACKMAP:
 251     return LowerSTACKMAP(*OutStreamer, SM, *MI);
 252   case TargetOpcode::PATCHPOINT:
 253     return LowerPATCHPOINT(*OutStreamer, SM, *MI);
 254   case TargetOpcode::STATEPOINT:
 255     return LowerSTATEPOINT(*OutStreamer, SM, *MI);
 256   }
 257
 258   MCInst OutInst;
 259   if (!lowerToMCInst(MI, OutInst))
 260     EmitToStreamer(*OutStreamer, OutInst);
 261 }
 262
 263 bool RISCVAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
 264                                       const char *ExtraCode, raw_ostream &OS) {
 265   // First try the generic code, which knows about modifiers like 'c' and 'n'.
 266   if (!AsmPrinter::PrintAsmOperand(MI, OpNo, ExtraCode, OS))
 267     return false;
 268
 269   const MachineOperand &MO = MI->getOperand(OpNo);
 270   if (ExtraCode && ExtraCode[0]) {
 271     if (ExtraCode[1] != 0)
 272       return true; // Unknown modifier.
 273
 274     switch (ExtraCode[0]) {
 275     default:
 276       return true; // Unknown modifier.
 277     case 'z':      // Print zero register if zero, regular printing otherwise.
 278       if (MO.isImm() && MO.getImm() == 0) {
 279         OS << RISCVInstPrinter::getRegisterName(RISCV::X0);
 280         return false;
 281       }
 282       break;
 283     case 'i': // Literal 'i' if operand is not a register.
 284       if (!MO.isReg())
 285         OS << 'i';
 286       return false;
 287     }
 288   }
 289
 290   switch (MO.getType()) {
 291   case MachineOperand::MO_Immediate:
 292     OS << MO.getImm();
 293     return false;
 294   case MachineOperand::MO_Register:
 295     OS << RISCVInstPrinter::getRegisterName(MO.getReg());
 296     return false;
 297   case MachineOperand::MO_GlobalAddress:
 298     PrintSymbolOperand(MO, OS);
 299     return false;
 300   case MachineOperand::MO_BlockAddress: {
 301     MCSymbol *Sym = GetBlockAddressSymbol(MO.getBlockAddress());
 302     Sym->print(OS, MAI);
 303     return false;
 304   }
 305   default:
 306     break;
 307   }
 308
 309   return true;
 310 }
 311
 312 bool RISCVAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
 313                                             unsigned OpNo,
 314                                             const char *ExtraCode,
 315                                             raw_ostream &OS) {
 316   if (ExtraCode)
 317     return AsmPrinter::PrintAsmMemoryOperand(MI, OpNo, ExtraCode, OS);
 318
 319   const MachineOperand &AddrReg = MI->getOperand(OpNo);
 320   assert(MI->getNumOperands() > OpNo + 1 && "Expected additional operand");
 321   const MachineOperand &Offset = MI->getOperand(OpNo + 1);
 322   // All memory operands should have a register and an immediate operand (see
 323   // RISCVDAGToDAGISel::SelectInlineAsmMemoryOperand).
 324   if (!AddrReg.isReg())
 325     return true;
 326   if (!Offset.isImm() && !Offset.isGlobal() && !Offset.isBlockAddress() &&
 327       !Offset.isMCSymbol())
 328     return true;
 329
 330   MCOperand MCO;
 331   if (!lowerOperand(Offset, MCO))
 332     return true;
 333
 334   if (Offset.isImm())
 335     OS << MCO.getImm();
 336   else if (Offset.isGlobal() || Offset.isBlockAddress() || Offset.isMCSymbol())
 337     OS << *MCO.getExpr();
 338   OS << "(" << RISCVInstPrinter::getRegisterName(AddrReg.getReg()) << ")";
 339   return false;
 340 }
 341
 342 bool RISCVAsmPrinter::emitDirectiveOptionArch() {
 343   RISCVTargetStreamer &RTS =
 344       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 345   SmallVector<RISCVOptionArchArg> NeedEmitStdOptionArgs;
 346   const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
 347   for (const auto &Feature : RISCVFeatureKV) {
 348     if (STI->hasFeature(Feature.Value) == MCSTI.hasFeature(Feature.Value))
 349       continue;
 350
 351     if (!llvm::RISCVISAInfo::isSupportedExtensionFeature(Feature.Key))
 352       continue;
 353
 354     auto Delta = STI->hasFeature(Feature.Value) ? RISCVOptionArchArgType::Plus
 355                                                 : RISCVOptionArchArgType::Minus;
 356     NeedEmitStdOptionArgs.emplace_back(Delta, Feature.Key);
 357   }
 358   if (!NeedEmitStdOptionArgs.empty()) {
 359     RTS.emitDirectiveOptionPush();
 360     RTS.emitDirectiveOptionArch(NeedEmitStdOptionArgs);
 361     return true;
 362   }
 363
 364   return false;
 365 }
 366
 367 bool RISCVAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
 368   STI = &MF.getSubtarget<RISCVSubtarget>();
 369   RISCVTargetStreamer &RTS =
 370       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 371
 372   bool EmittedOptionArch = emitDirectiveOptionArch();
 373
 374   SetupMachineFunction(MF);
 375   emitFunctionBody();
 376
 377   if (EmittedOptionArch)
 378     RTS.emitDirectiveOptionPop();
 379   return false;
 380 }
 381
 382 void RISCVAsmPrinter::emitStartOfAsmFile(Module &M) {
 383   RISCVTargetStreamer &RTS =
 384       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 385   if (const MDString *ModuleTargetABI =
 386           dyn_cast_or_null<MDString>(M.getModuleFlag("target-abi")))
 387     RTS.setTargetABI(RISCVABI::getTargetABI(ModuleTargetABI->getString()));
 388   if (TM.getTargetTriple().isOSBinFormatELF())
 389     emitAttributes();
 390 }
 391
 392 void RISCVAsmPrinter::emitEndOfAsmFile(Module &M) {
 393   RISCVTargetStreamer &RTS =
 394       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 395
 396   if (TM.getTargetTriple().isOSBinFormatELF())
 397     RTS.finishAttributeSection();
 398   EmitHwasanMemaccessSymbols(M);
 399 }
 400
 401 void RISCVAsmPrinter::emitAttributes() {
 402   RISCVTargetStreamer &RTS =
 403       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 404   // Use MCSubtargetInfo from TargetMachine. Individual functions may have
 405   // attributes that differ from other functions in the module and we have no
 406   // way to know which function is correct.
 407   RTS.emitTargetAttributes(*TM.getMCSubtargetInfo(), /*EmitStackAlign*/ true);
 408 }
 409
 410 void RISCVAsmPrinter::emitFunctionEntryLabel() {
 411   const auto *RMFI = MF->getInfo<RISCVMachineFunctionInfo>();
 412   if (RMFI->isVectorCall()) {
 413     auto &RTS =
 414         static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 415     RTS.emitDirectiveVariantCC(*CurrentFnSym);
 416   }
 417   return AsmPrinter::emitFunctionEntryLabel();
 418 }
 419
 420 // Force static initialization.
 421 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVAsmPrinter() {
 422   RegisterAsmPrinter<RISCVAsmPrinter> X(getTheRISCV32Target());
 423   RegisterAsmPrinter<RISCVAsmPrinter> Y(getTheRISCV64Target());
 424 }
 425
 426 void RISCVAsmPrinter::LowerHWASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
 427   Register Reg = MI.getOperand(0).getReg();
 428   uint32_t AccessInfo = MI.getOperand(1).getImm();
 429   MCSymbol *&Sym =
 430       HwasanMemaccessSymbols[HwasanMemaccessTuple(Reg, AccessInfo)];
 431   if (!Sym) {
 432     // FIXME: Make this work on non-ELF.
 433     if (!TM.getTargetTriple().isOSBinFormatELF())
 434       report_fatal_error("llvm.hwasan.check.memaccess only supported on ELF");
 435
 436     std::string SymName = "__hwasan_check_x" + utostr(Reg - RISCV::X0) + "_" +
 437                           utostr(AccessInfo) + "_short";
 438     Sym = OutContext.getOrCreateSymbol(SymName);
 439   }
 440   auto Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, OutContext);
 441   auto Expr = RISCVMCExpr::create(Res, RISCVMCExpr::VK_RISCV_CALL, OutContext);
 442
 443   EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr));
 444 }
 445
 446 void RISCVAsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
 447   Register AddrReg = MI.getOperand(0).getReg();
 448   assert(std::next(MI.getIterator())->isCall() &&
 449          "KCFI_CHECK not followed by a call instruction");
 450   assert(std::next(MI.getIterator())->getOperand(0).getReg() == AddrReg &&
 451          "KCFI_CHECK call target doesn't match call operand");
 452
 453   // Temporary registers for comparing the hashes. If a register is used
 454   // for the call target, or reserved by the user, we can clobber another
 455   // temporary register as the check is immediately followed by the
 456   // call. The check defaults to X6/X7, but can fall back to X28-X31 if
 457   // needed.
 458   unsigned ScratchRegs[] = {RISCV::X6, RISCV::X7};
 459   unsigned NextReg = RISCV::X28;
 460   auto isRegAvailable = [&](unsigned Reg) {
 461     return Reg != AddrReg && !STI->isRegisterReservedByUser(Reg);
 462   };
 463   for (auto &Reg : ScratchRegs) {
 464     if (isRegAvailable(Reg))
 465       continue;
 466     while (!isRegAvailable(NextReg))
 467       ++NextReg;
 468     Reg = NextReg++;
 469     if (Reg > RISCV::X31)
 470       report_fatal_error("Unable to find scratch registers for KCFI_CHECK");
 471   }
 472
 473   if (AddrReg == RISCV::X0) {
 474     // Checking X0 makes no sense. Instead of emitting a load, zero
 475     // ScratchRegs[0].
 476     EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::ADDI)
 477                                      .addReg(ScratchRegs[0])
 478                                      .addReg(RISCV::X0)
 479                                      .addImm(0));
 480   } else {
 481     // Adjust the offset for patchable-function-prefix. This assumes that
 482     // patchable-function-prefix is the same for all functions.
 483     int NopSize = STI->hasStdExtCOrZca() ? 2 : 4;
 484     int64_t PrefixNops = 0;
 485     (void)MI.getMF()
 486         ->getFunction()
 487         .getFnAttribute("patchable-function-prefix")
 488         .getValueAsString()
 489         .getAsInteger(10, PrefixNops);
 490
 491     // Load the target function type hash.
 492     EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::LW)
 493                                      .addReg(ScratchRegs[0])
 494                                      .addReg(AddrReg)
 495                                      .addImm(-(PrefixNops * NopSize + 4)));
 496   }
 497
 498   // Load the expected 32-bit type hash.
 499   const int64_t Type = MI.getOperand(1).getImm();
 500   const int64_t Hi20 = ((Type + 0x800) >> 12) & 0xFFFFF;
 501   const int64_t Lo12 = SignExtend64<12>(Type);
 502   if (Hi20) {
 503     EmitToStreamer(
 504         *OutStreamer,
 505         MCInstBuilder(RISCV::LUI).addReg(ScratchRegs[1]).addImm(Hi20));
 506   }
 507   if (Lo12 || Hi20 == 0) {
 508     EmitToStreamer(*OutStreamer,
 509                    MCInstBuilder((STI->hasFeature(RISCV::Feature64Bit) && Hi20)
 510                                      ? RISCV::ADDIW
 511                                      : RISCV::ADDI)
 512                        .addReg(ScratchRegs[1])
 513                        .addReg(ScratchRegs[1])
 514                        .addImm(Lo12));
 515   }
 516
 517   // Compare the hashes and trap if there's a mismatch.
 518   MCSymbol *Pass = OutContext.createTempSymbol();
 519   EmitToStreamer(*OutStreamer,
 520                  MCInstBuilder(RISCV::BEQ)
 521                      .addReg(ScratchRegs[0])
 522                      .addReg(ScratchRegs[1])
 523                      .addExpr(MCSymbolRefExpr::create(Pass, OutContext)));
 524
 525   MCSymbol *Trap = OutContext.createTempSymbol();
 526   OutStreamer->emitLabel(Trap);
 527   EmitToStreamer(*OutStreamer, MCInstBuilder(RISCV::EBREAK));
 528   emitKCFITrapEntry(*MI.getMF(), Trap);
 529   OutStreamer->emitLabel(Pass);
 530 }
 531
 532 void RISCVAsmPrinter::EmitHwasanMemaccessSymbols(Module &M) {
 533   if (HwasanMemaccessSymbols.empty())
 534     return;
 535
 536   assert(TM.getTargetTriple().isOSBinFormatELF());
 537   // Use MCSubtargetInfo from TargetMachine. Individual functions may have
 538   // attributes that differ from other functions in the module and we have no
 539   // way to know which function is correct.
 540   const MCSubtargetInfo &MCSTI = *TM.getMCSubtargetInfo();
 541
 542   MCSymbol *HwasanTagMismatchV2Sym =
 543       OutContext.getOrCreateSymbol("__hwasan_tag_mismatch_v2");
 544   // Annotate symbol as one having incompatible calling convention, so
 545   // run-time linkers can instead eagerly bind this function.
 546   auto &RTS =
 547       static_cast<RISCVTargetStreamer &>(*OutStreamer->getTargetStreamer());
 548   RTS.emitDirectiveVariantCC(*HwasanTagMismatchV2Sym);
 549
 550   const MCSymbolRefExpr *HwasanTagMismatchV2Ref =
 551       MCSymbolRefExpr::create(HwasanTagMismatchV2Sym, OutContext);
 552   auto Expr = RISCVMCExpr::create(HwasanTagMismatchV2Ref,
 553                                   RISCVMCExpr::VK_RISCV_CALL, OutContext);
 554
 555   for (auto &P : HwasanMemaccessSymbols) {
 556     unsigned Reg = std::get<0>(P.first);
 557     uint32_t AccessInfo = std::get<1>(P.first);
 558     MCSymbol *Sym = P.second;
 559
 560     unsigned Size =
 561         1 << ((AccessInfo >> HWASanAccessInfo::AccessSizeShift) & 0xf);
 562     OutStreamer->switchSection(OutContext.getELFSection(
 563         ".text.hot", ELF::SHT_PROGBITS,
 564         ELF::SHF_EXECINSTR | ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, Sym->getName(),
 565         /*IsComdat=*/true));
 566
 567     OutStreamer->emitSymbolAttribute(Sym, MCSA_ELF_TypeFunction);
 568     OutStreamer->emitSymbolAttribute(Sym, MCSA_Weak);
 569     OutStreamer->emitSymbolAttribute(Sym, MCSA_Hidden);
 570     OutStreamer->emitLabel(Sym);
 571
 572     // Extract shadow offset from ptr
 573     OutStreamer->emitInstruction(
 574         MCInstBuilder(RISCV::SLLI).addReg(RISCV::X6).addReg(Reg).addImm(8),
 575         MCSTI);
 576     OutStreamer->emitInstruction(MCInstBuilder(RISCV::SRLI)
 577                                      .addReg(RISCV::X6)
 578                                      .addReg(RISCV::X6)
 579                                      .addImm(12),
 580                                  MCSTI);
 581     // load shadow tag in X6, X5 contains shadow base
 582     OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADD)
 583                                      .addReg(RISCV::X6)
 584                                      .addReg(RISCV::X5)
 585                                      .addReg(RISCV::X6),
 586                                  MCSTI);
 587     OutStreamer->emitInstruction(
 588         MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
 589         MCSTI);
 590     // Extract tag from X5 and compare it with loaded tag from shadow
 591     OutStreamer->emitInstruction(
 592         MCInstBuilder(RISCV::SRLI).addReg(RISCV::X7).addReg(Reg).addImm(56),
 593         MCSTI);
 594     MCSymbol *HandleMismatchOrPartialSym = OutContext.createTempSymbol();
 595     // X7 contains tag from memory, while X6 contains tag from the pointer
 596     OutStreamer->emitInstruction(
 597         MCInstBuilder(RISCV::BNE)
 598             .addReg(RISCV::X7)
 599             .addReg(RISCV::X6)
 600             .addExpr(MCSymbolRefExpr::create(HandleMismatchOrPartialSym,
 601                                              OutContext)),
 602         MCSTI);
 603     MCSymbol *ReturnSym = OutContext.createTempSymbol();
 604     OutStreamer->emitLabel(ReturnSym);
 605     OutStreamer->emitInstruction(MCInstBuilder(RISCV::JALR)
 606                                      .addReg(RISCV::X0)
 607                                      .addReg(RISCV::X1)
 608                                      .addImm(0),
 609                                  MCSTI);
 610     OutStreamer->emitLabel(HandleMismatchOrPartialSym);
 611
 612     OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
 613                                      .addReg(RISCV::X28)
 614                                      .addReg(RISCV::X0)
 615                                      .addImm(16),
 616                                  MCSTI);
 617     MCSymbol *HandleMismatchSym = OutContext.createTempSymbol();
 618     OutStreamer->emitInstruction(
 619         MCInstBuilder(RISCV::BGEU)
 620             .addReg(RISCV::X6)
 621             .addReg(RISCV::X28)
 622             .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 623         MCSTI);
 624
 625     OutStreamer->emitInstruction(
 626         MCInstBuilder(RISCV::ANDI).addReg(RISCV::X28).addReg(Reg).addImm(0xF),
 627         MCSTI);
 628
 629     if (Size != 1)
 630       OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
 631                                        .addReg(RISCV::X28)
 632                                        .addReg(RISCV::X28)
 633                                        .addImm(Size - 1),
 634                                    MCSTI);
 635     OutStreamer->emitInstruction(
 636         MCInstBuilder(RISCV::BGE)
 637             .addReg(RISCV::X28)
 638             .addReg(RISCV::X6)
 639             .addExpr(MCSymbolRefExpr::create(HandleMismatchSym, OutContext)),
 640         MCSTI);
 641
 642     OutStreamer->emitInstruction(
 643         MCInstBuilder(RISCV::ORI).addReg(RISCV::X6).addReg(Reg).addImm(0xF),
 644         MCSTI);
 645     OutStreamer->emitInstruction(
 646         MCInstBuilder(RISCV::LBU).addReg(RISCV::X6).addReg(RISCV::X6).addImm(0),
 647         MCSTI);
 648     OutStreamer->emitInstruction(
 649         MCInstBuilder(RISCV::BEQ)
 650             .addReg(RISCV::X6)
 651             .addReg(RISCV::X7)
 652             .addExpr(MCSymbolRefExpr::create(ReturnSym, OutContext)),
 653         MCSTI);
 654
 655     OutStreamer->emitLabel(HandleMismatchSym);
 656
 657     // | Previous stack frames...        |
 658     // +=================================+ <-- [SP + 256]
 659     // |              ...                |
 660     // |                                 |
 661     // | Stack frame space for x12 - x31.|
 662     // |                                 |
 663     // |              ...                |
 664     // +---------------------------------+ <-- [SP + 96]
 665     // | Saved x11(arg1), as             |
 666     // | __hwasan_check_* clobbers it.   |
 667     // +---------------------------------+ <-- [SP + 88]
 668     // | Saved x10(arg0), as             |
 669     // | __hwasan_check_* clobbers it.   |
 670     // +---------------------------------+ <-- [SP + 80]
 671     // |                                 |
 672     // | Stack frame space for x9.       |
 673     // +---------------------------------+ <-- [SP + 72]
 674     // |                                 |
 675     // | Saved x8(fp), as                |
 676     // | __hwasan_check_* clobbers it.   |
 677     // +---------------------------------+ <-- [SP + 64]
 678     // |              ...                |
 679     // |                                 |
 680     // | Stack frame space for x2 - x7.  |
 681     // |                                 |
 682     // |              ...                |
 683     // +---------------------------------+ <-- [SP + 16]
 684     // | Return address (x1) for caller  |
 685     // | of __hwasan_check_*.            |
 686     // +---------------------------------+ <-- [SP + 8]
 687     // | Reserved place for x0, possibly |
 688     // | junk, since we don't save it.   |
 689     // +---------------------------------+ <-- [x2 / SP]
 690
 691     // Adjust sp
 692     OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
 693                                      .addReg(RISCV::X2)
 694                                      .addReg(RISCV::X2)
 695                                      .addImm(-256),
 696                                  MCSTI);
 697
 698     // store x10(arg0) by new sp
 699     OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
 700                                      .addReg(RISCV::X10)
 701                                      .addReg(RISCV::X2)
 702                                      .addImm(8 * 10),
 703                                  MCSTI);
 704     // store x11(arg1) by new sp
 705     OutStreamer->emitInstruction(MCInstBuilder(RISCV::SD)
 706                                      .addReg(RISCV::X11)
 707                                      .addReg(RISCV::X2)
 708                                      .addImm(8 * 11),
 709                                  MCSTI);
 710
 711     // store x8(fp) by new sp
 712     OutStreamer->emitInstruction(
 713         MCInstBuilder(RISCV::SD).addReg(RISCV::X8).addReg(RISCV::X2).addImm(8 *
 714                                                                             8),
 715         MCSTI);
 716     // store x1(ra) by new sp
 717     OutStreamer->emitInstruction(
 718         MCInstBuilder(RISCV::SD).addReg(RISCV::X1).addReg(RISCV::X2).addImm(1 *
 719                                                                             8),
 720         MCSTI);
 721     if (Reg != RISCV::X10)
 722       OutStreamer->emitInstruction(MCInstBuilder(RISCV::ADDI)
 723                                        .addReg(RISCV::X10)
 724                                        .addReg(Reg)
 725                                        .addImm(0),
 726                                    MCSTI);
 727     OutStreamer->emitInstruction(
 728         MCInstBuilder(RISCV::ADDI)
 729             .addReg(RISCV::X11)
 730             .addReg(RISCV::X0)
 731             .addImm(AccessInfo & HWASanAccessInfo::RuntimeMask),
 732         MCSTI);
 733
 734     OutStreamer->emitInstruction(MCInstBuilder(RISCV::PseudoCALL).addExpr(Expr),
 735                                  MCSTI);
 736   }
 737 }
 738
 739 static MCOperand lowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym,
 740                                     const AsmPrinter &AP) {
 741   MCContext &Ctx = AP.OutContext;
 742   RISCVMCExpr::VariantKind Kind;
 743
 744   switch (MO.getTargetFlags()) {
 745   default:
 746     llvm_unreachable("Unknown target flag on GV operand");
 747   case RISCVII::MO_None:
 748     Kind = RISCVMCExpr::VK_RISCV_None;
 749     break;
 750   case RISCVII::MO_CALL:
 751     Kind = RISCVMCExpr::VK_RISCV_CALL_PLT;
 752     break;
 753   case RISCVII::MO_LO:
 754     Kind = RISCVMCExpr::VK_RISCV_LO;
 755     break;
 756   case RISCVII::MO_HI:
 757     Kind = RISCVMCExpr::VK_RISCV_HI;
 758     break;
 759   case RISCVII::MO_PCREL_LO:
 760     Kind = RISCVMCExpr::VK_RISCV_PCREL_LO;
 761     break;
 762   case RISCVII::MO_PCREL_HI:
 763     Kind = RISCVMCExpr::VK_RISCV_PCREL_HI;
 764     break;
 765   case RISCVII::MO_GOT_HI:
 766     Kind = RISCVMCExpr::VK_RISCV_GOT_HI;
 767     break;
 768   case RISCVII::MO_TPREL_LO:
 769     Kind = RISCVMCExpr::VK_RISCV_TPREL_LO;
 770     break;
 771   case RISCVII::MO_TPREL_HI:
 772     Kind = RISCVMCExpr::VK_RISCV_TPREL_HI;
 773     break;
 774   case RISCVII::MO_TPREL_ADD:
 775     Kind = RISCVMCExpr::VK_RISCV_TPREL_ADD;
 776     break;
 777   case RISCVII::MO_TLS_GOT_HI:
 778     Kind = RISCVMCExpr::VK_RISCV_TLS_GOT_HI;
 779     break;
 780   case RISCVII::MO_TLS_GD_HI:
 781     Kind = RISCVMCExpr::VK_RISCV_TLS_GD_HI;
 782     break;
 783   case RISCVII::MO_TLSDESC_HI:
 784     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_HI;
 785     break;
 786   case RISCVII::MO_TLSDESC_LOAD_LO:
 787     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_LOAD_LO;
 788     break;
 789   case RISCVII::MO_TLSDESC_ADD_LO:
 790     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_ADD_LO;
 791     break;
 792   case RISCVII::MO_TLSDESC_CALL:
 793     Kind = RISCVMCExpr::VK_RISCV_TLSDESC_CALL;
 794     break;
 795   }
 796
 797   const MCExpr *ME =
 798       MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_None, Ctx);
 799
 800   if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
 801     ME = MCBinaryExpr::createAdd(
 802         ME, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
 803
 804   if (Kind != RISCVMCExpr::VK_RISCV_None)
 805     ME = RISCVMCExpr::create(ME, Kind, Ctx);
 806   return MCOperand::createExpr(ME);
 807 }
 808
 809 bool RISCVAsmPrinter::lowerOperand(const MachineOperand &MO,
 810                                    MCOperand &MCOp) const {
 811   switch (MO.getType()) {
 812   default:
 813     report_fatal_error("lowerOperand: unknown operand type");
 814   case MachineOperand::MO_Register:
 815     // Ignore all implicit register operands.
 816     if (MO.isImplicit())
 817       return false;
 818     MCOp = MCOperand::createReg(MO.getReg());
 819     break;
 820   case MachineOperand::MO_RegisterMask:
 821     // Regmasks are like implicit defs.
 822     return false;
 823   case MachineOperand::MO_Immediate:
 824     MCOp = MCOperand::createImm(MO.getImm());
 825     break;
 826   case MachineOperand::MO_MachineBasicBlock:
 827     MCOp = lowerSymbolOperand(MO, MO.getMBB()->getSymbol(), *this);
 828     break;
 829   case MachineOperand::MO_GlobalAddress:
 830     MCOp = lowerSymbolOperand(MO, getSymbolPreferLocal(*MO.getGlobal()), *this);
 831     break;
 832   case MachineOperand::MO_BlockAddress:
 833     MCOp = lowerSymbolOperand(MO, GetBlockAddressSymbol(MO.getBlockAddress()),
 834                               *this);
 835     break;
 836   case MachineOperand::MO_ExternalSymbol:
 837     MCOp = lowerSymbolOperand(MO, GetExternalSymbolSymbol(MO.getSymbolName()),
 838                               *this);
 839     break;
 840   case MachineOperand::MO_ConstantPoolIndex:
 841     MCOp = lowerSymbolOperand(MO, GetCPISymbol(MO.getIndex()), *this);
 842     break;
 843   case MachineOperand::MO_JumpTableIndex:
 844     MCOp = lowerSymbolOperand(MO, GetJTISymbol(MO.getIndex()), *this);
 845     break;
 846   case MachineOperand::MO_MCSymbol:
 847     MCOp = lowerSymbolOperand(MO, MO.getMCSymbol(), *this);
 848     break;
 849   }
 850   return true;
 851 }
 852
 853 static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
 854                                             MCInst &OutMI) {
 855   const RISCVVPseudosTable::PseudoInfo *RVV =
 856       RISCVVPseudosTable::getPseudoInfo(MI->getOpcode());
 857   if (!RVV)
 858     return false;
 859
 860   OutMI.setOpcode(RVV->BaseInstr);
 861
 862   const MachineBasicBlock *MBB = MI->getParent();
 863   assert(MBB && "MI expected to be in a basic block");
 864   const MachineFunction *MF = MBB->getParent();
 865   assert(MF && "MBB expected to be in a machine function");
 866
 867   const RISCVSubtarget &Subtarget = MF->getSubtarget<RISCVSubtarget>();
 868   const TargetInstrInfo *TII = Subtarget.getInstrInfo();
 869   const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
 870   assert(TRI && "TargetRegisterInfo expected");
 871
 872   const MCInstrDesc &MCID = MI->getDesc();
 873   uint64_t TSFlags = MCID.TSFlags;
 874   unsigned NumOps = MI->getNumExplicitOperands();
 875
 876   // Skip policy, SEW, VL, VXRM/FRM operands which are the last operands if
 877   // present.
 878   if (RISCVII::hasVecPolicyOp(TSFlags))
 879     --NumOps;
 880   if (RISCVII::hasSEWOp(TSFlags))
 881     --NumOps;
 882   if (RISCVII::hasVLOp(TSFlags))
 883     --NumOps;
 884   if (RISCVII::hasRoundModeOp(TSFlags))
 885     --NumOps;
 886
 887   bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
 888   for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {
 889     const MachineOperand &MO = MI->getOperand(OpNo);
 890     // Skip vl ouput. It should be the second output.
 891     if (hasVLOutput && OpNo == 1)
 892       continue;
 893
 894     // Skip merge op. It should be the first operand after the defs.
 895     if (OpNo == MI->getNumExplicitDefs() && MO.isReg() && MO.isTied()) {
 896       assert(MCID.getOperandConstraint(OpNo, MCOI::TIED_TO) == 0 &&
 897              "Expected tied to first def.");
 898       const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
 899       // Skip if the next operand in OutMI is not supposed to be tied. Unless it
 900       // is a _TIED instruction.
 901       if (OutMCID.getOperandConstraint(OutMI.getNumOperands(), MCOI::TIED_TO) <
 902               0 &&
 903           !RISCVII::isTiedPseudo(TSFlags))
 904         continue;
 905     }
 906
 907     MCOperand MCOp;
 908     switch (MO.getType()) {
 909     default:
 910       llvm_unreachable("Unknown operand type");
 911     case MachineOperand::MO_Register: {
 912       Register Reg = MO.getReg();
 913
 914       if (RISCV::VRM2RegClass.contains(Reg) ||
 915           RISCV::VRM4RegClass.contains(Reg) ||
 916           RISCV::VRM8RegClass.contains(Reg)) {
 917         Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
 918         assert(Reg && "Subregister does not exist");
 919       } else if (RISCV::FPR16RegClass.contains(Reg)) {
 920         Reg =
 921             TRI->getMatchingSuperReg(Reg, RISCV::sub_16, &RISCV::FPR32RegClass);
 922         assert(Reg && "Subregister does not exist");
 923       } else if (RISCV::FPR64RegClass.contains(Reg)) {
 924         Reg = TRI->getSubReg(Reg, RISCV::sub_32);
 925         assert(Reg && "Superregister does not exist");
 926       } else if (RISCV::VRN2M1RegClass.contains(Reg) ||
 927                  RISCV::VRN2M2RegClass.contains(Reg) ||
 928                  RISCV::VRN2M4RegClass.contains(Reg) ||
 929                  RISCV::VRN3M1RegClass.contains(Reg) ||
 930                  RISCV::VRN3M2RegClass.contains(Reg) ||
 931                  RISCV::VRN4M1RegClass.contains(Reg) ||
 932                  RISCV::VRN4M2RegClass.contains(Reg) ||
 933                  RISCV::VRN5M1RegClass.contains(Reg) ||
 934                  RISCV::VRN6M1RegClass.contains(Reg) ||
 935                  RISCV::VRN7M1RegClass.contains(Reg) ||
 936                  RISCV::VRN8M1RegClass.contains(Reg)) {
 937         Reg = TRI->getSubReg(Reg, RISCV::sub_vrm1_0);
 938         assert(Reg && "Subregister does not exist");
 939       }
 940
 941       MCOp = MCOperand::createReg(Reg);
 942       break;
 943     }
 944     case MachineOperand::MO_Immediate:
 945       MCOp = MCOperand::createImm(MO.getImm());
 946       break;
 947     }
 948     OutMI.addOperand(MCOp);
 949   }
 950
 951   // Unmasked pseudo instructions need to append dummy mask operand to
 952   // V instructions. All V instructions are modeled as the masked version.
 953   const MCInstrDesc &OutMCID = TII->get(OutMI.getOpcode());
 954   if (OutMI.getNumOperands() < OutMCID.getNumOperands()) {
 955     assert(OutMCID.operands()[OutMI.getNumOperands()].RegClass ==
 956                RISCV::VMV0RegClassID &&
 957            "Expected only mask operand to be missing");
 958     OutMI.addOperand(MCOperand::createReg(RISCV::NoRegister));
 959   }
 960
 961   assert(OutMI.getNumOperands() == OutMCID.getNumOperands());
 962   return true;
 963 }
 964
 965 bool RISCVAsmPrinter::lowerToMCInst(const MachineInstr *MI, MCInst &OutMI) {
 966   if (lowerRISCVVMachineInstrToMCInst(MI, OutMI))
 967     return false;
 968
 969   OutMI.setOpcode(MI->getOpcode());
 970
 971   for (const MachineOperand &MO : MI->operands()) {
 972     MCOperand MCOp;
 973     if (lowerOperand(MO, MCOp))
 974       OutMI.addOperand(MCOp);
 975   }
 976
 977   switch (OutMI.getOpcode()) {
 978   case TargetOpcode::PATCHABLE_FUNCTION_ENTER: {
 979     const Function &F = MI->getParent()->getParent()->getFunction();
 980     if (F.hasFnAttribute("patchable-function-entry")) {
 981       unsigned Num;
 982       if (F.getFnAttribute("patchable-function-entry")
 983               .getValueAsString()
 984               .getAsInteger(10, Num))
 985         return false;
 986       emitNops(Num);
 987       return true;
 988     }
 989     break;
 990   }
 991   }
 992   return false;
 993 }