llvm/tools/llvm-exegesis/lib/Assembler.cpp

   1 //===-- Assembler.cpp -------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "Assembler.h"
  10
  11 #include "SnippetRepetitor.h"
  12 #include "SubprocessMemory.h"
  13 #include "Target.h"
  14 #include "llvm/Analysis/TargetLibraryInfo.h"
  15 #include "llvm/CodeGen/FunctionLoweringInfo.h"
  16 #include "llvm/CodeGen/GlobalISel/CallLowering.h"
  17 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
  18 #include "llvm/CodeGen/MachineInstrBuilder.h"
  19 #include "llvm/CodeGen/MachineModuleInfo.h"
  20 #include "llvm/CodeGen/MachineRegisterInfo.h"
  21 #include "llvm/CodeGen/TargetInstrInfo.h"
  22 #include "llvm/CodeGen/TargetLowering.h"
  23 #include "llvm/CodeGen/TargetPassConfig.h"
  24 #include "llvm/CodeGen/TargetSubtargetInfo.h"
  25 #include "llvm/ExecutionEngine/Orc/LLJIT.h"
  26 #include "llvm/IR/BasicBlock.h"
  27 #include "llvm/IR/Instructions.h"
  28 #include "llvm/IR/LegacyPassManager.h"
  29 #include "llvm/MC/MCInstrInfo.h"
  30 #include "llvm/Object/SymbolSize.h"
  31 #include "llvm/Support/Alignment.h"
  32 #include "llvm/Support/MemoryBuffer.h"
  33 #include "llvm/Support/raw_ostream.h"
  34
  35 #ifdef HAVE_LIBPFM
  36 #include "perfmon/perf_event.h"
  37 #endif // HAVE_LIBPFM
  38
  39 #ifdef __linux__
  40 #include <unistd.h>
  41 #endif
  42
  43 namespace llvm {
  44 namespace exegesis {
  45
  46 static constexpr const char ModuleID[] = "ExegesisInfoTest";
  47 static constexpr const char FunctionID[] = "foo";
  48 static const Align kFunctionAlignment(4096);
  49
  50 // Fills the given basic block with register setup code, and returns true if
  51 // all registers could be setup correctly.
  52 static bool generateSnippetSetupCode(const ExegesisTarget &ET,
  53                                      const MCSubtargetInfo *const MSI,
  54                                      BasicBlockFiller &BBF,
  55                                      const BenchmarkKey &Key,
  56                                      bool GenerateMemoryInstructions) {
  57   bool IsSnippetSetupComplete = true;
  58   if (GenerateMemoryInstructions) {
  59     BBF.addInstructions(ET.generateMemoryInitialSetup());
  60     for (const MemoryMapping &MM : Key.MemoryMappings) {
  61 #ifdef __linux__
  62       // The frontend that generates that parses the memory mapping information
  63       // from the user should validate that the requested address is a multiple
  64       // of the page size. Assert that this is true here.
  65       assert(MM.Address % getpagesize() == 0 &&
  66              "Memory mappings need to be aligned to page boundaries.");
  67 #endif
  68       BBF.addInstructions(ET.generateMmap(
  69           MM.Address, Key.MemoryValues.at(MM.MemoryValueName).SizeBytes,
  70           ET.getAuxiliaryMemoryStartAddress() +
  71               sizeof(int) * (Key.MemoryValues.at(MM.MemoryValueName).Index +
  72                              SubprocessMemory::AuxiliaryMemoryOffset)));
  73     }
  74     BBF.addInstructions(ET.setStackRegisterToAuxMem());
  75   }
  76   Register StackPointerRegister = BBF.MF.getSubtarget()
  77                                       .getTargetLowering()
  78                                       ->getStackPointerRegisterToSaveRestore();
  79   for (const RegisterValue &RV : Key.RegisterInitialValues) {
  80     if (GenerateMemoryInstructions) {
  81       // If we're generating memory instructions, don't load in the value for
  82       // the register with the stack pointer as it will be used later to finish
  83       // the setup.
  84       if (RV.Register == StackPointerRegister)
  85         continue;
  86     }
  87     // Load a constant in the register.
  88     const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value);
  89     if (SetRegisterCode.empty())
  90       IsSnippetSetupComplete = false;
  91     BBF.addInstructions(SetRegisterCode);
  92   }
  93   if (GenerateMemoryInstructions) {
  94 #ifdef HAVE_LIBPFM
  95     BBF.addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_RESET, true));
  96 #endif // HAVE_LIBPFM
  97     for (const RegisterValue &RV : Key.RegisterInitialValues) {
  98       // Load in the stack register now as we're done using it elsewhere
  99       // and need to set the value in preparation for executing the
 100       // snippet.
 101       if (RV.Register != StackPointerRegister)
 102         continue;
 103       const auto SetRegisterCode = ET.setRegTo(*MSI, RV.Register, RV.Value);
 104       if (SetRegisterCode.empty())
 105         IsSnippetSetupComplete = false;
 106       BBF.addInstructions(SetRegisterCode);
 107       break;
 108     }
 109   }
 110   return IsSnippetSetupComplete;
 111 }
 112
 113 // Small utility function to add named passes.
 114 static bool addPass(PassManagerBase &PM, StringRef PassName,
 115                     TargetPassConfig &TPC) {
 116   const PassRegistry *PR = PassRegistry::getPassRegistry();
 117   const PassInfo *PI = PR->getPassInfo(PassName);
 118   if (!PI) {
 119     errs() << " run-pass " << PassName << " is not registered.\n";
 120     return true;
 121   }
 122
 123   if (!PI->getNormalCtor()) {
 124     errs() << " cannot create pass: " << PI->getPassName() << "\n";
 125     return true;
 126   }
 127   Pass *P = PI->getNormalCtor()();
 128   std::string Banner = std::string("After ") + std::string(P->getPassName());
 129   PM.add(P);
 130   TPC.printAndVerify(Banner);
 131
 132   return false;
 133 }
 134
 135 MachineFunction &createVoidVoidPtrMachineFunction(StringRef FunctionName,
 136                                                   Module *Module,
 137                                                   MachineModuleInfo *MMI) {
 138   Type *const ReturnType = Type::getInt32Ty(Module->getContext());
 139   Type *const MemParamType = PointerType::get(
 140       Type::getInt8Ty(Module->getContext()), 0 /*default address space*/);
 141   FunctionType *FunctionType =
 142       FunctionType::get(ReturnType, {MemParamType}, false);
 143   Function *const F = Function::Create(
 144       FunctionType, GlobalValue::ExternalLinkage, FunctionName, Module);
 145   BasicBlock *BB = BasicBlock::Create(Module->getContext(), "", F);
 146   new UnreachableInst(Module->getContext(), BB);
 147   return MMI->getOrCreateMachineFunction(*F);
 148 }
 149
 150 BasicBlockFiller::BasicBlockFiller(MachineFunction &MF, MachineBasicBlock *MBB,
 151                                    const MCInstrInfo *MCII)
 152     : MF(MF), MBB(MBB), MCII(MCII) {}
 153
 154 void BasicBlockFiller::addInstruction(const MCInst &Inst, const DebugLoc &DL) {
 155   const unsigned Opcode = Inst.getOpcode();
 156   const MCInstrDesc &MCID = MCII->get(Opcode);
 157   MachineInstrBuilder Builder = BuildMI(MBB, DL, MCID);
 158   for (unsigned OpIndex = 0, E = Inst.getNumOperands(); OpIndex < E;
 159        ++OpIndex) {
 160     const MCOperand &Op = Inst.getOperand(OpIndex);
 161     if (Op.isReg()) {
 162       const bool IsDef = OpIndex < MCID.getNumDefs();
 163       unsigned Flags = 0;
 164       const MCOperandInfo &OpInfo = MCID.operands().begin()[OpIndex];
 165       if (IsDef && !OpInfo.isOptionalDef())
 166         Flags |= RegState::Define;
 167       Builder.addReg(Op.getReg(), Flags);
 168     } else if (Op.isImm()) {
 169       Builder.addImm(Op.getImm());
 170     } else if (!Op.isValid()) {
 171       llvm_unreachable("Operand is not set");
 172     } else {
 173       llvm_unreachable("Not yet implemented");
 174     }
 175   }
 176 }
 177
 178 void BasicBlockFiller::addInstructions(ArrayRef<MCInst> Insts,
 179                                        const DebugLoc &DL) {
 180   for (const MCInst &Inst : Insts)
 181     addInstruction(Inst, DL);
 182 }
 183
 184 void BasicBlockFiller::addReturn(const ExegesisTarget &ET,
 185                                  bool SubprocessCleanup, const DebugLoc &DL) {
 186   // Insert cleanup code
 187   if (SubprocessCleanup) {
 188 #ifdef HAVE_LIBPFM
 189     addInstructions(ET.configurePerfCounter(PERF_EVENT_IOC_DISABLE, false));
 190 #endif // HAVE_LIBPFM
 191 #ifdef __linux__
 192     addInstructions(ET.generateExitSyscall(0));
 193 #endif // __linux__
 194   }
 195   // Insert the return code.
 196   const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
 197   if (TII->getReturnOpcode() < TII->getNumOpcodes()) {
 198     BuildMI(MBB, DL, TII->get(TII->getReturnOpcode()));
 199   } else {
 200     MachineIRBuilder MIB(MF);
 201     MIB.setMBB(*MBB);
 202
 203     FunctionLoweringInfo FuncInfo;
 204     FuncInfo.CanLowerReturn = true;
 205     MF.getSubtarget().getCallLowering()->lowerReturn(MIB, nullptr, {}, FuncInfo,
 206                                                      0);
 207   }
 208 }
 209
 210 FunctionFiller::FunctionFiller(MachineFunction &MF,
 211                                std::vector<unsigned> RegistersSetUp)
 212     : MF(MF), MCII(MF.getTarget().getMCInstrInfo()), Entry(addBasicBlock()),
 213       RegistersSetUp(std::move(RegistersSetUp)) {}
 214
 215 BasicBlockFiller FunctionFiller::addBasicBlock() {
 216   MachineBasicBlock *MBB = MF.CreateMachineBasicBlock();
 217   MF.push_back(MBB);
 218   return BasicBlockFiller(MF, MBB, MCII);
 219 }
 220
 221 ArrayRef<unsigned> FunctionFiller::getRegistersSetUp() const {
 222   return RegistersSetUp;
 223 }
 224
 225 static std::unique_ptr<Module>
 226 createModule(const std::unique_ptr<LLVMContext> &Context, const DataLayout &DL) {
 227   auto Mod = std::make_unique<Module>(ModuleID, *Context);
 228   Mod->setDataLayout(DL);
 229   return Mod;
 230 }
 231
 232 BitVector getFunctionReservedRegs(const TargetMachine &TM) {
 233   std::unique_ptr<LLVMContext> Context = std::make_unique<LLVMContext>();
 234   std::unique_ptr<Module> Module = createModule(Context, TM.createDataLayout());
 235   auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(&TM);
 236   MachineFunction &MF = createVoidVoidPtrMachineFunction(
 237       FunctionID, Module.get(), &MMIWP->getMMI());
 238   // Saving reserved registers for client.
 239   return MF.getSubtarget().getRegisterInfo()->getReservedRegs(MF);
 240 }
 241
 242 Error assembleToStream(const ExegesisTarget &ET,
 243                        std::unique_ptr<TargetMachine> TM,
 244                        ArrayRef<unsigned> LiveIns, const FillFunction &Fill,
 245                        raw_pwrite_stream &AsmStream, const BenchmarkKey &Key,
 246                        bool GenerateMemoryInstructions) {
 247   auto Context = std::make_unique<LLVMContext>();
 248   std::unique_ptr<Module> Module =
 249       createModule(Context, TM->createDataLayout());
 250   auto MMIWP = std::make_unique<MachineModuleInfoWrapperPass>(TM.get());
 251   MachineFunction &MF = createVoidVoidPtrMachineFunction(
 252       FunctionID, Module.get(), &MMIWP.get()->getMMI());
 253   MF.ensureAlignment(kFunctionAlignment);
 254
 255   // We need to instruct the passes that we're done with SSA and virtual
 256   // registers.
 257   auto &Properties = MF.getProperties();
 258   Properties.set(MachineFunctionProperties::Property::NoVRegs);
 259   Properties.reset(MachineFunctionProperties::Property::IsSSA);
 260   Properties.set(MachineFunctionProperties::Property::NoPHIs);
 261
 262   for (const unsigned Reg : LiveIns)
 263     MF.getRegInfo().addLiveIn(Reg);
 264
 265   if (GenerateMemoryInstructions) {
 266     for (const unsigned Reg : ET.getArgumentRegisters())
 267       MF.getRegInfo().addLiveIn(Reg);
 268     // Add a live in for registers that need saving so that the machine verifier
 269     // doesn't fail if the register is never defined.
 270     for (const unsigned Reg : ET.getRegistersNeedSaving())
 271       MF.getRegInfo().addLiveIn(Reg);
 272   }
 273
 274   std::vector<unsigned> RegistersSetUp;
 275   RegistersSetUp.reserve(Key.RegisterInitialValues.size());
 276   for (const auto &InitValue : Key.RegisterInitialValues) {
 277     RegistersSetUp.push_back(InitValue.Register);
 278   }
 279   FunctionFiller Sink(MF, std::move(RegistersSetUp));
 280   auto Entry = Sink.getEntry();
 281
 282   for (const unsigned Reg : LiveIns)
 283     Entry.MBB->addLiveIn(Reg);
 284
 285   if (GenerateMemoryInstructions) {
 286     for (const unsigned Reg : ET.getArgumentRegisters())
 287       Entry.MBB->addLiveIn(Reg);
 288     // Add a live in for registers that need saving so that the machine verifier
 289     // doesn't fail if the register is never defined.
 290     for (const unsigned Reg : ET.getRegistersNeedSaving())
 291       Entry.MBB->addLiveIn(Reg);
 292   }
 293
 294   const bool IsSnippetSetupComplete = generateSnippetSetupCode(
 295       ET, TM->getMCSubtargetInfo(), Entry, Key, GenerateMemoryInstructions);
 296
 297   // If the snippet setup is not complete, we disable liveliness tracking. This
 298   // means that we won't know what values are in the registers.
 299   // FIXME: this should probably be an assertion.
 300   if (!IsSnippetSetupComplete)
 301     Properties.reset(MachineFunctionProperties::Property::TracksLiveness);
 302
 303   Fill(Sink);
 304
 305   // prologue/epilogue pass needs the reserved registers to be frozen, this
 306   // is usually done by the SelectionDAGISel pass.
 307   MF.getRegInfo().freezeReservedRegs();
 308
 309   // We create the pass manager, run the passes to populate AsmBuffer.
 310   MCContext &MCContext = MMIWP->getMMI().getContext();
 311   legacy::PassManager PM;
 312
 313   TargetLibraryInfoImpl TLII(Triple(Module->getTargetTriple()));
 314   PM.add(new TargetLibraryInfoWrapperPass(TLII));
 315
 316   TargetPassConfig *TPC = TM->createPassConfig(PM);
 317   PM.add(TPC);
 318   PM.add(MMIWP.release());
 319   TPC->printAndVerify("MachineFunctionGenerator::assemble");
 320   // Add target-specific passes.
 321   ET.addTargetSpecificPasses(PM);
 322   TPC->printAndVerify("After ExegesisTarget::addTargetSpecificPasses");
 323   // Adding the following passes:
 324   // - postrapseudos: expands pseudo return instructions used on some targets.
 325   // - machineverifier: checks that the MachineFunction is well formed.
 326   // - prologepilog: saves and restore callee saved registers.
 327   for (const char *PassName :
 328        {"postrapseudos", "machineverifier", "prologepilog"})
 329     if (addPass(PM, PassName, *TPC))
 330       return make_error<Failure>("Unable to add a mandatory pass");
 331   TPC->setInitialized();
 332
 333   // AsmPrinter is responsible for generating the assembly into AsmBuffer.
 334   if (TM->addAsmPrinter(PM, AsmStream, nullptr, CodeGenFileType::ObjectFile,
 335                         MCContext))
 336     return make_error<Failure>("Cannot add AsmPrinter passes");
 337
 338   PM.run(*Module); // Run all the passes
 339   return Error::success();
 340 }
 341
 342 object::OwningBinary<object::ObjectFile>
 343 getObjectFromBuffer(StringRef InputData) {
 344   // Storing the generated assembly into a MemoryBuffer that owns the memory.
 345   std::unique_ptr<MemoryBuffer> Buffer =
 346       MemoryBuffer::getMemBufferCopy(InputData);
 347   // Create the ObjectFile from the MemoryBuffer.
 348   std::unique_ptr<object::ObjectFile> Obj =
 349       cantFail(object::ObjectFile::createObjectFile(Buffer->getMemBufferRef()));
 350   // Returning both the MemoryBuffer and the ObjectFile.
 351   return object::OwningBinary<object::ObjectFile>(std::move(Obj),
 352                                                   std::move(Buffer));
 353 }
 354
 355 object::OwningBinary<object::ObjectFile> getObjectFromFile(StringRef Filename) {
 356   return cantFail(object::ObjectFile::createObjectFile(Filename));
 357 }
 358
 359 Expected<ExecutableFunction> ExecutableFunction::create(
 360     std::unique_ptr<TargetMachine> TM,
 361     object::OwningBinary<object::ObjectFile> &&ObjectFileHolder) {
 362   assert(ObjectFileHolder.getBinary() && "cannot create object file");
 363   std::unique_ptr<LLVMContext> Ctx = std::make_unique<LLVMContext>();
 364
 365   auto SymbolSizes = object::computeSymbolSizes(*ObjectFileHolder.getBinary());
 366   // Get the size of the function that we want to call into (with the name of
 367   // FunctionID).
 368   auto SymbolIt = find_if(SymbolSizes, [&](const auto &Pair) {
 369     auto SymbolName = Pair.first.getName();
 370     if (SymbolName)
 371       return *SymbolName == FunctionID;
 372     // We should always succeed in finding the FunctionID, hence we suppress
 373     // the error here and assert later on the search result, rather than
 374     // propagating the Expected<> error back to the caller.
 375     consumeError(SymbolName.takeError());
 376     return false;
 377   });
 378   assert(SymbolIt != SymbolSizes.end() &&
 379          "Cannot find the symbol for FunctionID");
 380   uintptr_t CodeSize = SymbolIt->second;
 381
 382   auto EJITOrErr = orc::LLJITBuilder().create();
 383   if (!EJITOrErr)
 384     return EJITOrErr.takeError();
 385
 386   auto EJIT = std::move(*EJITOrErr);
 387
 388   if (auto ObjErr =
 389           EJIT->addObjectFile(std::get<1>(ObjectFileHolder.takeBinary())))
 390     return std::move(ObjErr);
 391
 392   auto FunctionAddressOrErr = EJIT->lookup(FunctionID);
 393   if (!FunctionAddressOrErr)
 394     return FunctionAddressOrErr.takeError();
 395
 396   const uint64_t FunctionAddress = FunctionAddressOrErr->getValue();
 397
 398   assert(isAligned(kFunctionAlignment, FunctionAddress) &&
 399          "function is not properly aligned");
 400
 401   StringRef FBytes =
 402       StringRef(reinterpret_cast<const char *>(FunctionAddress), CodeSize);
 403   return ExecutableFunction(std::move(Ctx), std::move(EJIT), FBytes);
 404 }
 405
 406 ExecutableFunction::ExecutableFunction(std::unique_ptr<LLVMContext> Ctx,
 407                                        std::unique_ptr<orc::LLJIT> EJIT,
 408                                        StringRef FB)
 409     : FunctionBytes(FB), Context(std::move(Ctx)), ExecJIT(std::move(EJIT)) {}
 410
 411 Error getBenchmarkFunctionBytes(const StringRef InputData,
 412                                 std::vector<uint8_t> &Bytes) {
 413   const auto Holder = getObjectFromBuffer(InputData);
 414   const auto *Obj = Holder.getBinary();
 415   // See RuntimeDyldImpl::loadObjectImpl(Obj) for much more complete
 416   // implementation.
 417
 418   // Find the only function in the object file.
 419   SmallVector<object::SymbolRef, 1> Functions;
 420   for (auto &Sym : Obj->symbols()) {
 421     auto SymType = Sym.getType();
 422     if (SymType && *SymType == object::SymbolRef::Type::ST_Function)
 423       Functions.push_back(Sym);
 424   }
 425   if (Functions.size() != 1)
 426     return make_error<Failure>("Exactly one function expected");
 427
 428   // Find the containing section - it is assumed to contain only this function.
 429   auto SectionOrErr = Functions.front().getSection();
 430   if (!SectionOrErr || *SectionOrErr == Obj->section_end())
 431     return make_error<Failure>("Section not found");
 432
 433   auto Address = Functions.front().getAddress();
 434   if (!Address || *Address != SectionOrErr.get()->getAddress())
 435     return make_error<Failure>("Unexpected layout");
 436
 437   auto ContentsOrErr = SectionOrErr.get()->getContents();
 438   if (!ContentsOrErr)
 439     return ContentsOrErr.takeError();
 440   Bytes.assign(ContentsOrErr->begin(), ContentsOrErr->end());
 441   return Error::success();
 442 }
 443
 444 } // namespace exegesis
 445 } // namespace llvm