tools/llvm-exegesis/lib/X86/Target.cpp

   1 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 #include "../Target.h"
  10
  11 #include "../Latency.h"
  12 #include "../Uops.h"
  13 #include "MCTargetDesc/X86BaseInfo.h"
  14 #include "MCTargetDesc/X86MCTargetDesc.h"
  15 #include "X86.h"
  16 #include "X86RegisterInfo.h"
  17 #include "X86Subtarget.h"
  18 #include "llvm/MC/MCInstBuilder.h"
  19
  20 namespace exegesis {
  21
  22 namespace {
  23
  24 // A chunk of instruction's operands that represents a single memory access.
  25 struct MemoryOperandRange {
  26   MemoryOperandRange(llvm::ArrayRef<Operand> Operands) : Ops(Operands) {}
  27
  28   // Setup InstructionTemplate so the memory access represented by this object
  29   // points to [reg] + offset.
  30   void fillOrDie(InstructionTemplate &IT, unsigned Reg, unsigned Offset) {
  31     switch (Ops.size()) {
  32     case 5:
  33       IT.getValueFor(Ops[0]) = llvm::MCOperand::createReg(Reg);    // BaseReg
  34       IT.getValueFor(Ops[1]) = llvm::MCOperand::createImm(1);      // ScaleAmt
  35       IT.getValueFor(Ops[2]) = llvm::MCOperand::createReg(0);      // IndexReg
  36       IT.getValueFor(Ops[3]) = llvm::MCOperand::createImm(Offset); // Disp
  37       IT.getValueFor(Ops[4]) = llvm::MCOperand::createReg(0);      // Segment
  38       break;
  39     default:
  40       llvm::errs() << Ops.size() << "-op are not handled right now ("
  41                    << IT.Instr.Name << ")\n";
  42       llvm_unreachable("Invalid memory configuration");
  43     }
  44   }
  45
  46   // Returns whether Range can be filled.
  47   static bool isValid(const MemoryOperandRange &Range) {
  48     return Range.Ops.size() == 5;
  49   }
  50
  51   // Returns whether Op is a valid memory operand.
  52   static bool isMemoryOperand(const Operand &Op) {
  53     return Op.isMemory() && Op.isExplicit();
  54   }
  55
  56   llvm::ArrayRef<Operand> Ops;
  57 };
  58
  59 // X86 memory access involve non constant number of operands, this function
  60 // extracts contiguous memory operands into MemoryOperandRange so it's easier to
  61 // check and fill.
  62 static std::vector<MemoryOperandRange>
  63 getMemoryOperandRanges(llvm::ArrayRef<Operand> Operands) {
  64   std::vector<MemoryOperandRange> Result;
  65   while (!Operands.empty()) {
  66     Operands = Operands.drop_until(MemoryOperandRange::isMemoryOperand);
  67     auto MemoryOps = Operands.take_while(MemoryOperandRange::isMemoryOperand);
  68     if (!MemoryOps.empty())
  69       Result.push_back(MemoryOps);
  70     Operands = Operands.drop_front(MemoryOps.size());
  71   }
  72   return Result;
  73 }
  74
  75 static llvm::Error IsInvalidOpcode(const Instruction &Instr) {
  76   const auto OpcodeName = Instr.Name;
  77   if (OpcodeName.startswith("POPF") || OpcodeName.startswith("PUSHF") ||
  78       OpcodeName.startswith("ADJCALLSTACK"))
  79     return llvm::make_error<BenchmarkFailure>(
  80         "unsupported opcode: Push/Pop/AdjCallStack");
  81   const bool ValidMemoryOperands = llvm::all_of(
  82       getMemoryOperandRanges(Instr.Operands), MemoryOperandRange::isValid);
  83   if (!ValidMemoryOperands)
  84     return llvm::make_error<BenchmarkFailure>(
  85         "unsupported opcode: non uniform memory access");
  86   // We do not handle instructions with OPERAND_PCREL.
  87   for (const Operand &Op : Instr.Operands)
  88     if (Op.isExplicit() &&
  89         Op.getExplicitOperandInfo().OperandType == llvm::MCOI::OPERAND_PCREL)
  90       return llvm::make_error<BenchmarkFailure>(
  91           "unsupported opcode: PC relative operand");
  92   // We do not handle second-form X87 instructions. We only handle first-form
  93   // ones (_Fp), see comment in X86InstrFPStack.td.
  94   for (const Operand &Op : Instr.Operands)
  95     if (Op.isReg() && Op.isExplicit() &&
  96         Op.getExplicitOperandInfo().RegClass == llvm::X86::RSTRegClassID)
  97       return llvm::make_error<BenchmarkFailure>(
  98           "unsupported second-form X87 instruction");
  99   return llvm::Error::success();
 100 }
 101
 102 static unsigned GetX86FPFlags(const Instruction &Instr) {
 103   return Instr.Description->TSFlags & llvm::X86II::FPTypeMask;
 104 }
 105
 106 class X86LatencySnippetGenerator : public LatencySnippetGenerator {
 107 public:
 108   using LatencySnippetGenerator::LatencySnippetGenerator;
 109
 110   llvm::Expected<std::vector<CodeTemplate>>
 111   generateCodeTemplates(const Instruction &Instr) const override {
 112     if (auto E = IsInvalidOpcode(Instr))
 113       return std::move(E);
 114
 115     switch (GetX86FPFlags(Instr)) {
 116     case llvm::X86II::NotFP:
 117       return LatencySnippetGenerator::generateCodeTemplates(Instr);
 118     case llvm::X86II::ZeroArgFP:
 119     case llvm::X86II::OneArgFP:
 120     case llvm::X86II::SpecialFP:
 121     case llvm::X86II::CompareFP:
 122     case llvm::X86II::CondMovFP:
 123       return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
 124     case llvm::X86II::OneArgFPRW:
 125     case llvm::X86II::TwoArgFP:
 126       // These are instructions like
 127       //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
 128       //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
 129       // They are intrinsically serial and do not modify the state of the stack.
 130       return generateSelfAliasingCodeTemplates(Instr);
 131     default:
 132       llvm_unreachable("Unknown FP Type!");
 133     }
 134   }
 135 };
 136
 137 class X86UopsSnippetGenerator : public UopsSnippetGenerator {
 138 public:
 139   using UopsSnippetGenerator::UopsSnippetGenerator;
 140
 141   llvm::Expected<std::vector<CodeTemplate>>
 142   generateCodeTemplates(const Instruction &Instr) const override {
 143     if (auto E = IsInvalidOpcode(Instr))
 144       return std::move(E);
 145
 146     switch (GetX86FPFlags(Instr)) {
 147     case llvm::X86II::NotFP:
 148       return UopsSnippetGenerator::generateCodeTemplates(Instr);
 149     case llvm::X86II::ZeroArgFP:
 150     case llvm::X86II::OneArgFP:
 151     case llvm::X86II::SpecialFP:
 152       return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
 153     case llvm::X86II::OneArgFPRW:
 154     case llvm::X86II::TwoArgFP:
 155       // These are instructions like
 156       //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
 157       //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
 158       // They are intrinsically serial and do not modify the state of the stack.
 159       // We generate the same code for latency and uops.
 160       return generateSelfAliasingCodeTemplates(Instr);
 161     case llvm::X86II::CompareFP:
 162     case llvm::X86II::CondMovFP:
 163       // We can compute uops for any FP instruction that does not grow or shrink
 164       // the stack (either do not touch the stack or push as much as they pop).
 165       return generateUnconstrainedCodeTemplates(
 166           Instr, "instruction does not grow/shrink the FP stack");
 167     default:
 168       llvm_unreachable("Unknown FP Type!");
 169     }
 170   }
 171 };
 172
 173 static unsigned GetLoadImmediateOpcode(unsigned RegBitWidth) {
 174   switch (RegBitWidth) {
 175   case 8:
 176     return llvm::X86::MOV8ri;
 177   case 16:
 178     return llvm::X86::MOV16ri;
 179   case 32:
 180     return llvm::X86::MOV32ri;
 181   case 64:
 182     return llvm::X86::MOV64ri;
 183   }
 184   llvm_unreachable("Invalid Value Width");
 185 }
 186
 187 // Generates instruction to load an immediate value into a register.
 188 static llvm::MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
 189                                   const llvm::APInt &Value) {
 190   if (Value.getBitWidth() > RegBitWidth)
 191     llvm_unreachable("Value must fit in the Register");
 192   return llvm::MCInstBuilder(GetLoadImmediateOpcode(RegBitWidth))
 193       .addReg(Reg)
 194       .addImm(Value.getZExtValue());
 195 }
 196
 197 // Allocates scratch memory on the stack.
 198 static llvm::MCInst allocateStackSpace(unsigned Bytes) {
 199   return llvm::MCInstBuilder(llvm::X86::SUB64ri8)
 200       .addReg(llvm::X86::RSP)
 201       .addReg(llvm::X86::RSP)
 202       .addImm(Bytes);
 203 }
 204
 205 // Fills scratch memory at offset `OffsetBytes` with value `Imm`.
 206 static llvm::MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
 207                                    uint64_t Imm) {
 208   return llvm::MCInstBuilder(MovOpcode)
 209       // Address = ESP
 210       .addReg(llvm::X86::RSP) // BaseReg
 211       .addImm(1)              // ScaleAmt
 212       .addReg(0)              // IndexReg
 213       .addImm(OffsetBytes)    // Disp
 214       .addReg(0)              // Segment
 215       // Immediate.
 216       .addImm(Imm);
 217 }
 218
 219 // Loads scratch memory into register `Reg` using opcode `RMOpcode`.
 220 static llvm::MCInst loadToReg(unsigned Reg, unsigned RMOpcode) {
 221   return llvm::MCInstBuilder(RMOpcode)
 222       .addReg(Reg)
 223       // Address = ESP
 224       .addReg(llvm::X86::RSP) // BaseReg
 225       .addImm(1)              // ScaleAmt
 226       .addReg(0)              // IndexReg
 227       .addImm(0)              // Disp
 228       .addReg(0);             // Segment
 229 }
 230
 231 // Releases scratch memory.
 232 static llvm::MCInst releaseStackSpace(unsigned Bytes) {
 233   return llvm::MCInstBuilder(llvm::X86::ADD64ri8)
 234       .addReg(llvm::X86::RSP)
 235       .addReg(llvm::X86::RSP)
 236       .addImm(Bytes);
 237 }
 238
 239 // Reserves some space on the stack, fills it with the content of the provided
 240 // constant and provide methods to load the stack value into a register.
 241 struct ConstantInliner {
 242   explicit ConstantInliner(const llvm::APInt &Constant) : Constant_(Constant) {}
 243
 244   std::vector<llvm::MCInst> loadAndFinalize(unsigned Reg, unsigned RegBitWidth,
 245                                             unsigned Opcode) {
 246     assert((RegBitWidth & 7) == 0 &&
 247            "RegBitWidth must be a multiple of 8 bits");
 248     initStack(RegBitWidth / 8);
 249     add(loadToReg(Reg, Opcode));
 250     add(releaseStackSpace(RegBitWidth / 8));
 251     return std::move(Instructions);
 252   }
 253
 254   std::vector<llvm::MCInst> loadX87STAndFinalize(unsigned Reg) {
 255     initStack(kF80Bytes);
 256     add(llvm::MCInstBuilder(llvm::X86::LD_F80m)
 257             // Address = ESP
 258             .addReg(llvm::X86::RSP) // BaseReg
 259             .addImm(1)              // ScaleAmt
 260             .addReg(0)              // IndexReg
 261             .addImm(0)              // Disp
 262             .addReg(0));            // Segment
 263     if (Reg != llvm::X86::ST0)
 264       add(llvm::MCInstBuilder(llvm::X86::ST_Frr).addReg(Reg));
 265     add(releaseStackSpace(kF80Bytes));
 266     return std::move(Instructions);
 267   }
 268
 269   std::vector<llvm::MCInst> loadX87FPAndFinalize(unsigned Reg) {
 270     initStack(kF80Bytes);
 271     add(llvm::MCInstBuilder(llvm::X86::LD_Fp80m)
 272             .addReg(Reg)
 273             // Address = ESP
 274             .addReg(llvm::X86::RSP) // BaseReg
 275             .addImm(1)              // ScaleAmt
 276             .addReg(0)              // IndexReg
 277             .addImm(0)              // Disp
 278             .addReg(0));            // Segment
 279     add(releaseStackSpace(kF80Bytes));
 280     return std::move(Instructions);
 281   }
 282
 283   std::vector<llvm::MCInst> popFlagAndFinalize() {
 284     initStack(8);
 285     add(llvm::MCInstBuilder(llvm::X86::POPF64));
 286     return std::move(Instructions);
 287   }
 288
 289 private:
 290   static constexpr const unsigned kF80Bytes = 10; // 80 bits.
 291
 292   ConstantInliner &add(const llvm::MCInst &Inst) {
 293     Instructions.push_back(Inst);
 294     return *this;
 295   }
 296
 297   void initStack(unsigned Bytes) {
 298     assert(Constant_.getBitWidth() <= Bytes * 8 &&
 299            "Value does not have the correct size");
 300     const llvm::APInt WideConstant = Constant_.getBitWidth() < Bytes * 8
 301                                          ? Constant_.sext(Bytes * 8)
 302                                          : Constant_;
 303     add(allocateStackSpace(Bytes));
 304     size_t ByteOffset = 0;
 305     for (; Bytes - ByteOffset >= 4; ByteOffset += 4)
 306       add(fillStackSpace(
 307           llvm::X86::MOV32mi, ByteOffset,
 308           WideConstant.extractBits(32, ByteOffset * 8).getZExtValue()));
 309     if (Bytes - ByteOffset >= 2) {
 310       add(fillStackSpace(
 311           llvm::X86::MOV16mi, ByteOffset,
 312           WideConstant.extractBits(16, ByteOffset * 8).getZExtValue()));
 313       ByteOffset += 2;
 314     }
 315     if (Bytes - ByteOffset >= 1)
 316       add(fillStackSpace(
 317           llvm::X86::MOV8mi, ByteOffset,
 318           WideConstant.extractBits(8, ByteOffset * 8).getZExtValue()));
 319   }
 320
 321   llvm::APInt Constant_;
 322   std::vector<llvm::MCInst> Instructions;
 323 };
 324
 325 class ExegesisX86Target : public ExegesisTarget {
 326   void addTargetSpecificPasses(llvm::PassManagerBase &PM) const override {
 327     // Lowers FP pseudo-instructions, e.g. ABS_Fp32 -> ABS_F.
 328     PM.add(llvm::createX86FloatingPointStackifierPass());
 329   }
 330
 331   unsigned getScratchMemoryRegister(const llvm::Triple &TT) const override {
 332     if (!TT.isArch64Bit()) {
 333       // FIXME: This would require popping from the stack, so we would have to
 334       // add some additional setup code.
 335       return 0;
 336     }
 337     return TT.isOSWindows() ? llvm::X86::RCX : llvm::X86::RDI;
 338   }
 339
 340   unsigned getMaxMemoryAccessSize() const override { return 64; }
 341
 342   void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
 343                           unsigned Offset) const override {
 344     // FIXME: For instructions that read AND write to memory, we use the same
 345     // value for input and output.
 346     for (auto &MemoryRange : getMemoryOperandRanges(IT.Instr.Operands))
 347       MemoryRange.fillOrDie(IT, Reg, Offset);
 348   }
 349
 350   std::vector<llvm::MCInst> setRegTo(const llvm::MCSubtargetInfo &STI,
 351                                      unsigned Reg,
 352                                      const llvm::APInt &Value) const override {
 353     if (llvm::X86::GR8RegClass.contains(Reg))
 354       return {loadImmediate(Reg, 8, Value)};
 355     if (llvm::X86::GR16RegClass.contains(Reg))
 356       return {loadImmediate(Reg, 16, Value)};
 357     if (llvm::X86::GR32RegClass.contains(Reg))
 358       return {loadImmediate(Reg, 32, Value)};
 359     if (llvm::X86::GR64RegClass.contains(Reg))
 360       return {loadImmediate(Reg, 64, Value)};
 361     ConstantInliner CI(Value);
 362     if (llvm::X86::VR64RegClass.contains(Reg))
 363       return CI.loadAndFinalize(Reg, 64, llvm::X86::MMX_MOVQ64rm);
 364     if (llvm::X86::VR128XRegClass.contains(Reg)) {
 365       if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
 366         return CI.loadAndFinalize(Reg, 128, llvm::X86::VMOVDQU32Z128rm);
 367       if (STI.getFeatureBits()[llvm::X86::FeatureAVX])
 368         return CI.loadAndFinalize(Reg, 128, llvm::X86::VMOVDQUrm);
 369       return CI.loadAndFinalize(Reg, 128, llvm::X86::MOVDQUrm);
 370     }
 371     if (llvm::X86::VR256XRegClass.contains(Reg)) {
 372       if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
 373         return CI.loadAndFinalize(Reg, 256, llvm::X86::VMOVDQU32Z256rm);
 374       if (STI.getFeatureBits()[llvm::X86::FeatureAVX])
 375         return CI.loadAndFinalize(Reg, 256, llvm::X86::VMOVDQUYrm);
 376     }
 377     if (llvm::X86::VR512RegClass.contains(Reg))
 378       if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
 379         return CI.loadAndFinalize(Reg, 512, llvm::X86::VMOVDQU32Zrm);
 380     if (llvm::X86::RSTRegClass.contains(Reg)) {
 381       return CI.loadX87STAndFinalize(Reg);
 382     }
 383     if (llvm::X86::RFP32RegClass.contains(Reg) ||
 384         llvm::X86::RFP64RegClass.contains(Reg) ||
 385         llvm::X86::RFP80RegClass.contains(Reg)) {
 386       return CI.loadX87FPAndFinalize(Reg);
 387     }
 388     if (Reg == llvm::X86::EFLAGS)
 389       return CI.popFlagAndFinalize();
 390     return {}; // Not yet implemented.
 391   }
 392
 393   std::unique_ptr<SnippetGenerator>
 394   createLatencySnippetGenerator(const LLVMState &State) const override {
 395     return llvm::make_unique<X86LatencySnippetGenerator>(State);
 396   }
 397
 398   std::unique_ptr<SnippetGenerator>
 399   createUopsSnippetGenerator(const LLVMState &State) const override {
 400     return llvm::make_unique<X86UopsSnippetGenerator>(State);
 401   }
 402
 403   bool matchesArch(llvm::Triple::ArchType Arch) const override {
 404     return Arch == llvm::Triple::x86_64 || Arch == llvm::Triple::x86;
 405   }
 406 };
 407
 408 } // namespace
 409
 410 static ExegesisTarget *getTheExegesisX86Target() {
 411   static ExegesisX86Target Target;
 412   return &Target;
 413 }
 414
 415 void InitializeX86ExegesisTarget() {
 416   ExegesisTarget::registerTarget(getTheExegesisX86Target());
 417 }
 418
 419 } // namespace exegesis