tools/llvm-exegesis/lib/X86/Target.cpp

   1 //===-- Target.cpp ----------------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 #include "../Target.h"
   9
  10 #include "../Latency.h"
  11 #include "../SnippetGenerator.h"
  12 #include "../Uops.h"
  13 #include "MCTargetDesc/X86BaseInfo.h"
  14 #include "MCTargetDesc/X86MCTargetDesc.h"
  15 #include "X86.h"
  16 #include "X86RegisterInfo.h"
  17 #include "X86Subtarget.h"
  18 #include "llvm/MC/MCInstBuilder.h"
  19
  20 namespace llvm {
  21 namespace exegesis {
  22
  23 // Returns an error if we cannot handle the memory references in this
  24 // instruction.
  25 static Error isInvalidMemoryInstr(const Instruction &Instr) {
  26   switch (Instr.Description->TSFlags & X86II::FormMask) {
  27   default:
  28     llvm_unreachable("Unknown FormMask value");
  29   // These have no memory access.
  30   case X86II::Pseudo:
  31   case X86II::RawFrm:
  32   case X86II::AddCCFrm:
  33   case X86II::MRMDestReg:
  34   case X86II::MRMSrcReg:
  35   case X86II::MRMSrcReg4VOp3:
  36   case X86II::MRMSrcRegOp4:
  37   case X86II::MRMSrcRegCC:
  38   case X86II::MRMXrCC:
  39   case X86II::MRMXr:
  40   case X86II::MRM0r:
  41   case X86II::MRM1r:
  42   case X86II::MRM2r:
  43   case X86II::MRM3r:
  44   case X86II::MRM4r:
  45   case X86II::MRM5r:
  46   case X86II::MRM6r:
  47   case X86II::MRM7r:
  48   case X86II::MRM_C0:
  49   case X86II::MRM_C1:
  50   case X86II::MRM_C2:
  51   case X86II::MRM_C3:
  52   case X86II::MRM_C4:
  53   case X86II::MRM_C5:
  54   case X86II::MRM_C6:
  55   case X86II::MRM_C7:
  56   case X86II::MRM_C8:
  57   case X86II::MRM_C9:
  58   case X86II::MRM_CA:
  59   case X86II::MRM_CB:
  60   case X86II::MRM_CC:
  61   case X86II::MRM_CD:
  62   case X86II::MRM_CE:
  63   case X86II::MRM_CF:
  64   case X86II::MRM_D0:
  65   case X86II::MRM_D1:
  66   case X86II::MRM_D2:
  67   case X86II::MRM_D3:
  68   case X86II::MRM_D4:
  69   case X86II::MRM_D5:
  70   case X86II::MRM_D6:
  71   case X86II::MRM_D7:
  72   case X86II::MRM_D8:
  73   case X86II::MRM_D9:
  74   case X86II::MRM_DA:
  75   case X86II::MRM_DB:
  76   case X86II::MRM_DC:
  77   case X86II::MRM_DD:
  78   case X86II::MRM_DE:
  79   case X86II::MRM_DF:
  80   case X86II::MRM_E0:
  81   case X86II::MRM_E1:
  82   case X86II::MRM_E2:
  83   case X86II::MRM_E3:
  84   case X86II::MRM_E4:
  85   case X86II::MRM_E5:
  86   case X86II::MRM_E6:
  87   case X86II::MRM_E7:
  88   case X86II::MRM_E8:
  89   case X86II::MRM_E9:
  90   case X86II::MRM_EA:
  91   case X86II::MRM_EB:
  92   case X86II::MRM_EC:
  93   case X86II::MRM_ED:
  94   case X86II::MRM_EE:
  95   case X86II::MRM_EF:
  96   case X86II::MRM_F0:
  97   case X86II::MRM_F1:
  98   case X86II::MRM_F2:
  99   case X86II::MRM_F3:
 100   case X86II::MRM_F4:
 101   case X86II::MRM_F5:
 102   case X86II::MRM_F6:
 103   case X86II::MRM_F7:
 104   case X86II::MRM_F8:
 105   case X86II::MRM_F9:
 106   case X86II::MRM_FA:
 107   case X86II::MRM_FB:
 108   case X86II::MRM_FC:
 109   case X86II::MRM_FD:
 110   case X86II::MRM_FE:
 111   case X86II::MRM_FF:
 112   case X86II::RawFrmImm8:
 113     return Error::success();
 114   case X86II::AddRegFrm:
 115     return (Instr.Description->Opcode == X86::POP16r || Instr.Description->Opcode == X86::POP32r ||
 116             Instr.Description->Opcode == X86::PUSH16r || Instr.Description->Opcode == X86::PUSH32r)
 117                ? make_error<BenchmarkFailure>(
 118                      "unsupported opcode: unsupported memory access")
 119                : Error::success();
 120   // These access memory and are handled.
 121   case X86II::MRMDestMem:
 122   case X86II::MRMSrcMem:
 123   case X86II::MRMSrcMem4VOp3:
 124   case X86II::MRMSrcMemOp4:
 125   case X86II::MRMSrcMemCC:
 126   case X86II::MRMXmCC:
 127   case X86II::MRMXm:
 128   case X86II::MRM0m:
 129   case X86II::MRM1m:
 130   case X86II::MRM2m:
 131   case X86II::MRM3m:
 132   case X86II::MRM4m:
 133   case X86II::MRM5m:
 134   case X86II::MRM6m:
 135   case X86II::MRM7m:
 136     return Error::success();
 137   // These access memory and are not handled yet.
 138   case X86II::RawFrmImm16:
 139   case X86II::RawFrmMemOffs:
 140   case X86II::RawFrmSrc:
 141   case X86II::RawFrmDst:
 142   case X86II::RawFrmDstSrc:
 143     return make_error<BenchmarkFailure>(
 144         "unsupported opcode: non uniform memory access");
 145   }
 146 }
 147
 148 static llvm::Error IsInvalidOpcode(const Instruction &Instr) {
 149   const auto OpcodeName = Instr.Name;
 150   if ((Instr.Description->TSFlags & X86II::FormMask) == X86II::Pseudo)
 151     return llvm::make_error<BenchmarkFailure>(
 152         "unsupported opcode: pseudo instruction");
 153   if (OpcodeName.startswith("POPF") || OpcodeName.startswith("PUSHF") ||
 154       OpcodeName.startswith("ADJCALLSTACK"))
 155     return llvm::make_error<BenchmarkFailure>(
 156         "unsupported opcode: Push/Pop/AdjCallStack");
 157   if (llvm::Error Error = isInvalidMemoryInstr(Instr))
 158     return Error;
 159   // We do not handle instructions with OPERAND_PCREL.
 160   for (const Operand &Op : Instr.Operands)
 161     if (Op.isExplicit() &&
 162         Op.getExplicitOperandInfo().OperandType == llvm::MCOI::OPERAND_PCREL)
 163       return llvm::make_error<BenchmarkFailure>(
 164           "unsupported opcode: PC relative operand");
 165   // We do not handle second-form X87 instructions. We only handle first-form
 166   // ones (_Fp), see comment in X86InstrFPStack.td.
 167   for (const Operand &Op : Instr.Operands)
 168     if (Op.isReg() && Op.isExplicit() &&
 169         Op.getExplicitOperandInfo().RegClass == llvm::X86::RSTRegClassID)
 170       return llvm::make_error<BenchmarkFailure>(
 171           "unsupported second-form X87 instruction");
 172   return llvm::Error::success();
 173 }
 174
 175 static unsigned getX86FPFlags(const Instruction &Instr) {
 176   return Instr.Description->TSFlags & llvm::X86II::FPTypeMask;
 177 }
 178
 179 namespace {
 180 class X86LatencySnippetGenerator : public LatencySnippetGenerator {
 181 public:
 182   using LatencySnippetGenerator::LatencySnippetGenerator;
 183
 184   llvm::Expected<std::vector<CodeTemplate>>
 185   generateCodeTemplates(const Instruction &Instr) const override;
 186 };
 187 } // namespace
 188
 189 llvm::Expected<std::vector<CodeTemplate>>
 190 X86LatencySnippetGenerator::generateCodeTemplates(
 191     const Instruction &Instr) const {
 192   if (auto E = IsInvalidOpcode(Instr))
 193     return std::move(E);
 194
 195   switch (getX86FPFlags(Instr)) {
 196   case llvm::X86II::NotFP:
 197     return LatencySnippetGenerator::generateCodeTemplates(Instr);
 198   case llvm::X86II::ZeroArgFP:
 199   case llvm::X86II::OneArgFP:
 200   case llvm::X86II::SpecialFP:
 201   case llvm::X86II::CompareFP:
 202   case llvm::X86II::CondMovFP:
 203     return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
 204   case llvm::X86II::OneArgFPRW:
 205   case llvm::X86II::TwoArgFP:
 206     // These are instructions like
 207     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
 208     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
 209     // They are intrinsically serial and do not modify the state of the stack.
 210     return generateSelfAliasingCodeTemplates(Instr);
 211   default:
 212     llvm_unreachable("Unknown FP Type!");
 213   }
 214 }
 215
 216 namespace {
 217 class X86UopsSnippetGenerator : public UopsSnippetGenerator {
 218 public:
 219   using UopsSnippetGenerator::UopsSnippetGenerator;
 220
 221   llvm::Expected<std::vector<CodeTemplate>>
 222   generateCodeTemplates(const Instruction &Instr) const override;
 223 };
 224 } // namespace
 225
 226 llvm::Expected<std::vector<CodeTemplate>>
 227 X86UopsSnippetGenerator::generateCodeTemplates(
 228     const Instruction &Instr) const {
 229   if (auto E = IsInvalidOpcode(Instr))
 230     return std::move(E);
 231
 232   switch (getX86FPFlags(Instr)) {
 233   case llvm::X86II::NotFP:
 234     return UopsSnippetGenerator::generateCodeTemplates(Instr);
 235   case llvm::X86II::ZeroArgFP:
 236   case llvm::X86II::OneArgFP:
 237   case llvm::X86II::SpecialFP:
 238     return llvm::make_error<BenchmarkFailure>("Unsupported x87 Instruction");
 239   case llvm::X86II::OneArgFPRW:
 240   case llvm::X86II::TwoArgFP:
 241     // These are instructions like
 242     //   - `ST(0) = fsqrt(ST(0))` (OneArgFPRW)
 243     //   - `ST(0) = ST(0) + ST(i)` (TwoArgFP)
 244     // They are intrinsically serial and do not modify the state of the stack.
 245     // We generate the same code for latency and uops.
 246     return generateSelfAliasingCodeTemplates(Instr);
 247   case llvm::X86II::CompareFP:
 248   case llvm::X86II::CondMovFP:
 249     // We can compute uops for any FP instruction that does not grow or shrink
 250     // the stack (either do not touch the stack or push as much as they pop).
 251     return generateUnconstrainedCodeTemplates(
 252         Instr, "instruction does not grow/shrink the FP stack");
 253   default:
 254     llvm_unreachable("Unknown FP Type!");
 255   }
 256 }
 257
 258 static unsigned getLoadImmediateOpcode(unsigned RegBitWidth) {
 259   switch (RegBitWidth) {
 260   case 8:
 261     return llvm::X86::MOV8ri;
 262   case 16:
 263     return llvm::X86::MOV16ri;
 264   case 32:
 265     return llvm::X86::MOV32ri;
 266   case 64:
 267     return llvm::X86::MOV64ri;
 268   }
 269   llvm_unreachable("Invalid Value Width");
 270 }
 271
 272 // Generates instruction to load an immediate value into a register.
 273 static llvm::MCInst loadImmediate(unsigned Reg, unsigned RegBitWidth,
 274                                   const llvm::APInt &Value) {
 275   if (Value.getBitWidth() > RegBitWidth)
 276     llvm_unreachable("Value must fit in the Register");
 277   return llvm::MCInstBuilder(getLoadImmediateOpcode(RegBitWidth))
 278       .addReg(Reg)
 279       .addImm(Value.getZExtValue());
 280 }
 281
 282 // Allocates scratch memory on the stack.
 283 static llvm::MCInst allocateStackSpace(unsigned Bytes) {
 284   return llvm::MCInstBuilder(llvm::X86::SUB64ri8)
 285       .addReg(llvm::X86::RSP)
 286       .addReg(llvm::X86::RSP)
 287       .addImm(Bytes);
 288 }
 289
 290 // Fills scratch memory at offset `OffsetBytes` with value `Imm`.
 291 static llvm::MCInst fillStackSpace(unsigned MovOpcode, unsigned OffsetBytes,
 292                                    uint64_t Imm) {
 293   return llvm::MCInstBuilder(MovOpcode)
 294       // Address = ESP
 295       .addReg(llvm::X86::RSP) // BaseReg
 296       .addImm(1)              // ScaleAmt
 297       .addReg(0)              // IndexReg
 298       .addImm(OffsetBytes)    // Disp
 299       .addReg(0)              // Segment
 300       // Immediate.
 301       .addImm(Imm);
 302 }
 303
 304 // Loads scratch memory into register `Reg` using opcode `RMOpcode`.
 305 static llvm::MCInst loadToReg(unsigned Reg, unsigned RMOpcode) {
 306   return llvm::MCInstBuilder(RMOpcode)
 307       .addReg(Reg)
 308       // Address = ESP
 309       .addReg(llvm::X86::RSP) // BaseReg
 310       .addImm(1)              // ScaleAmt
 311       .addReg(0)              // IndexReg
 312       .addImm(0)              // Disp
 313       .addReg(0);             // Segment
 314 }
 315
 316 // Releases scratch memory.
 317 static llvm::MCInst releaseStackSpace(unsigned Bytes) {
 318   return llvm::MCInstBuilder(llvm::X86::ADD64ri8)
 319       .addReg(llvm::X86::RSP)
 320       .addReg(llvm::X86::RSP)
 321       .addImm(Bytes);
 322 }
 323
 324 // Reserves some space on the stack, fills it with the content of the provided
 325 // constant and provide methods to load the stack value into a register.
 326 namespace {
 327 struct ConstantInliner {
 328   explicit ConstantInliner(const llvm::APInt &Constant) : Constant_(Constant) {}
 329
 330   std::vector<llvm::MCInst> loadAndFinalize(unsigned Reg, unsigned RegBitWidth,
 331                                             unsigned Opcode);
 332
 333   std::vector<llvm::MCInst> loadX87STAndFinalize(unsigned Reg);
 334
 335   std::vector<llvm::MCInst> loadX87FPAndFinalize(unsigned Reg);
 336
 337   std::vector<llvm::MCInst> popFlagAndFinalize();
 338
 339 private:
 340   ConstantInliner &add(const llvm::MCInst &Inst) {
 341     Instructions.push_back(Inst);
 342     return *this;
 343   }
 344
 345   void initStack(unsigned Bytes);
 346
 347   static constexpr const unsigned kF80Bytes = 10; // 80 bits.
 348
 349   llvm::APInt Constant_;
 350   std::vector<llvm::MCInst> Instructions;
 351 };
 352 } // namespace
 353
 354 std::vector<llvm::MCInst> ConstantInliner::loadAndFinalize(unsigned Reg,
 355                                                            unsigned RegBitWidth,
 356                                                            unsigned Opcode) {
 357   assert((RegBitWidth & 7) == 0 && "RegBitWidth must be a multiple of 8 bits");
 358   initStack(RegBitWidth / 8);
 359   add(loadToReg(Reg, Opcode));
 360   add(releaseStackSpace(RegBitWidth / 8));
 361   return std::move(Instructions);
 362 }
 363
 364 std::vector<llvm::MCInst> ConstantInliner::loadX87STAndFinalize(unsigned Reg) {
 365   initStack(kF80Bytes);
 366   add(llvm::MCInstBuilder(llvm::X86::LD_F80m)
 367           // Address = ESP
 368           .addReg(llvm::X86::RSP) // BaseReg
 369           .addImm(1)              // ScaleAmt
 370           .addReg(0)              // IndexReg
 371           .addImm(0)              // Disp
 372           .addReg(0));            // Segment
 373   if (Reg != llvm::X86::ST0)
 374     add(llvm::MCInstBuilder(llvm::X86::ST_Frr).addReg(Reg));
 375   add(releaseStackSpace(kF80Bytes));
 376   return std::move(Instructions);
 377 }
 378
 379 std::vector<llvm::MCInst> ConstantInliner::loadX87FPAndFinalize(unsigned Reg) {
 380   initStack(kF80Bytes);
 381   add(llvm::MCInstBuilder(llvm::X86::LD_Fp80m)
 382           .addReg(Reg)
 383           // Address = ESP
 384           .addReg(llvm::X86::RSP) // BaseReg
 385           .addImm(1)              // ScaleAmt
 386           .addReg(0)              // IndexReg
 387           .addImm(0)              // Disp
 388           .addReg(0));            // Segment
 389   add(releaseStackSpace(kF80Bytes));
 390   return std::move(Instructions);
 391 }
 392
 393 std::vector<llvm::MCInst> ConstantInliner::popFlagAndFinalize() {
 394   initStack(8);
 395   add(llvm::MCInstBuilder(llvm::X86::POPF64));
 396   return std::move(Instructions);
 397 }
 398
 399 void ConstantInliner::initStack(unsigned Bytes) {
 400   assert(Constant_.getBitWidth() <= Bytes * 8 &&
 401          "Value does not have the correct size");
 402   const llvm::APInt WideConstant = Constant_.getBitWidth() < Bytes * 8
 403                                        ? Constant_.sext(Bytes * 8)
 404                                        : Constant_;
 405   add(allocateStackSpace(Bytes));
 406   size_t ByteOffset = 0;
 407   for (; Bytes - ByteOffset >= 4; ByteOffset += 4)
 408     add(fillStackSpace(
 409         llvm::X86::MOV32mi, ByteOffset,
 410         WideConstant.extractBits(32, ByteOffset * 8).getZExtValue()));
 411   if (Bytes - ByteOffset >= 2) {
 412     add(fillStackSpace(
 413         llvm::X86::MOV16mi, ByteOffset,
 414         WideConstant.extractBits(16, ByteOffset * 8).getZExtValue()));
 415     ByteOffset += 2;
 416   }
 417   if (Bytes - ByteOffset >= 1)
 418     add(fillStackSpace(
 419         llvm::X86::MOV8mi, ByteOffset,
 420         WideConstant.extractBits(8, ByteOffset * 8).getZExtValue()));
 421 }
 422
 423 #include "X86GenExegesis.inc"
 424
 425 namespace {
 426 class ExegesisX86Target : public ExegesisTarget {
 427 public:
 428   ExegesisX86Target() : ExegesisTarget(X86CpuPfmCounters) {}
 429
 430 private:
 431   void addTargetSpecificPasses(llvm::PassManagerBase &PM) const override;
 432
 433   unsigned getScratchMemoryRegister(const llvm::Triple &TT) const override;
 434
 435   unsigned getMaxMemoryAccessSize() const override { return 64; }
 436
 437   void randomizeMCOperand(const Instruction &Instr, const Variable &Var,
 438                           llvm::MCOperand &AssignedValue,
 439                           const llvm::BitVector &ForbiddenRegs) const override;
 440
 441   void fillMemoryOperands(InstructionTemplate &IT, unsigned Reg,
 442                           unsigned Offset) const override;
 443
 444   std::vector<llvm::MCInst> setRegTo(const llvm::MCSubtargetInfo &STI,
 445                                      unsigned Reg,
 446                                      const llvm::APInt &Value) const override;
 447
 448   ArrayRef<unsigned> getUnavailableRegisters() const override {
 449     return makeArrayRef(kUnavailableRegisters,
 450                         sizeof(kUnavailableRegisters) /
 451                             sizeof(kUnavailableRegisters[0]));
 452   }
 453
 454   std::unique_ptr<SnippetGenerator>
 455   createLatencySnippetGenerator(const LLVMState &State) const override {
 456     return std::make_unique<X86LatencySnippetGenerator>(State);
 457   }
 458
 459   std::unique_ptr<SnippetGenerator>
 460   createUopsSnippetGenerator(const LLVMState &State) const override {
 461     return std::make_unique<X86UopsSnippetGenerator>(State);
 462   }
 463
 464   bool matchesArch(llvm::Triple::ArchType Arch) const override {
 465     return Arch == llvm::Triple::x86_64 || Arch == llvm::Triple::x86;
 466   }
 467
 468   static const unsigned kUnavailableRegisters[4];
 469 };
 470
 471 // We disable a few registers that cannot be encoded on instructions with a REX
 472 // prefix.
 473 const unsigned ExegesisX86Target::kUnavailableRegisters[4] = {X86::AH, X86::BH,
 474                                                               X86::CH, X86::DH};
 475 } // namespace
 476
 477 void ExegesisX86Target::addTargetSpecificPasses(
 478     llvm::PassManagerBase &PM) const {
 479   // Lowers FP pseudo-instructions, e.g. ABS_Fp32 -> ABS_F.
 480   PM.add(llvm::createX86FloatingPointStackifierPass());
 481 }
 482
 483 unsigned
 484 ExegesisX86Target::getScratchMemoryRegister(const llvm::Triple &TT) const {
 485   if (!TT.isArch64Bit()) {
 486     // FIXME: This would require popping from the stack, so we would have to
 487     // add some additional setup code.
 488     return 0;
 489   }
 490   return TT.isOSWindows() ? llvm::X86::RCX : llvm::X86::RDI;
 491 }
 492
 493 void ExegesisX86Target::randomizeMCOperand(
 494     const Instruction &Instr, const Variable &Var,
 495     llvm::MCOperand &AssignedValue,
 496     const llvm::BitVector &ForbiddenRegs) const {
 497   ExegesisTarget::randomizeMCOperand(Instr, Var, AssignedValue, ForbiddenRegs);
 498
 499   const Operand &Op = Instr.getPrimaryOperand(Var);
 500   switch (Op.getExplicitOperandInfo().OperandType) {
 501   case llvm::X86::OperandType::OPERAND_COND_CODE:
 502     AssignedValue = llvm::MCOperand::createImm(
 503         randomIndex(llvm::X86::CondCode::LAST_VALID_COND));
 504     break;
 505   default:
 506     break;
 507   }
 508 }
 509
 510 void ExegesisX86Target::fillMemoryOperands(InstructionTemplate &IT,
 511                                            unsigned Reg,
 512                                            unsigned Offset) const {
 513   assert(!isInvalidMemoryInstr(IT.Instr) &&
 514          "fillMemoryOperands requires a valid memory instruction");
 515   int MemOpIdx = X86II::getMemoryOperandNo(IT.Instr.Description->TSFlags);
 516   assert(MemOpIdx >= 0 && "invalid memory operand index");
 517   // getMemoryOperandNo() ignores tied operands, so we have to add them back.
 518   for (unsigned I = 0; I <= static_cast<unsigned>(MemOpIdx); ++I) {
 519     const auto &Op = IT.Instr.Operands[I];
 520     if (Op.isTied() && Op.getTiedToIndex() < I) {
 521       ++MemOpIdx;
 522     }
 523   }
 524   // Now fill in the memory operands.
 525   const auto SetOp = [&IT](int OpIdx, const MCOperand &OpVal) {
 526     const auto Op = IT.Instr.Operands[OpIdx];
 527     assert(Op.isMemory() && Op.isExplicit() && "invalid memory pattern");
 528     IT.getValueFor(Op) = OpVal;
 529   };
 530   SetOp(MemOpIdx + 0, MCOperand::createReg(Reg));    // BaseReg
 531   SetOp(MemOpIdx + 1, MCOperand::createImm(1));      // ScaleAmt
 532   SetOp(MemOpIdx + 2, MCOperand::createReg(0));      // IndexReg
 533   SetOp(MemOpIdx + 3, MCOperand::createImm(Offset)); // Disp
 534   SetOp(MemOpIdx + 4, MCOperand::createReg(0));      // Segment
 535 }
 536
 537 std::vector<llvm::MCInst>
 538 ExegesisX86Target::setRegTo(const llvm::MCSubtargetInfo &STI, unsigned Reg,
 539                             const llvm::APInt &Value) const {
 540   if (llvm::X86::GR8RegClass.contains(Reg))
 541     return {loadImmediate(Reg, 8, Value)};
 542   if (llvm::X86::GR16RegClass.contains(Reg))
 543     return {loadImmediate(Reg, 16, Value)};
 544   if (llvm::X86::GR32RegClass.contains(Reg))
 545     return {loadImmediate(Reg, 32, Value)};
 546   if (llvm::X86::GR64RegClass.contains(Reg))
 547     return {loadImmediate(Reg, 64, Value)};
 548   ConstantInliner CI(Value);
 549   if (llvm::X86::VR64RegClass.contains(Reg))
 550     return CI.loadAndFinalize(Reg, 64, llvm::X86::MMX_MOVQ64rm);
 551   if (llvm::X86::VR128XRegClass.contains(Reg)) {
 552     if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
 553       return CI.loadAndFinalize(Reg, 128, llvm::X86::VMOVDQU32Z128rm);
 554     if (STI.getFeatureBits()[llvm::X86::FeatureAVX])
 555       return CI.loadAndFinalize(Reg, 128, llvm::X86::VMOVDQUrm);
 556     return CI.loadAndFinalize(Reg, 128, llvm::X86::MOVDQUrm);
 557   }
 558   if (llvm::X86::VR256XRegClass.contains(Reg)) {
 559     if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
 560       return CI.loadAndFinalize(Reg, 256, llvm::X86::VMOVDQU32Z256rm);
 561     if (STI.getFeatureBits()[llvm::X86::FeatureAVX])
 562       return CI.loadAndFinalize(Reg, 256, llvm::X86::VMOVDQUYrm);
 563   }
 564   if (llvm::X86::VR512RegClass.contains(Reg))
 565     if (STI.getFeatureBits()[llvm::X86::FeatureAVX512])
 566       return CI.loadAndFinalize(Reg, 512, llvm::X86::VMOVDQU32Zrm);
 567   if (llvm::X86::RSTRegClass.contains(Reg)) {
 568     return CI.loadX87STAndFinalize(Reg);
 569   }
 570   if (llvm::X86::RFP32RegClass.contains(Reg) ||
 571       llvm::X86::RFP64RegClass.contains(Reg) ||
 572       llvm::X86::RFP80RegClass.contains(Reg)) {
 573     return CI.loadX87FPAndFinalize(Reg);
 574   }
 575   if (Reg == llvm::X86::EFLAGS)
 576     return CI.popFlagAndFinalize();
 577   return {}; // Not yet implemented.
 578 }
 579
 580 static ExegesisTarget *getTheExegesisX86Target() {
 581   static ExegesisX86Target Target;
 582   return &Target;
 583 }
 584
 585 void InitializeX86ExegesisTarget() {
 586   ExegesisTarget::registerTarget(getTheExegesisX86Target());
 587 }
 588
 589 } // namespace exegesis
 590 } // namespace llvm