llvm/lib/Target/Hexagon/HexagonSplitDouble.cpp

   1 //===- HexagonSplitDouble.cpp ---------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "HexagonInstrInfo.h"
  10 #include "HexagonRegisterInfo.h"
  11 #include "HexagonSubtarget.h"
  12 #include "llvm/ADT/BitVector.h"
  13 #include "llvm/ADT/STLExtras.h"
  14 #include "llvm/ADT/SmallVector.h"
  15 #include "llvm/ADT/StringRef.h"
  16 #include "llvm/CodeGen/MachineBasicBlock.h"
  17 #include "llvm/CodeGen/MachineFunction.h"
  18 #include "llvm/CodeGen/MachineFunctionPass.h"
  19 #include "llvm/CodeGen/MachineInstr.h"
  20 #include "llvm/CodeGen/MachineInstrBuilder.h"
  21 #include "llvm/CodeGen/MachineLoopInfo.h"
  22 #include "llvm/CodeGen/MachineMemOperand.h"
  23 #include "llvm/CodeGen/MachineOperand.h"
  24 #include "llvm/CodeGen/MachineRegisterInfo.h"
  25 #include "llvm/CodeGen/TargetRegisterInfo.h"
  26 #include "llvm/Config/llvm-config.h"
  27 #include "llvm/IR/DebugLoc.h"
  28 #include "llvm/Pass.h"
  29 #include "llvm/Support/CommandLine.h"
  30 #include "llvm/Support/Compiler.h"
  31 #include "llvm/Support/Debug.h"
  32 #include "llvm/Support/ErrorHandling.h"
  33 #include "llvm/Support/raw_ostream.h"
  34 #include <algorithm>
  35 #include <cassert>
  36 #include <cstdint>
  37 #include <limits>
  38 #include <map>
  39 #include <set>
  40 #include <utility>
  41 #include <vector>
  42
  43 #define DEBUG_TYPE "hsdr"
  44
  45 using namespace llvm;
  46
  47 namespace llvm {
  48
  49   FunctionPass *createHexagonSplitDoubleRegs();
  50   void initializeHexagonSplitDoubleRegsPass(PassRegistry&);
  51
  52 } // end namespace llvm
  53
  54 static cl::opt<int> MaxHSDR("max-hsdr", cl::Hidden, cl::init(-1),
  55     cl::desc("Maximum number of split partitions"));
  56 static cl::opt<bool> MemRefsFixed("hsdr-no-mem", cl::Hidden, cl::init(true),
  57     cl::desc("Do not split loads or stores"));
  58   static cl::opt<bool> SplitAll("hsdr-split-all", cl::Hidden, cl::init(false),
  59       cl::desc("Split all partitions"));
  60
  61 namespace {
  62
  63   class HexagonSplitDoubleRegs : public MachineFunctionPass {
  64   public:
  65     static char ID;
  66
  67     HexagonSplitDoubleRegs() : MachineFunctionPass(ID) {}
  68
  69     StringRef getPassName() const override {
  70       return "Hexagon Split Double Registers";
  71     }
  72
  73     void getAnalysisUsage(AnalysisUsage &AU) const override {
  74       AU.addRequired<MachineLoopInfo>();
  75       AU.addPreserved<MachineLoopInfo>();
  76       MachineFunctionPass::getAnalysisUsage(AU);
  77     }
  78
  79     bool runOnMachineFunction(MachineFunction &MF) override;
  80
  81   private:
  82     static const TargetRegisterClass *const DoubleRC;
  83
  84     const HexagonRegisterInfo *TRI = nullptr;
  85     const HexagonInstrInfo *TII = nullptr;
  86     const MachineLoopInfo *MLI;
  87     MachineRegisterInfo *MRI;
  88
  89     using USet = std::set<unsigned>;
  90     using UUSetMap = std::map<unsigned, USet>;
  91     using UUPair = std::pair<unsigned, unsigned>;
  92     using UUPairMap = std::map<unsigned, UUPair>;
  93     using LoopRegMap = std::map<const MachineLoop *, USet>;
  94
  95     bool isInduction(unsigned Reg, LoopRegMap &IRM) const;
  96     bool isVolatileInstr(const MachineInstr *MI) const;
  97     bool isFixedInstr(const MachineInstr *MI) const;
  98     void partitionRegisters(UUSetMap &P2Rs);
  99     int32_t profit(const MachineInstr *MI) const;
 100     int32_t profit(Register Reg) const;
 101     bool isProfitable(const USet &Part, LoopRegMap &IRM) const;
 102
 103     void collectIndRegsForLoop(const MachineLoop *L, USet &Rs);
 104     void collectIndRegs(LoopRegMap &IRM);
 105
 106     void createHalfInstr(unsigned Opc, MachineInstr *MI,
 107         const UUPairMap &PairMap, unsigned SubR);
 108     void splitMemRef(MachineInstr *MI, const UUPairMap &PairMap);
 109     void splitImmediate(MachineInstr *MI, const UUPairMap &PairMap);
 110     void splitCombine(MachineInstr *MI, const UUPairMap &PairMap);
 111     void splitExt(MachineInstr *MI, const UUPairMap &PairMap);
 112     void splitShift(MachineInstr *MI, const UUPairMap &PairMap);
 113     void splitAslOr(MachineInstr *MI, const UUPairMap &PairMap);
 114     bool splitInstr(MachineInstr *MI, const UUPairMap &PairMap);
 115     void replaceSubregUses(MachineInstr *MI, const UUPairMap &PairMap);
 116     void collapseRegPairs(MachineInstr *MI, const UUPairMap &PairMap);
 117     bool splitPartition(const USet &Part);
 118
 119     static int Counter;
 120
 121     static void dump_partition(raw_ostream&, const USet&,
 122        const TargetRegisterInfo&);
 123   };
 124
 125 } // end anonymous namespace
 126
 127 char HexagonSplitDoubleRegs::ID;
 128 int HexagonSplitDoubleRegs::Counter = 0;
 129 const TargetRegisterClass *const HexagonSplitDoubleRegs::DoubleRC =
 130     &Hexagon::DoubleRegsRegClass;
 131
 132 INITIALIZE_PASS(HexagonSplitDoubleRegs, "hexagon-split-double",
 133   "Hexagon Split Double Registers", false, false)
 134
 135 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 136 LLVM_DUMP_METHOD void HexagonSplitDoubleRegs::dump_partition(raw_ostream &os,
 137       const USet &Part, const TargetRegisterInfo &TRI) {
 138   dbgs() << '{';
 139   for (auto I : Part)
 140     dbgs() << ' ' << printReg(I, &TRI);
 141   dbgs() << " }";
 142 }
 143 #endif
 144
 145 bool HexagonSplitDoubleRegs::isInduction(unsigned Reg, LoopRegMap &IRM) const {
 146   for (auto I : IRM) {
 147     const USet &Rs = I.second;
 148     if (Rs.find(Reg) != Rs.end())
 149       return true;
 150   }
 151   return false;
 152 }
 153
 154 bool HexagonSplitDoubleRegs::isVolatileInstr(const MachineInstr *MI) const {
 155   for (auto &MO : MI->memoperands())
 156     if (MO->isVolatile() || MO->isAtomic())
 157       return true;
 158   return false;
 159 }
 160
 161 bool HexagonSplitDoubleRegs::isFixedInstr(const MachineInstr *MI) const {
 162   if (MI->mayLoadOrStore())
 163     if (MemRefsFixed || isVolatileInstr(MI))
 164       return true;
 165   if (MI->isDebugInstr())
 166     return false;
 167
 168   unsigned Opc = MI->getOpcode();
 169   switch (Opc) {
 170     default:
 171       return true;
 172
 173     case TargetOpcode::PHI:
 174     case TargetOpcode::COPY:
 175       break;
 176
 177     case Hexagon::L2_loadrd_io:
 178       // Not handling stack stores (only reg-based addresses).
 179       if (MI->getOperand(1).isReg())
 180         break;
 181       return true;
 182     case Hexagon::S2_storerd_io:
 183       // Not handling stack stores (only reg-based addresses).
 184       if (MI->getOperand(0).isReg())
 185         break;
 186       return true;
 187     case Hexagon::L2_loadrd_pi:
 188     case Hexagon::S2_storerd_pi:
 189
 190     case Hexagon::A2_tfrpi:
 191     case Hexagon::A2_combineii:
 192     case Hexagon::A4_combineir:
 193     case Hexagon::A4_combineii:
 194     case Hexagon::A4_combineri:
 195     case Hexagon::A2_combinew:
 196     case Hexagon::CONST64:
 197
 198     case Hexagon::A2_sxtw:
 199
 200     case Hexagon::A2_andp:
 201     case Hexagon::A2_orp:
 202     case Hexagon::A2_xorp:
 203     case Hexagon::S2_asl_i_p_or:
 204     case Hexagon::S2_asl_i_p:
 205     case Hexagon::S2_asr_i_p:
 206     case Hexagon::S2_lsr_i_p:
 207       break;
 208   }
 209
 210   for (auto &Op : MI->operands()) {
 211     if (!Op.isReg())
 212       continue;
 213     Register R = Op.getReg();
 214     if (!R.isVirtual())
 215       return true;
 216   }
 217   return false;
 218 }
 219
 220 void HexagonSplitDoubleRegs::partitionRegisters(UUSetMap &P2Rs) {
 221   using UUMap = std::map<unsigned, unsigned>;
 222   using UVect = std::vector<unsigned>;
 223
 224   unsigned NumRegs = MRI->getNumVirtRegs();
 225   BitVector DoubleRegs(NumRegs);
 226   for (unsigned i = 0; i < NumRegs; ++i) {
 227     unsigned R = Register::index2VirtReg(i);
 228     if (MRI->getRegClass(R) == DoubleRC)
 229       DoubleRegs.set(i);
 230   }
 231
 232   BitVector FixedRegs(NumRegs);
 233   for (int x = DoubleRegs.find_first(); x >= 0; x = DoubleRegs.find_next(x)) {
 234     unsigned R = Register::index2VirtReg(x);
 235     MachineInstr *DefI = MRI->getVRegDef(R);
 236     // In some cases a register may exist, but never be defined or used.
 237     // It should never appear anywhere, but mark it as "fixed", just to be
 238     // safe.
 239     if (!DefI || isFixedInstr(DefI))
 240       FixedRegs.set(x);
 241   }
 242
 243   UUSetMap AssocMap;
 244   for (int x = DoubleRegs.find_first(); x >= 0; x = DoubleRegs.find_next(x)) {
 245     if (FixedRegs[x])
 246       continue;
 247     unsigned R = Register::index2VirtReg(x);
 248     LLVM_DEBUG(dbgs() << printReg(R, TRI) << " ~~");
 249     USet &Asc = AssocMap[R];
 250     for (auto U = MRI->use_nodbg_begin(R), Z = MRI->use_nodbg_end();
 251          U != Z; ++U) {
 252       MachineOperand &Op = *U;
 253       MachineInstr *UseI = Op.getParent();
 254       if (isFixedInstr(UseI))
 255         continue;
 256       for (unsigned i = 0, n = UseI->getNumOperands(); i < n; ++i) {
 257         MachineOperand &MO = UseI->getOperand(i);
 258         // Skip non-registers or registers with subregisters.
 259         if (&MO == &Op || !MO.isReg() || MO.getSubReg())
 260           continue;
 261         Register T = MO.getReg();
 262         if (!T.isVirtual()) {
 263           FixedRegs.set(x);
 264           continue;
 265         }
 266         if (MRI->getRegClass(T) != DoubleRC)
 267           continue;
 268         unsigned u = Register::virtReg2Index(T);
 269         if (FixedRegs[u])
 270           continue;
 271         LLVM_DEBUG(dbgs() << ' ' << printReg(T, TRI));
 272         Asc.insert(T);
 273         // Make it symmetric.
 274         AssocMap[T].insert(R);
 275       }
 276     }
 277     LLVM_DEBUG(dbgs() << '\n');
 278   }
 279
 280   UUMap R2P;
 281   unsigned NextP = 1;
 282   USet Visited;
 283   for (int x = DoubleRegs.find_first(); x >= 0; x = DoubleRegs.find_next(x)) {
 284     unsigned R = Register::index2VirtReg(x);
 285     if (Visited.count(R))
 286       continue;
 287     // Create a new partition for R.
 288     unsigned ThisP = FixedRegs[x] ? 0 : NextP++;
 289     UVect WorkQ;
 290     WorkQ.push_back(R);
 291     for (unsigned i = 0; i < WorkQ.size(); ++i) {
 292       unsigned T = WorkQ[i];
 293       if (Visited.count(T))
 294         continue;
 295       R2P[T] = ThisP;
 296       Visited.insert(T);
 297       // Add all registers associated with T.
 298       USet &Asc = AssocMap[T];
 299       append_range(WorkQ, Asc);
 300     }
 301   }
 302
 303   for (auto I : R2P)
 304     P2Rs[I.second].insert(I.first);
 305 }
 306
 307 static inline int32_t profitImm(unsigned Imm) {
 308   int32_t P = 0;
 309   if (Imm == 0 || Imm == 0xFFFFFFFF)
 310     P += 10;
 311   return P;
 312 }
 313
 314 int32_t HexagonSplitDoubleRegs::profit(const MachineInstr *MI) const {
 315   unsigned ImmX = 0;
 316   unsigned Opc = MI->getOpcode();
 317   switch (Opc) {
 318     case TargetOpcode::PHI:
 319       for (const auto &Op : MI->operands())
 320         if (!Op.getSubReg())
 321           return 0;
 322       return 10;
 323     case TargetOpcode::COPY:
 324       if (MI->getOperand(1).getSubReg() != 0)
 325         return 10;
 326       return 0;
 327
 328     case Hexagon::L2_loadrd_io:
 329     case Hexagon::S2_storerd_io:
 330       return -1;
 331     case Hexagon::L2_loadrd_pi:
 332     case Hexagon::S2_storerd_pi:
 333       return 2;
 334
 335     case Hexagon::A2_tfrpi:
 336     case Hexagon::CONST64: {
 337       uint64_t D = MI->getOperand(1).getImm();
 338       unsigned Lo = D & 0xFFFFFFFFULL;
 339       unsigned Hi = D >> 32;
 340       return profitImm(Lo) + profitImm(Hi);
 341     }
 342     case Hexagon::A2_combineii:
 343     case Hexagon::A4_combineii: {
 344       const MachineOperand &Op1 = MI->getOperand(1);
 345       const MachineOperand &Op2 = MI->getOperand(2);
 346       int32_t Prof1 = Op1.isImm() ? profitImm(Op1.getImm()) : 0;
 347       int32_t Prof2 = Op2.isImm() ? profitImm(Op2.getImm()) : 0;
 348       return Prof1 + Prof2;
 349     }
 350     case Hexagon::A4_combineri:
 351       ImmX++;
 352       // Fall through into A4_combineir.
 353       LLVM_FALLTHROUGH;
 354     case Hexagon::A4_combineir: {
 355       ImmX++;
 356       const MachineOperand &OpX = MI->getOperand(ImmX);
 357       if (OpX.isImm()) {
 358         int64_t V = OpX.getImm();
 359         if (V == 0 || V == -1)
 360           return 10;
 361       }
 362       // Fall through into A2_combinew.
 363       LLVM_FALLTHROUGH;
 364     }
 365     case Hexagon::A2_combinew:
 366       return 2;
 367
 368     case Hexagon::A2_sxtw:
 369       return 3;
 370
 371     case Hexagon::A2_andp:
 372     case Hexagon::A2_orp:
 373     case Hexagon::A2_xorp: {
 374       Register Rs = MI->getOperand(1).getReg();
 375       Register Rt = MI->getOperand(2).getReg();
 376       return profit(Rs) + profit(Rt);
 377     }
 378
 379     case Hexagon::S2_asl_i_p_or: {
 380       unsigned S = MI->getOperand(3).getImm();
 381       if (S == 0 || S == 32)
 382         return 10;
 383       return -1;
 384     }
 385     case Hexagon::S2_asl_i_p:
 386     case Hexagon::S2_asr_i_p:
 387     case Hexagon::S2_lsr_i_p:
 388       unsigned S = MI->getOperand(2).getImm();
 389       if (S == 0 || S == 32)
 390         return 10;
 391       if (S == 16)
 392         return 5;
 393       if (S == 48)
 394         return 7;
 395       return -10;
 396   }
 397
 398   return 0;
 399 }
 400
 401 int32_t HexagonSplitDoubleRegs::profit(Register Reg) const {
 402   assert(Reg.isVirtual());
 403
 404   const MachineInstr *DefI = MRI->getVRegDef(Reg);
 405   switch (DefI->getOpcode()) {
 406     case Hexagon::A2_tfrpi:
 407     case Hexagon::CONST64:
 408     case Hexagon::A2_combineii:
 409     case Hexagon::A4_combineii:
 410     case Hexagon::A4_combineri:
 411     case Hexagon::A4_combineir:
 412     case Hexagon::A2_combinew:
 413       return profit(DefI);
 414     default:
 415       break;
 416   }
 417   return 0;
 418 }
 419
 420 bool HexagonSplitDoubleRegs::isProfitable(const USet &Part, LoopRegMap &IRM)
 421       const {
 422   unsigned FixedNum = 0, LoopPhiNum = 0;
 423   int32_t TotalP = 0;
 424
 425   for (unsigned DR : Part) {
 426     MachineInstr *DefI = MRI->getVRegDef(DR);
 427     int32_t P = profit(DefI);
 428     if (P == std::numeric_limits<int>::min())
 429       return false;
 430     TotalP += P;
 431     // Reduce the profitability of splitting induction registers.
 432     if (isInduction(DR, IRM))
 433       TotalP -= 30;
 434
 435     for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end();
 436          U != W; ++U) {
 437       MachineInstr *UseI = U->getParent();
 438       if (isFixedInstr(UseI)) {
 439         FixedNum++;
 440         // Calculate the cost of generating REG_SEQUENCE instructions.
 441         for (auto &Op : UseI->operands()) {
 442           if (Op.isReg() && Part.count(Op.getReg()))
 443             if (Op.getSubReg())
 444               TotalP -= 2;
 445         }
 446         continue;
 447       }
 448       // If a register from this partition is used in a fixed instruction,
 449       // and there is also a register in this partition that is used in
 450       // a loop phi node, then decrease the splitting profit as this can
 451       // confuse the modulo scheduler.
 452       if (UseI->isPHI()) {
 453         const MachineBasicBlock *PB = UseI->getParent();
 454         const MachineLoop *L = MLI->getLoopFor(PB);
 455         if (L && L->getHeader() == PB)
 456           LoopPhiNum++;
 457       }
 458       // Splittable instruction.
 459       int32_t P = profit(UseI);
 460       if (P == std::numeric_limits<int>::min())
 461         return false;
 462       TotalP += P;
 463     }
 464   }
 465
 466   if (FixedNum > 0 && LoopPhiNum > 0)
 467     TotalP -= 20*LoopPhiNum;
 468
 469   LLVM_DEBUG(dbgs() << "Partition profit: " << TotalP << '\n');
 470   if (SplitAll)
 471     return true;
 472   return TotalP > 0;
 473 }
 474
 475 void HexagonSplitDoubleRegs::collectIndRegsForLoop(const MachineLoop *L,
 476       USet &Rs) {
 477   const MachineBasicBlock *HB = L->getHeader();
 478   const MachineBasicBlock *LB = L->getLoopLatch();
 479   if (!HB || !LB)
 480     return;
 481
 482   // Examine the latch branch. Expect it to be a conditional branch to
 483   // the header (either "br-cond header" or "br-cond exit; br header").
 484   MachineBasicBlock *TB = nullptr, *FB = nullptr;
 485   MachineBasicBlock *TmpLB = const_cast<MachineBasicBlock*>(LB);
 486   SmallVector<MachineOperand,2> Cond;
 487   bool BadLB = TII->analyzeBranch(*TmpLB, TB, FB, Cond, false);
 488   // Only analyzable conditional branches. HII::analyzeBranch will put
 489   // the branch opcode as the first element of Cond, and the predicate
 490   // operand as the second.
 491   if (BadLB || Cond.size() != 2)
 492     return;
 493   // Only simple jump-conditional (with or without negation).
 494   if (!TII->PredOpcodeHasJMP_c(Cond[0].getImm()))
 495     return;
 496   // Must go to the header.
 497   if (TB != HB && FB != HB)
 498     return;
 499   assert(Cond[1].isReg() && "Unexpected Cond vector from analyzeBranch");
 500   // Expect a predicate register.
 501   Register PR = Cond[1].getReg();
 502   assert(MRI->getRegClass(PR) == &Hexagon::PredRegsRegClass);
 503
 504   // Get the registers on which the loop controlling compare instruction
 505   // depends.
 506   Register CmpR1, CmpR2;
 507   const MachineInstr *CmpI = MRI->getVRegDef(PR);
 508   while (CmpI->getOpcode() == Hexagon::C2_not)
 509     CmpI = MRI->getVRegDef(CmpI->getOperand(1).getReg());
 510
 511   int Mask = 0, Val = 0;
 512   bool OkCI = TII->analyzeCompare(*CmpI, CmpR1, CmpR2, Mask, Val);
 513   if (!OkCI)
 514     return;
 515   // Eliminate non-double input registers.
 516   if (CmpR1 && MRI->getRegClass(CmpR1) != DoubleRC)
 517     CmpR1 = 0;
 518   if (CmpR2 && MRI->getRegClass(CmpR2) != DoubleRC)
 519     CmpR2 = 0;
 520   if (!CmpR1 && !CmpR2)
 521     return;
 522
 523   // Now examine the top of the loop: the phi nodes that could poten-
 524   // tially define loop induction registers. The registers defined by
 525   // such a phi node would be used in a 64-bit add, which then would
 526   // be used in the loop compare instruction.
 527
 528   // Get the set of all double registers defined by phi nodes in the
 529   // loop header.
 530   using UVect = std::vector<unsigned>;
 531
 532   UVect DP;
 533   for (auto &MI : *HB) {
 534     if (!MI.isPHI())
 535       break;
 536     const MachineOperand &MD = MI.getOperand(0);
 537     Register R = MD.getReg();
 538     if (MRI->getRegClass(R) == DoubleRC)
 539       DP.push_back(R);
 540   }
 541   if (DP.empty())
 542     return;
 543
 544   auto NoIndOp = [this, CmpR1, CmpR2] (unsigned R) -> bool {
 545     for (auto I = MRI->use_nodbg_begin(R), E = MRI->use_nodbg_end();
 546          I != E; ++I) {
 547       const MachineInstr *UseI = I->getParent();
 548       if (UseI->getOpcode() != Hexagon::A2_addp)
 549         continue;
 550       // Get the output from the add. If it is one of the inputs to the
 551       // loop-controlling compare instruction, then R is likely an induc-
 552       // tion register.
 553       Register T = UseI->getOperand(0).getReg();
 554       if (T == CmpR1 || T == CmpR2)
 555         return false;
 556     }
 557     return true;
 558   };
 559   UVect::iterator End = llvm::remove_if(DP, NoIndOp);
 560   Rs.insert(DP.begin(), End);
 561   Rs.insert(CmpR1);
 562   Rs.insert(CmpR2);
 563
 564   LLVM_DEBUG({
 565     dbgs() << "For loop at " << printMBBReference(*HB) << " ind regs: ";
 566     dump_partition(dbgs(), Rs, *TRI);
 567     dbgs() << '\n';
 568   });
 569 }
 570
 571 void HexagonSplitDoubleRegs::collectIndRegs(LoopRegMap &IRM) {
 572   using LoopVector = std::vector<MachineLoop *>;
 573
 574   LoopVector WorkQ;
 575
 576   append_range(WorkQ, *MLI);
 577   for (unsigned i = 0; i < WorkQ.size(); ++i)
 578     append_range(WorkQ, *WorkQ[i]);
 579
 580   USet Rs;
 581   for (unsigned i = 0, n = WorkQ.size(); i < n; ++i) {
 582     MachineLoop *L = WorkQ[i];
 583     Rs.clear();
 584     collectIndRegsForLoop(L, Rs);
 585     if (!Rs.empty())
 586       IRM.insert(std::make_pair(L, Rs));
 587   }
 588 }
 589
 590 void HexagonSplitDoubleRegs::createHalfInstr(unsigned Opc, MachineInstr *MI,
 591       const UUPairMap &PairMap, unsigned SubR) {
 592   MachineBasicBlock &B = *MI->getParent();
 593   DebugLoc DL = MI->getDebugLoc();
 594   MachineInstr *NewI = BuildMI(B, MI, DL, TII->get(Opc));
 595
 596   for (auto &Op : MI->operands()) {
 597     if (!Op.isReg()) {
 598       NewI->addOperand(Op);
 599       continue;
 600     }
 601     // For register operands, set the subregister.
 602     Register R = Op.getReg();
 603     unsigned SR = Op.getSubReg();
 604     bool isVirtReg = R.isVirtual();
 605     bool isKill = Op.isKill();
 606     if (isVirtReg && MRI->getRegClass(R) == DoubleRC) {
 607       isKill = false;
 608       UUPairMap::const_iterator F = PairMap.find(R);
 609       if (F == PairMap.end()) {
 610         SR = SubR;
 611       } else {
 612         const UUPair &P = F->second;
 613         R = (SubR == Hexagon::isub_lo) ? P.first : P.second;
 614         SR = 0;
 615       }
 616     }
 617     auto CO = MachineOperand::CreateReg(R, Op.isDef(), Op.isImplicit(), isKill,
 618           Op.isDead(), Op.isUndef(), Op.isEarlyClobber(), SR, Op.isDebug(),
 619           Op.isInternalRead());
 620     NewI->addOperand(CO);
 621   }
 622 }
 623
 624 void HexagonSplitDoubleRegs::splitMemRef(MachineInstr *MI,
 625       const UUPairMap &PairMap) {
 626   bool Load = MI->mayLoad();
 627   unsigned OrigOpc = MI->getOpcode();
 628   bool PostInc = (OrigOpc == Hexagon::L2_loadrd_pi ||
 629                   OrigOpc == Hexagon::S2_storerd_pi);
 630   MachineInstr *LowI, *HighI;
 631   MachineBasicBlock &B = *MI->getParent();
 632   DebugLoc DL = MI->getDebugLoc();
 633
 634   // Index of the base-address-register operand.
 635   unsigned AdrX = PostInc ? (Load ? 2 : 1)
 636                           : (Load ? 1 : 0);
 637   MachineOperand &AdrOp = MI->getOperand(AdrX);
 638   unsigned RSA = getRegState(AdrOp);
 639   MachineOperand &ValOp = Load ? MI->getOperand(0)
 640                                : (PostInc ? MI->getOperand(3)
 641                                           : MI->getOperand(2));
 642   UUPairMap::const_iterator F = PairMap.find(ValOp.getReg());
 643   assert(F != PairMap.end());
 644
 645   if (Load) {
 646     const UUPair &P = F->second;
 647     int64_t Off = PostInc ? 0 : MI->getOperand(2).getImm();
 648     LowI = BuildMI(B, MI, DL, TII->get(Hexagon::L2_loadri_io), P.first)
 649              .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg())
 650              .addImm(Off);
 651     HighI = BuildMI(B, MI, DL, TII->get(Hexagon::L2_loadri_io), P.second)
 652               .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg())
 653               .addImm(Off+4);
 654   } else {
 655     const UUPair &P = F->second;
 656     int64_t Off = PostInc ? 0 : MI->getOperand(1).getImm();
 657     LowI = BuildMI(B, MI, DL, TII->get(Hexagon::S2_storeri_io))
 658              .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg())
 659              .addImm(Off)
 660              .addReg(P.first);
 661     HighI = BuildMI(B, MI, DL, TII->get(Hexagon::S2_storeri_io))
 662               .addReg(AdrOp.getReg(), RSA & ~RegState::Kill, AdrOp.getSubReg())
 663               .addImm(Off+4)
 664               .addReg(P.second);
 665   }
 666
 667   if (PostInc) {
 668     // Create the increment of the address register.
 669     int64_t Inc = Load ? MI->getOperand(3).getImm()
 670                        : MI->getOperand(2).getImm();
 671     MachineOperand &UpdOp = Load ? MI->getOperand(1) : MI->getOperand(0);
 672     const TargetRegisterClass *RC = MRI->getRegClass(UpdOp.getReg());
 673     Register NewR = MRI->createVirtualRegister(RC);
 674     assert(!UpdOp.getSubReg() && "Def operand with subreg");
 675     BuildMI(B, MI, DL, TII->get(Hexagon::A2_addi), NewR)
 676       .addReg(AdrOp.getReg(), RSA)
 677       .addImm(Inc);
 678     MRI->replaceRegWith(UpdOp.getReg(), NewR);
 679     // The original instruction will be deleted later.
 680   }
 681
 682   // Generate a new pair of memory-operands.
 683   MachineFunction &MF = *B.getParent();
 684   for (auto &MO : MI->memoperands()) {
 685     const MachinePointerInfo &Ptr = MO->getPointerInfo();
 686     MachineMemOperand::Flags F = MO->getFlags();
 687     Align A = MO->getAlign();
 688
 689     auto *Tmp1 = MF.getMachineMemOperand(Ptr, F, 4 /*size*/, A);
 690     LowI->addMemOperand(MF, Tmp1);
 691     auto *Tmp2 =
 692         MF.getMachineMemOperand(Ptr, F, 4 /*size*/, std::min(A, Align(4)));
 693     HighI->addMemOperand(MF, Tmp2);
 694   }
 695 }
 696
 697 void HexagonSplitDoubleRegs::splitImmediate(MachineInstr *MI,
 698       const UUPairMap &PairMap) {
 699   MachineOperand &Op0 = MI->getOperand(0);
 700   MachineOperand &Op1 = MI->getOperand(1);
 701   assert(Op0.isReg() && Op1.isImm());
 702   uint64_t V = Op1.getImm();
 703
 704   MachineBasicBlock &B = *MI->getParent();
 705   DebugLoc DL = MI->getDebugLoc();
 706   UUPairMap::const_iterator F = PairMap.find(Op0.getReg());
 707   assert(F != PairMap.end());
 708   const UUPair &P = F->second;
 709
 710   // The operand to A2_tfrsi can only have 32 significant bits. Immediate
 711   // values in MachineOperand are stored as 64-bit integers, and so the
 712   // value -1 may be represented either as 64-bit -1, or 4294967295. Both
 713   // will have the 32 higher bits truncated in the end, but -1 will remain
 714   // as -1, while the latter may appear to be a large unsigned value
 715   // requiring a constant extender. The casting to int32_t will select the
 716   // former representation. (The same reasoning applies to all 32-bit
 717   // values.)
 718   BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.first)
 719     .addImm(int32_t(V & 0xFFFFFFFFULL));
 720   BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.second)
 721     .addImm(int32_t(V >> 32));
 722 }
 723
 724 void HexagonSplitDoubleRegs::splitCombine(MachineInstr *MI,
 725       const UUPairMap &PairMap) {
 726   MachineOperand &Op0 = MI->getOperand(0);
 727   MachineOperand &Op1 = MI->getOperand(1);
 728   MachineOperand &Op2 = MI->getOperand(2);
 729   assert(Op0.isReg());
 730
 731   MachineBasicBlock &B = *MI->getParent();
 732   DebugLoc DL = MI->getDebugLoc();
 733   UUPairMap::const_iterator F = PairMap.find(Op0.getReg());
 734   assert(F != PairMap.end());
 735   const UUPair &P = F->second;
 736
 737   if (!Op1.isReg()) {
 738     BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.second)
 739       .add(Op1);
 740   } else {
 741     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.second)
 742       .addReg(Op1.getReg(), getRegState(Op1), Op1.getSubReg());
 743   }
 744
 745   if (!Op2.isReg()) {
 746     BuildMI(B, MI, DL, TII->get(Hexagon::A2_tfrsi), P.first)
 747       .add(Op2);
 748   } else {
 749     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.first)
 750       .addReg(Op2.getReg(), getRegState(Op2), Op2.getSubReg());
 751   }
 752 }
 753
 754 void HexagonSplitDoubleRegs::splitExt(MachineInstr *MI,
 755       const UUPairMap &PairMap) {
 756   MachineOperand &Op0 = MI->getOperand(0);
 757   MachineOperand &Op1 = MI->getOperand(1);
 758   assert(Op0.isReg() && Op1.isReg());
 759
 760   MachineBasicBlock &B = *MI->getParent();
 761   DebugLoc DL = MI->getDebugLoc();
 762   UUPairMap::const_iterator F = PairMap.find(Op0.getReg());
 763   assert(F != PairMap.end());
 764   const UUPair &P = F->second;
 765   unsigned RS = getRegState(Op1);
 766
 767   BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), P.first)
 768     .addReg(Op1.getReg(), RS & ~RegState::Kill, Op1.getSubReg());
 769   BuildMI(B, MI, DL, TII->get(Hexagon::S2_asr_i_r), P.second)
 770     .addReg(Op1.getReg(), RS, Op1.getSubReg())
 771     .addImm(31);
 772 }
 773
 774 void HexagonSplitDoubleRegs::splitShift(MachineInstr *MI,
 775       const UUPairMap &PairMap) {
 776   using namespace Hexagon;
 777
 778   MachineOperand &Op0 = MI->getOperand(0);
 779   MachineOperand &Op1 = MI->getOperand(1);
 780   MachineOperand &Op2 = MI->getOperand(2);
 781   assert(Op0.isReg() && Op1.isReg() && Op2.isImm());
 782   int64_t Sh64 = Op2.getImm();
 783   assert(Sh64 >= 0 && Sh64 < 64);
 784   unsigned S = Sh64;
 785
 786   UUPairMap::const_iterator F = PairMap.find(Op0.getReg());
 787   assert(F != PairMap.end());
 788   const UUPair &P = F->second;
 789   Register LoR = P.first;
 790   Register HiR = P.second;
 791
 792   unsigned Opc = MI->getOpcode();
 793   bool Right = (Opc == S2_lsr_i_p || Opc == S2_asr_i_p);
 794   bool Left = !Right;
 795   bool Signed = (Opc == S2_asr_i_p);
 796
 797   MachineBasicBlock &B = *MI->getParent();
 798   DebugLoc DL = MI->getDebugLoc();
 799   unsigned RS = getRegState(Op1);
 800   unsigned ShiftOpc = Left ? S2_asl_i_r
 801                            : (Signed ? S2_asr_i_r : S2_lsr_i_r);
 802   unsigned LoSR = isub_lo;
 803   unsigned HiSR = isub_hi;
 804
 805   if (S == 0) {
 806     // No shift, subregister copy.
 807     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR)
 808       .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR);
 809     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), HiR)
 810       .addReg(Op1.getReg(), RS, HiSR);
 811   } else if (S < 32) {
 812     const TargetRegisterClass *IntRC = &IntRegsRegClass;
 813     Register TmpR = MRI->createVirtualRegister(IntRC);
 814     // Expansion:
 815     // Shift left:    DR = shl R, #s
 816     //   LoR  = shl R.lo, #s
 817     //   TmpR = extractu R.lo, #s, #32-s
 818     //   HiR  = or (TmpR, asl(R.hi, #s))
 819     // Shift right:   DR = shr R, #s
 820     //   HiR  = shr R.hi, #s
 821     //   TmpR = shr R.lo, #s
 822     //   LoR  = insert TmpR, R.hi, #s, #32-s
 823
 824     // Shift left:
 825     //   LoR  = shl R.lo, #s
 826     // Shift right:
 827     //   TmpR = shr R.lo, #s
 828
 829     // Make a special case for A2_aslh and A2_asrh (they are predicable as
 830     // opposed to S2_asl_i_r/S2_asr_i_r).
 831     if (S == 16 && Left)
 832       BuildMI(B, MI, DL, TII->get(A2_aslh), LoR)
 833         .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR);
 834     else if (S == 16 && Signed)
 835       BuildMI(B, MI, DL, TII->get(A2_asrh), TmpR)
 836         .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR);
 837     else
 838       BuildMI(B, MI, DL, TII->get(ShiftOpc), (Left ? LoR : TmpR))
 839         .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR)
 840         .addImm(S);
 841
 842     if (Left) {
 843       // TmpR = extractu R.lo, #s, #32-s
 844       BuildMI(B, MI, DL, TII->get(S2_extractu), TmpR)
 845         .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR)
 846         .addImm(S)
 847         .addImm(32-S);
 848       // HiR  = or (TmpR, asl(R.hi, #s))
 849       BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR)
 850         .addReg(TmpR)
 851         .addReg(Op1.getReg(), RS, HiSR)
 852         .addImm(S);
 853     } else {
 854       // HiR  = shr R.hi, #s
 855       BuildMI(B, MI, DL, TII->get(ShiftOpc), HiR)
 856         .addReg(Op1.getReg(), RS & ~RegState::Kill, HiSR)
 857         .addImm(S);
 858       // LoR  = insert TmpR, R.hi, #s, #32-s
 859       BuildMI(B, MI, DL, TII->get(S2_insert), LoR)
 860         .addReg(TmpR)
 861         .addReg(Op1.getReg(), RS, HiSR)
 862         .addImm(S)
 863         .addImm(32-S);
 864     }
 865   } else if (S == 32) {
 866     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), (Left ? HiR : LoR))
 867       .addReg(Op1.getReg(), RS & ~RegState::Kill, (Left ? LoSR : HiSR));
 868     if (!Signed)
 869       BuildMI(B, MI, DL, TII->get(A2_tfrsi), (Left ? LoR : HiR))
 870         .addImm(0);
 871     else  // Must be right shift.
 872       BuildMI(B, MI, DL, TII->get(S2_asr_i_r), HiR)
 873         .addReg(Op1.getReg(), RS, HiSR)
 874         .addImm(31);
 875   } else if (S < 64) {
 876     S -= 32;
 877     if (S == 16 && Left)
 878       BuildMI(B, MI, DL, TII->get(A2_aslh), HiR)
 879         .addReg(Op1.getReg(), RS & ~RegState::Kill, LoSR);
 880     else if (S == 16 && Signed)
 881       BuildMI(B, MI, DL, TII->get(A2_asrh), LoR)
 882         .addReg(Op1.getReg(), RS & ~RegState::Kill, HiSR);
 883     else
 884       BuildMI(B, MI, DL, TII->get(ShiftOpc), (Left ? HiR : LoR))
 885         .addReg(Op1.getReg(), RS & ~RegState::Kill, (Left ? LoSR : HiSR))
 886         .addImm(S);
 887
 888     if (Signed)
 889       BuildMI(B, MI, DL, TII->get(S2_asr_i_r), HiR)
 890         .addReg(Op1.getReg(), RS, HiSR)
 891         .addImm(31);
 892     else
 893       BuildMI(B, MI, DL, TII->get(A2_tfrsi), (Left ? LoR : HiR))
 894         .addImm(0);
 895   }
 896 }
 897
 898 void HexagonSplitDoubleRegs::splitAslOr(MachineInstr *MI,
 899       const UUPairMap &PairMap) {
 900   using namespace Hexagon;
 901
 902   MachineOperand &Op0 = MI->getOperand(0);
 903   MachineOperand &Op1 = MI->getOperand(1);
 904   MachineOperand &Op2 = MI->getOperand(2);
 905   MachineOperand &Op3 = MI->getOperand(3);
 906   assert(Op0.isReg() && Op1.isReg() && Op2.isReg() && Op3.isImm());
 907   int64_t Sh64 = Op3.getImm();
 908   assert(Sh64 >= 0 && Sh64 < 64);
 909   unsigned S = Sh64;
 910
 911   UUPairMap::const_iterator F = PairMap.find(Op0.getReg());
 912   assert(F != PairMap.end());
 913   const UUPair &P = F->second;
 914   unsigned LoR = P.first;
 915   unsigned HiR = P.second;
 916
 917   MachineBasicBlock &B = *MI->getParent();
 918   DebugLoc DL = MI->getDebugLoc();
 919   unsigned RS1 = getRegState(Op1);
 920   unsigned RS2 = getRegState(Op2);
 921   const TargetRegisterClass *IntRC = &IntRegsRegClass;
 922
 923   unsigned LoSR = isub_lo;
 924   unsigned HiSR = isub_hi;
 925
 926   // Op0 = S2_asl_i_p_or Op1, Op2, Op3
 927   // means:  Op0 = or (Op1, asl(Op2, Op3))
 928
 929   // Expansion of
 930   //   DR = or (R1, asl(R2, #s))
 931   //
 932   //   LoR  = or (R1.lo, asl(R2.lo, #s))
 933   //   Tmp1 = extractu R2.lo, #s, #32-s
 934   //   Tmp2 = or R1.hi, Tmp1
 935   //   HiR  = or (Tmp2, asl(R2.hi, #s))
 936
 937   if (S == 0) {
 938     // DR  = or (R1, asl(R2, #0))
 939     //    -> or (R1, R2)
 940     // i.e. LoR = or R1.lo, R2.lo
 941     //      HiR = or R1.hi, R2.hi
 942     BuildMI(B, MI, DL, TII->get(A2_or), LoR)
 943       .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR)
 944       .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR);
 945     BuildMI(B, MI, DL, TII->get(A2_or), HiR)
 946       .addReg(Op1.getReg(), RS1, HiSR)
 947       .addReg(Op2.getReg(), RS2, HiSR);
 948   } else if (S < 32) {
 949     BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), LoR)
 950       .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR)
 951       .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR)
 952       .addImm(S);
 953     Register TmpR1 = MRI->createVirtualRegister(IntRC);
 954     BuildMI(B, MI, DL, TII->get(S2_extractu), TmpR1)
 955       .addReg(Op2.getReg(), RS2 & ~RegState::Kill, LoSR)
 956       .addImm(S)
 957       .addImm(32-S);
 958     Register TmpR2 = MRI->createVirtualRegister(IntRC);
 959     BuildMI(B, MI, DL, TII->get(A2_or), TmpR2)
 960       .addReg(Op1.getReg(), RS1, HiSR)
 961       .addReg(TmpR1);
 962     BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR)
 963       .addReg(TmpR2)
 964       .addReg(Op2.getReg(), RS2, HiSR)
 965       .addImm(S);
 966   } else if (S == 32) {
 967     // DR  = or (R1, asl(R2, #32))
 968     //    -> or R1, R2.lo
 969     // LoR = R1.lo
 970     // HiR = or R1.hi, R2.lo
 971     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR)
 972       .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR);
 973     BuildMI(B, MI, DL, TII->get(A2_or), HiR)
 974       .addReg(Op1.getReg(), RS1, HiSR)
 975       .addReg(Op2.getReg(), RS2, LoSR);
 976   } else if (S < 64) {
 977     // DR  = or (R1, asl(R2, #s))
 978     //
 979     // LoR = R1:lo
 980     // HiR = or (R1:hi, asl(R2:lo, #s-32))
 981     S -= 32;
 982     BuildMI(B, MI, DL, TII->get(TargetOpcode::COPY), LoR)
 983       .addReg(Op1.getReg(), RS1 & ~RegState::Kill, LoSR);
 984     BuildMI(B, MI, DL, TII->get(S2_asl_i_r_or), HiR)
 985       .addReg(Op1.getReg(), RS1, HiSR)
 986       .addReg(Op2.getReg(), RS2, LoSR)
 987       .addImm(S);
 988   }
 989 }
 990
 991 bool HexagonSplitDoubleRegs::splitInstr(MachineInstr *MI,
 992       const UUPairMap &PairMap) {
 993   using namespace Hexagon;
 994
 995   LLVM_DEBUG(dbgs() << "Splitting: " << *MI);
 996   bool Split = false;
 997   unsigned Opc = MI->getOpcode();
 998
 999   switch (Opc) {
1000     case TargetOpcode::PHI:
1001     case TargetOpcode::COPY: {
1002       Register DstR = MI->getOperand(0).getReg();
1003       if (MRI->getRegClass(DstR) == DoubleRC) {
1004         createHalfInstr(Opc, MI, PairMap, isub_lo);
1005         createHalfInstr(Opc, MI, PairMap, isub_hi);
1006         Split = true;
1007       }
1008       break;
1009     }
1010     case A2_andp:
1011       createHalfInstr(A2_and, MI, PairMap, isub_lo);
1012       createHalfInstr(A2_and, MI, PairMap, isub_hi);
1013       Split = true;
1014       break;
1015     case A2_orp:
1016       createHalfInstr(A2_or, MI, PairMap, isub_lo);
1017       createHalfInstr(A2_or, MI, PairMap, isub_hi);
1018       Split = true;
1019       break;
1020     case A2_xorp:
1021       createHalfInstr(A2_xor, MI, PairMap, isub_lo);
1022       createHalfInstr(A2_xor, MI, PairMap, isub_hi);
1023       Split = true;
1024       break;
1025
1026     case L2_loadrd_io:
1027     case L2_loadrd_pi:
1028     case S2_storerd_io:
1029     case S2_storerd_pi:
1030       splitMemRef(MI, PairMap);
1031       Split = true;
1032       break;
1033
1034     case A2_tfrpi:
1035     case CONST64:
1036       splitImmediate(MI, PairMap);
1037       Split = true;
1038       break;
1039
1040     case A2_combineii:
1041     case A4_combineir:
1042     case A4_combineii:
1043     case A4_combineri:
1044     case A2_combinew:
1045       splitCombine(MI, PairMap);
1046       Split = true;
1047       break;
1048
1049     case A2_sxtw:
1050       splitExt(MI, PairMap);
1051       Split = true;
1052       break;
1053
1054     case S2_asl_i_p:
1055     case S2_asr_i_p:
1056     case S2_lsr_i_p:
1057       splitShift(MI, PairMap);
1058       Split = true;
1059       break;
1060
1061     case S2_asl_i_p_or:
1062       splitAslOr(MI, PairMap);
1063       Split = true;
1064       break;
1065
1066     default:
1067       llvm_unreachable("Instruction not splitable");
1068       return false;
1069   }
1070
1071   return Split;
1072 }
1073
1074 void HexagonSplitDoubleRegs::replaceSubregUses(MachineInstr *MI,
1075       const UUPairMap &PairMap) {
1076   for (auto &Op : MI->operands()) {
1077     if (!Op.isReg() || !Op.isUse() || !Op.getSubReg())
1078       continue;
1079     Register R = Op.getReg();
1080     UUPairMap::const_iterator F = PairMap.find(R);
1081     if (F == PairMap.end())
1082       continue;
1083     const UUPair &P = F->second;
1084     switch (Op.getSubReg()) {
1085       case Hexagon::isub_lo:
1086         Op.setReg(P.first);
1087         break;
1088       case Hexagon::isub_hi:
1089         Op.setReg(P.second);
1090         break;
1091     }
1092     Op.setSubReg(0);
1093   }
1094 }
1095
1096 void HexagonSplitDoubleRegs::collapseRegPairs(MachineInstr *MI,
1097       const UUPairMap &PairMap) {
1098   MachineBasicBlock &B = *MI->getParent();
1099   DebugLoc DL = MI->getDebugLoc();
1100
1101   for (auto &Op : MI->operands()) {
1102     if (!Op.isReg() || !Op.isUse())
1103       continue;
1104     Register R = Op.getReg();
1105     if (!R.isVirtual())
1106       continue;
1107     if (MRI->getRegClass(R) != DoubleRC || Op.getSubReg())
1108       continue;
1109     UUPairMap::const_iterator F = PairMap.find(R);
1110     if (F == PairMap.end())
1111       continue;
1112     const UUPair &Pr = F->second;
1113     Register NewDR = MRI->createVirtualRegister(DoubleRC);
1114     BuildMI(B, MI, DL, TII->get(TargetOpcode::REG_SEQUENCE), NewDR)
1115       .addReg(Pr.first)
1116       .addImm(Hexagon::isub_lo)
1117       .addReg(Pr.second)
1118       .addImm(Hexagon::isub_hi);
1119     Op.setReg(NewDR);
1120   }
1121 }
1122
1123 bool HexagonSplitDoubleRegs::splitPartition(const USet &Part) {
1124   using MISet = std::set<MachineInstr *>;
1125
1126   const TargetRegisterClass *IntRC = &Hexagon::IntRegsRegClass;
1127   bool Changed = false;
1128
1129   LLVM_DEBUG(dbgs() << "Splitting partition: ";
1130              dump_partition(dbgs(), Part, *TRI); dbgs() << '\n');
1131
1132   UUPairMap PairMap;
1133
1134   MISet SplitIns;
1135   for (unsigned DR : Part) {
1136     MachineInstr *DefI = MRI->getVRegDef(DR);
1137     SplitIns.insert(DefI);
1138
1139     // Collect all instructions, including fixed ones.  We won't split them,
1140     // but we need to visit them again to insert the REG_SEQUENCE instructions.
1141     for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end();
1142          U != W; ++U)
1143       SplitIns.insert(U->getParent());
1144
1145     Register LoR = MRI->createVirtualRegister(IntRC);
1146     Register HiR = MRI->createVirtualRegister(IntRC);
1147     LLVM_DEBUG(dbgs() << "Created mapping: " << printReg(DR, TRI) << " -> "
1148                       << printReg(HiR, TRI) << ':' << printReg(LoR, TRI)
1149                       << '\n');
1150     PairMap.insert(std::make_pair(DR, UUPair(LoR, HiR)));
1151   }
1152
1153   MISet Erase;
1154   for (auto MI : SplitIns) {
1155     if (isFixedInstr(MI)) {
1156       collapseRegPairs(MI, PairMap);
1157     } else {
1158       bool Done = splitInstr(MI, PairMap);
1159       if (Done)
1160         Erase.insert(MI);
1161       Changed |= Done;
1162     }
1163   }
1164
1165   for (unsigned DR : Part) {
1166     // Before erasing "double" instructions, revisit all uses of the double
1167     // registers in this partition, and replace all uses of them with subre-
1168     // gisters, with the corresponding single registers.
1169     MISet Uses;
1170     for (auto U = MRI->use_nodbg_begin(DR), W = MRI->use_nodbg_end();
1171          U != W; ++U)
1172       Uses.insert(U->getParent());
1173     for (auto M : Uses)
1174       replaceSubregUses(M, PairMap);
1175   }
1176
1177   for (auto MI : Erase) {
1178     MachineBasicBlock *B = MI->getParent();
1179     B->erase(MI);
1180   }
1181
1182   return Changed;
1183 }
1184
1185 bool HexagonSplitDoubleRegs::runOnMachineFunction(MachineFunction &MF) {
1186   if (skipFunction(MF.getFunction()))
1187     return false;
1188
1189   LLVM_DEBUG(dbgs() << "Splitting double registers in function: "
1190                     << MF.getName() << '\n');
1191
1192   auto &ST = MF.getSubtarget<HexagonSubtarget>();
1193   TRI = ST.getRegisterInfo();
1194   TII = ST.getInstrInfo();
1195   MRI = &MF.getRegInfo();
1196   MLI = &getAnalysis<MachineLoopInfo>();
1197
1198   UUSetMap P2Rs;
1199   LoopRegMap IRM;
1200
1201   collectIndRegs(IRM);
1202   partitionRegisters(P2Rs);
1203
1204   LLVM_DEBUG({
1205     dbgs() << "Register partitioning: (partition #0 is fixed)\n";
1206     for (UUSetMap::iterator I = P2Rs.begin(), E = P2Rs.end(); I != E; ++I) {
1207       dbgs() << '#' << I->first << " -> ";
1208       dump_partition(dbgs(), I->second, *TRI);
1209       dbgs() << '\n';
1210     }
1211   });
1212
1213   bool Changed = false;
1214   int Limit = MaxHSDR;
1215
1216   for (UUSetMap::iterator I = P2Rs.begin(), E = P2Rs.end(); I != E; ++I) {
1217     if (I->first == 0)
1218       continue;
1219     if (Limit >= 0 && Counter >= Limit)
1220       break;
1221     USet &Part = I->second;
1222     LLVM_DEBUG(dbgs() << "Calculating profit for partition #" << I->first
1223                       << '\n');
1224     if (!isProfitable(Part, IRM))
1225       continue;
1226     Counter++;
1227     Changed |= splitPartition(Part);
1228   }
1229
1230   return Changed;
1231 }
1232
1233 FunctionPass *llvm::createHexagonSplitDoubleRegs() {
1234   return new HexagonSplitDoubleRegs();
1235 }