lib/Target/ARM/ARMLoadStoreOptimizer.cpp

   1 //===- ARMLoadStoreOptimizer.cpp - ARM load / store opt. pass -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file contains a pass that performs load / store related peephole
  10 /// optimizations. This pass should be run after register allocation.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "ARM.h"
  15 #include "ARMBaseInstrInfo.h"
  16 #include "ARMBaseRegisterInfo.h"
  17 #include "ARMISelLowering.h"
  18 #include "ARMMachineFunctionInfo.h"
  19 #include "ARMSubtarget.h"
  20 #include "MCTargetDesc/ARMAddressingModes.h"
  21 #include "MCTargetDesc/ARMBaseInfo.h"
  22 #include "Utils/ARMBaseInfo.h"
  23 #include "llvm/ADT/ArrayRef.h"
  24 #include "llvm/ADT/DenseMap.h"
  25 #include "llvm/ADT/DenseSet.h"
  26 #include "llvm/ADT/STLExtras.h"
  27 #include "llvm/ADT/SmallPtrSet.h"
  28 #include "llvm/ADT/SmallSet.h"
  29 #include "llvm/ADT/SmallVector.h"
  30 #include "llvm/ADT/Statistic.h"
  31 #include "llvm/ADT/iterator_range.h"
  32 #include "llvm/Analysis/AliasAnalysis.h"
  33 #include "llvm/CodeGen/LivePhysRegs.h"
  34 #include "llvm/CodeGen/MachineBasicBlock.h"
  35 #include "llvm/CodeGen/MachineFunction.h"
  36 #include "llvm/CodeGen/MachineFunctionPass.h"
  37 #include "llvm/CodeGen/MachineInstr.h"
  38 #include "llvm/CodeGen/MachineInstrBuilder.h"
  39 #include "llvm/CodeGen/MachineMemOperand.h"
  40 #include "llvm/CodeGen/MachineOperand.h"
  41 #include "llvm/CodeGen/MachineRegisterInfo.h"
  42 #include "llvm/CodeGen/RegisterClassInfo.h"
  43 #include "llvm/CodeGen/TargetFrameLowering.h"
  44 #include "llvm/CodeGen/TargetInstrInfo.h"
  45 #include "llvm/CodeGen/TargetLowering.h"
  46 #include "llvm/CodeGen/TargetRegisterInfo.h"
  47 #include "llvm/CodeGen/TargetSubtargetInfo.h"
  48 #include "llvm/IR/DataLayout.h"
  49 #include "llvm/IR/DebugLoc.h"
  50 #include "llvm/IR/DerivedTypes.h"
  51 #include "llvm/IR/Function.h"
  52 #include "llvm/IR/Type.h"
  53 #include "llvm/MC/MCInstrDesc.h"
  54 #include "llvm/Pass.h"
  55 #include "llvm/Support/Allocator.h"
  56 #include "llvm/Support/CommandLine.h"
  57 #include "llvm/Support/Debug.h"
  58 #include "llvm/Support/ErrorHandling.h"
  59 #include "llvm/Support/raw_ostream.h"
  60 #include <algorithm>
  61 #include <cassert>
  62 #include <cstddef>
  63 #include <cstdlib>
  64 #include <iterator>
  65 #include <limits>
  66 #include <utility>
  67
  68 using namespace llvm;
  69
  70 #define DEBUG_TYPE "arm-ldst-opt"
  71
  72 STATISTIC(NumLDMGened , "Number of ldm instructions generated");
  73 STATISTIC(NumSTMGened , "Number of stm instructions generated");
  74 STATISTIC(NumVLDMGened, "Number of vldm instructions generated");
  75 STATISTIC(NumVSTMGened, "Number of vstm instructions generated");
  76 STATISTIC(NumLdStMoved, "Number of load / store instructions moved");
  77 STATISTIC(NumLDRDFormed,"Number of ldrd created before allocation");
  78 STATISTIC(NumSTRDFormed,"Number of strd created before allocation");
  79 STATISTIC(NumLDRD2LDM,  "Number of ldrd instructions turned back into ldm");
  80 STATISTIC(NumSTRD2STM,  "Number of strd instructions turned back into stm");
  81 STATISTIC(NumLDRD2LDR,  "Number of ldrd instructions turned back into ldr's");
  82 STATISTIC(NumSTRD2STR,  "Number of strd instructions turned back into str's");
  83
  84 /// This switch disables formation of double/multi instructions that could
  85 /// potentially lead to (new) alignment traps even with CCR.UNALIGN_TRP
  86 /// disabled. This can be used to create libraries that are robust even when
  87 /// users provoke undefined behaviour by supplying misaligned pointers.
  88 /// \see mayCombineMisaligned()
  89 static cl::opt<bool>
  90 AssumeMisalignedLoadStores("arm-assume-misaligned-load-store", cl::Hidden,
  91     cl::init(false), cl::desc("Be more conservative in ARM load/store opt"));
  92
  93 #define ARM_LOAD_STORE_OPT_NAME "ARM load / store optimization pass"
  94
  95 namespace {
  96
  97   /// Post- register allocation pass the combine load / store instructions to
  98   /// form ldm / stm instructions.
  99   struct ARMLoadStoreOpt : public MachineFunctionPass {
 100     static char ID;
 101
 102     const MachineFunction *MF;
 103     const TargetInstrInfo *TII;
 104     const TargetRegisterInfo *TRI;
 105     const ARMSubtarget *STI;
 106     const TargetLowering *TL;
 107     ARMFunctionInfo *AFI;
 108     LivePhysRegs LiveRegs;
 109     RegisterClassInfo RegClassInfo;
 110     MachineBasicBlock::const_iterator LiveRegPos;
 111     bool LiveRegsValid;
 112     bool RegClassInfoValid;
 113     bool isThumb1, isThumb2;
 114
 115     ARMLoadStoreOpt() : MachineFunctionPass(ID) {}
 116
 117     bool runOnMachineFunction(MachineFunction &Fn) override;
 118
 119     MachineFunctionProperties getRequiredProperties() const override {
 120       return MachineFunctionProperties().set(
 121           MachineFunctionProperties::Property::NoVRegs);
 122     }
 123
 124     StringRef getPassName() const override { return ARM_LOAD_STORE_OPT_NAME; }
 125
 126   private:
 127     /// A set of load/store MachineInstrs with same base register sorted by
 128     /// offset.
 129     struct MemOpQueueEntry {
 130       MachineInstr *MI;
 131       int Offset;        ///< Load/Store offset.
 132       unsigned Position; ///< Position as counted from end of basic block.
 133
 134       MemOpQueueEntry(MachineInstr &MI, int Offset, unsigned Position)
 135           : MI(&MI), Offset(Offset), Position(Position) {}
 136     };
 137     using MemOpQueue = SmallVector<MemOpQueueEntry, 8>;
 138
 139     /// A set of MachineInstrs that fulfill (nearly all) conditions to get
 140     /// merged into a LDM/STM.
 141     struct MergeCandidate {
 142       /// List of instructions ordered by load/store offset.
 143       SmallVector<MachineInstr*, 4> Instrs;
 144
 145       /// Index in Instrs of the instruction being latest in the schedule.
 146       unsigned LatestMIIdx;
 147
 148       /// Index in Instrs of the instruction being earliest in the schedule.
 149       unsigned EarliestMIIdx;
 150
 151       /// Index into the basic block where the merged instruction will be
 152       /// inserted. (See MemOpQueueEntry.Position)
 153       unsigned InsertPos;
 154
 155       /// Whether the instructions can be merged into a ldm/stm instruction.
 156       bool CanMergeToLSMulti;
 157
 158       /// Whether the instructions can be merged into a ldrd/strd instruction.
 159       bool CanMergeToLSDouble;
 160     };
 161     SpecificBumpPtrAllocator<MergeCandidate> Allocator;
 162     SmallVector<const MergeCandidate*,4> Candidates;
 163     SmallVector<MachineInstr*,4> MergeBaseCandidates;
 164
 165     void moveLiveRegsBefore(const MachineBasicBlock &MBB,
 166                             MachineBasicBlock::const_iterator Before);
 167     unsigned findFreeReg(const TargetRegisterClass &RegClass);
 168     void UpdateBaseRegUses(MachineBasicBlock &MBB,
 169                            MachineBasicBlock::iterator MBBI, const DebugLoc &DL,
 170                            unsigned Base, unsigned WordOffset,
 171                            ARMCC::CondCodes Pred, unsigned PredReg);
 172     MachineInstr *CreateLoadStoreMulti(
 173         MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
 174         int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
 175         ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
 176         ArrayRef<std::pair<unsigned, bool>> Regs);
 177     MachineInstr *CreateLoadStoreDouble(
 178         MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
 179         int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
 180         ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
 181         ArrayRef<std::pair<unsigned, bool>> Regs) const;
 182     void FormCandidates(const MemOpQueue &MemOps);
 183     MachineInstr *MergeOpsUpdate(const MergeCandidate &Cand);
 184     bool FixInvalidRegPairOp(MachineBasicBlock &MBB,
 185                              MachineBasicBlock::iterator &MBBI);
 186     bool MergeBaseUpdateLoadStore(MachineInstr *MI);
 187     bool MergeBaseUpdateLSMultiple(MachineInstr *MI);
 188     bool MergeBaseUpdateLSDouble(MachineInstr &MI) const;
 189     bool LoadStoreMultipleOpti(MachineBasicBlock &MBB);
 190     bool MergeReturnIntoLDM(MachineBasicBlock &MBB);
 191     bool CombineMovBx(MachineBasicBlock &MBB);
 192   };
 193
 194 } // end anonymous namespace
 195
 196 char ARMLoadStoreOpt::ID = 0;
 197
 198 INITIALIZE_PASS(ARMLoadStoreOpt, "arm-ldst-opt", ARM_LOAD_STORE_OPT_NAME, false,
 199                 false)
 200
 201 static bool definesCPSR(const MachineInstr &MI) {
 202   for (const auto &MO : MI.operands()) {
 203     if (!MO.isReg())
 204       continue;
 205     if (MO.isDef() && MO.getReg() == ARM::CPSR && !MO.isDead())
 206       // If the instruction has live CPSR def, then it's not safe to fold it
 207       // into load / store.
 208       return true;
 209   }
 210
 211   return false;
 212 }
 213
 214 static int getMemoryOpOffset(const MachineInstr &MI) {
 215   unsigned Opcode = MI.getOpcode();
 216   bool isAM3 = Opcode == ARM::LDRD || Opcode == ARM::STRD;
 217   unsigned NumOperands = MI.getDesc().getNumOperands();
 218   unsigned OffField = MI.getOperand(NumOperands - 3).getImm();
 219
 220   if (Opcode == ARM::t2LDRi12 || Opcode == ARM::t2LDRi8 ||
 221       Opcode == ARM::t2STRi12 || Opcode == ARM::t2STRi8 ||
 222       Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8 ||
 223       Opcode == ARM::LDRi12   || Opcode == ARM::STRi12)
 224     return OffField;
 225
 226   // Thumb1 immediate offsets are scaled by 4
 227   if (Opcode == ARM::tLDRi || Opcode == ARM::tSTRi ||
 228       Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi)
 229     return OffField * 4;
 230
 231   int Offset = isAM3 ? ARM_AM::getAM3Offset(OffField)
 232     : ARM_AM::getAM5Offset(OffField) * 4;
 233   ARM_AM::AddrOpc Op = isAM3 ? ARM_AM::getAM3Op(OffField)
 234     : ARM_AM::getAM5Op(OffField);
 235
 236   if (Op == ARM_AM::sub)
 237     return -Offset;
 238
 239   return Offset;
 240 }
 241
 242 static const MachineOperand &getLoadStoreBaseOp(const MachineInstr &MI) {
 243   return MI.getOperand(1);
 244 }
 245
 246 static const MachineOperand &getLoadStoreRegOp(const MachineInstr &MI) {
 247   return MI.getOperand(0);
 248 }
 249
 250 static int getLoadStoreMultipleOpcode(unsigned Opcode, ARM_AM::AMSubMode Mode) {
 251   switch (Opcode) {
 252   default: llvm_unreachable("Unhandled opcode!");
 253   case ARM::LDRi12:
 254     ++NumLDMGened;
 255     switch (Mode) {
 256     default: llvm_unreachable("Unhandled submode!");
 257     case ARM_AM::ia: return ARM::LDMIA;
 258     case ARM_AM::da: return ARM::LDMDA;
 259     case ARM_AM::db: return ARM::LDMDB;
 260     case ARM_AM::ib: return ARM::LDMIB;
 261     }
 262   case ARM::STRi12:
 263     ++NumSTMGened;
 264     switch (Mode) {
 265     default: llvm_unreachable("Unhandled submode!");
 266     case ARM_AM::ia: return ARM::STMIA;
 267     case ARM_AM::da: return ARM::STMDA;
 268     case ARM_AM::db: return ARM::STMDB;
 269     case ARM_AM::ib: return ARM::STMIB;
 270     }
 271   case ARM::tLDRi:
 272   case ARM::tLDRspi:
 273     // tLDMIA is writeback-only - unless the base register is in the input
 274     // reglist.
 275     ++NumLDMGened;
 276     switch (Mode) {
 277     default: llvm_unreachable("Unhandled submode!");
 278     case ARM_AM::ia: return ARM::tLDMIA;
 279     }
 280   case ARM::tSTRi:
 281   case ARM::tSTRspi:
 282     // There is no non-writeback tSTMIA either.
 283     ++NumSTMGened;
 284     switch (Mode) {
 285     default: llvm_unreachable("Unhandled submode!");
 286     case ARM_AM::ia: return ARM::tSTMIA_UPD;
 287     }
 288   case ARM::t2LDRi8:
 289   case ARM::t2LDRi12:
 290     ++NumLDMGened;
 291     switch (Mode) {
 292     default: llvm_unreachable("Unhandled submode!");
 293     case ARM_AM::ia: return ARM::t2LDMIA;
 294     case ARM_AM::db: return ARM::t2LDMDB;
 295     }
 296   case ARM::t2STRi8:
 297   case ARM::t2STRi12:
 298     ++NumSTMGened;
 299     switch (Mode) {
 300     default: llvm_unreachable("Unhandled submode!");
 301     case ARM_AM::ia: return ARM::t2STMIA;
 302     case ARM_AM::db: return ARM::t2STMDB;
 303     }
 304   case ARM::VLDRS:
 305     ++NumVLDMGened;
 306     switch (Mode) {
 307     default: llvm_unreachable("Unhandled submode!");
 308     case ARM_AM::ia: return ARM::VLDMSIA;
 309     case ARM_AM::db: return 0; // Only VLDMSDB_UPD exists.
 310     }
 311   case ARM::VSTRS:
 312     ++NumVSTMGened;
 313     switch (Mode) {
 314     default: llvm_unreachable("Unhandled submode!");
 315     case ARM_AM::ia: return ARM::VSTMSIA;
 316     case ARM_AM::db: return 0; // Only VSTMSDB_UPD exists.
 317     }
 318   case ARM::VLDRD:
 319     ++NumVLDMGened;
 320     switch (Mode) {
 321     default: llvm_unreachable("Unhandled submode!");
 322     case ARM_AM::ia: return ARM::VLDMDIA;
 323     case ARM_AM::db: return 0; // Only VLDMDDB_UPD exists.
 324     }
 325   case ARM::VSTRD:
 326     ++NumVSTMGened;
 327     switch (Mode) {
 328     default: llvm_unreachable("Unhandled submode!");
 329     case ARM_AM::ia: return ARM::VSTMDIA;
 330     case ARM_AM::db: return 0; // Only VSTMDDB_UPD exists.
 331     }
 332   }
 333 }
 334
 335 static ARM_AM::AMSubMode getLoadStoreMultipleSubMode(unsigned Opcode) {
 336   switch (Opcode) {
 337   default: llvm_unreachable("Unhandled opcode!");
 338   case ARM::LDMIA_RET:
 339   case ARM::LDMIA:
 340   case ARM::LDMIA_UPD:
 341   case ARM::STMIA:
 342   case ARM::STMIA_UPD:
 343   case ARM::tLDMIA:
 344   case ARM::tLDMIA_UPD:
 345   case ARM::tSTMIA_UPD:
 346   case ARM::t2LDMIA_RET:
 347   case ARM::t2LDMIA:
 348   case ARM::t2LDMIA_UPD:
 349   case ARM::t2STMIA:
 350   case ARM::t2STMIA_UPD:
 351   case ARM::VLDMSIA:
 352   case ARM::VLDMSIA_UPD:
 353   case ARM::VSTMSIA:
 354   case ARM::VSTMSIA_UPD:
 355   case ARM::VLDMDIA:
 356   case ARM::VLDMDIA_UPD:
 357   case ARM::VSTMDIA:
 358   case ARM::VSTMDIA_UPD:
 359     return ARM_AM::ia;
 360
 361   case ARM::LDMDA:
 362   case ARM::LDMDA_UPD:
 363   case ARM::STMDA:
 364   case ARM::STMDA_UPD:
 365     return ARM_AM::da;
 366
 367   case ARM::LDMDB:
 368   case ARM::LDMDB_UPD:
 369   case ARM::STMDB:
 370   case ARM::STMDB_UPD:
 371   case ARM::t2LDMDB:
 372   case ARM::t2LDMDB_UPD:
 373   case ARM::t2STMDB:
 374   case ARM::t2STMDB_UPD:
 375   case ARM::VLDMSDB_UPD:
 376   case ARM::VSTMSDB_UPD:
 377   case ARM::VLDMDDB_UPD:
 378   case ARM::VSTMDDB_UPD:
 379     return ARM_AM::db;
 380
 381   case ARM::LDMIB:
 382   case ARM::LDMIB_UPD:
 383   case ARM::STMIB:
 384   case ARM::STMIB_UPD:
 385     return ARM_AM::ib;
 386   }
 387 }
 388
 389 static bool isT1i32Load(unsigned Opc) {
 390   return Opc == ARM::tLDRi || Opc == ARM::tLDRspi;
 391 }
 392
 393 static bool isT2i32Load(unsigned Opc) {
 394   return Opc == ARM::t2LDRi12 || Opc == ARM::t2LDRi8;
 395 }
 396
 397 static bool isi32Load(unsigned Opc) {
 398   return Opc == ARM::LDRi12 || isT1i32Load(Opc) || isT2i32Load(Opc) ;
 399 }
 400
 401 static bool isT1i32Store(unsigned Opc) {
 402   return Opc == ARM::tSTRi || Opc == ARM::tSTRspi;
 403 }
 404
 405 static bool isT2i32Store(unsigned Opc) {
 406   return Opc == ARM::t2STRi12 || Opc == ARM::t2STRi8;
 407 }
 408
 409 static bool isi32Store(unsigned Opc) {
 410   return Opc == ARM::STRi12 || isT1i32Store(Opc) || isT2i32Store(Opc);
 411 }
 412
 413 static bool isLoadSingle(unsigned Opc) {
 414   return isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD;
 415 }
 416
 417 static unsigned getImmScale(unsigned Opc) {
 418   switch (Opc) {
 419   default: llvm_unreachable("Unhandled opcode!");
 420   case ARM::tLDRi:
 421   case ARM::tSTRi:
 422   case ARM::tLDRspi:
 423   case ARM::tSTRspi:
 424     return 1;
 425   case ARM::tLDRHi:
 426   case ARM::tSTRHi:
 427     return 2;
 428   case ARM::tLDRBi:
 429   case ARM::tSTRBi:
 430     return 4;
 431   }
 432 }
 433
 434 static unsigned getLSMultipleTransferSize(const MachineInstr *MI) {
 435   switch (MI->getOpcode()) {
 436   default: return 0;
 437   case ARM::LDRi12:
 438   case ARM::STRi12:
 439   case ARM::tLDRi:
 440   case ARM::tSTRi:
 441   case ARM::tLDRspi:
 442   case ARM::tSTRspi:
 443   case ARM::t2LDRi8:
 444   case ARM::t2LDRi12:
 445   case ARM::t2STRi8:
 446   case ARM::t2STRi12:
 447   case ARM::VLDRS:
 448   case ARM::VSTRS:
 449     return 4;
 450   case ARM::VLDRD:
 451   case ARM::VSTRD:
 452     return 8;
 453   case ARM::LDMIA:
 454   case ARM::LDMDA:
 455   case ARM::LDMDB:
 456   case ARM::LDMIB:
 457   case ARM::STMIA:
 458   case ARM::STMDA:
 459   case ARM::STMDB:
 460   case ARM::STMIB:
 461   case ARM::tLDMIA:
 462   case ARM::tLDMIA_UPD:
 463   case ARM::tSTMIA_UPD:
 464   case ARM::t2LDMIA:
 465   case ARM::t2LDMDB:
 466   case ARM::t2STMIA:
 467   case ARM::t2STMDB:
 468   case ARM::VLDMSIA:
 469   case ARM::VSTMSIA:
 470     return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 4;
 471   case ARM::VLDMDIA:
 472   case ARM::VSTMDIA:
 473     return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 8;
 474   }
 475 }
 476
 477 /// Update future uses of the base register with the offset introduced
 478 /// due to writeback. This function only works on Thumb1.
 479 void ARMLoadStoreOpt::UpdateBaseRegUses(MachineBasicBlock &MBB,
 480                                         MachineBasicBlock::iterator MBBI,
 481                                         const DebugLoc &DL, unsigned Base,
 482                                         unsigned WordOffset,
 483                                         ARMCC::CondCodes Pred,
 484                                         unsigned PredReg) {
 485   assert(isThumb1 && "Can only update base register uses for Thumb1!");
 486   // Start updating any instructions with immediate offsets. Insert a SUB before
 487   // the first non-updateable instruction (if any).
 488   for (; MBBI != MBB.end(); ++MBBI) {
 489     bool InsertSub = false;
 490     unsigned Opc = MBBI->getOpcode();
 491
 492     if (MBBI->readsRegister(Base)) {
 493       int Offset;
 494       bool IsLoad =
 495         Opc == ARM::tLDRi || Opc == ARM::tLDRHi || Opc == ARM::tLDRBi;
 496       bool IsStore =
 497         Opc == ARM::tSTRi || Opc == ARM::tSTRHi || Opc == ARM::tSTRBi;
 498
 499       if (IsLoad || IsStore) {
 500         // Loads and stores with immediate offsets can be updated, but only if
 501         // the new offset isn't negative.
 502         // The MachineOperand containing the offset immediate is the last one
 503         // before predicates.
 504         MachineOperand &MO =
 505           MBBI->getOperand(MBBI->getDesc().getNumOperands() - 3);
 506         // The offsets are scaled by 1, 2 or 4 depending on the Opcode.
 507         Offset = MO.getImm() - WordOffset * getImmScale(Opc);
 508
 509         // If storing the base register, it needs to be reset first.
 510         unsigned InstrSrcReg = getLoadStoreRegOp(*MBBI).getReg();
 511
 512         if (Offset >= 0 && !(IsStore && InstrSrcReg == Base))
 513           MO.setImm(Offset);
 514         else
 515           InsertSub = true;
 516       } else if ((Opc == ARM::tSUBi8 || Opc == ARM::tADDi8) &&
 517                  !definesCPSR(*MBBI)) {
 518         // SUBS/ADDS using this register, with a dead def of the CPSR.
 519         // Merge it with the update; if the merged offset is too large,
 520         // insert a new sub instead.
 521         MachineOperand &MO =
 522           MBBI->getOperand(MBBI->getDesc().getNumOperands() - 3);
 523         Offset = (Opc == ARM::tSUBi8) ?
 524           MO.getImm() + WordOffset * 4 :
 525           MO.getImm() - WordOffset * 4 ;
 526         if (Offset >= 0 && TL->isLegalAddImmediate(Offset)) {
 527           // FIXME: Swap ADDS<->SUBS if Offset < 0, erase instruction if
 528           // Offset == 0.
 529           MO.setImm(Offset);
 530           // The base register has now been reset, so exit early.
 531           return;
 532         } else {
 533           InsertSub = true;
 534         }
 535       } else {
 536         // Can't update the instruction.
 537         InsertSub = true;
 538       }
 539     } else if (definesCPSR(*MBBI) || MBBI->isCall() || MBBI->isBranch()) {
 540       // Since SUBS sets the condition flags, we can't place the base reset
 541       // after an instruction that has a live CPSR def.
 542       // The base register might also contain an argument for a function call.
 543       InsertSub = true;
 544     }
 545
 546     if (InsertSub) {
 547       // An instruction above couldn't be updated, so insert a sub.
 548       BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBi8), Base)
 549           .add(t1CondCodeOp(true))
 550           .addReg(Base)
 551           .addImm(WordOffset * 4)
 552           .addImm(Pred)
 553           .addReg(PredReg);
 554       return;
 555     }
 556
 557     if (MBBI->killsRegister(Base) || MBBI->definesRegister(Base))
 558       // Register got killed. Stop updating.
 559       return;
 560   }
 561
 562   // End of block was reached.
 563   if (MBB.succ_size() > 0) {
 564     // FIXME: Because of a bug, live registers are sometimes missing from
 565     // the successor blocks' live-in sets. This means we can't trust that
 566     // information and *always* have to reset at the end of a block.
 567     // See PR21029.
 568     if (MBBI != MBB.end()) --MBBI;
 569     BuildMI(MBB, MBBI, DL, TII->get(ARM::tSUBi8), Base)
 570         .add(t1CondCodeOp(true))
 571         .addReg(Base)
 572         .addImm(WordOffset * 4)
 573         .addImm(Pred)
 574         .addReg(PredReg);
 575   }
 576 }
 577
 578 /// Return the first register of class \p RegClass that is not in \p Regs.
 579 unsigned ARMLoadStoreOpt::findFreeReg(const TargetRegisterClass &RegClass) {
 580   if (!RegClassInfoValid) {
 581     RegClassInfo.runOnMachineFunction(*MF);
 582     RegClassInfoValid = true;
 583   }
 584
 585   for (unsigned Reg : RegClassInfo.getOrder(&RegClass))
 586     if (!LiveRegs.contains(Reg))
 587       return Reg;
 588   return 0;
 589 }
 590
 591 /// Compute live registers just before instruction \p Before (in normal schedule
 592 /// direction). Computes backwards so multiple queries in the same block must
 593 /// come in reverse order.
 594 void ARMLoadStoreOpt::moveLiveRegsBefore(const MachineBasicBlock &MBB,
 595     MachineBasicBlock::const_iterator Before) {
 596   // Initialize if we never queried in this block.
 597   if (!LiveRegsValid) {
 598     LiveRegs.init(*TRI);
 599     LiveRegs.addLiveOuts(MBB);
 600     LiveRegPos = MBB.end();
 601     LiveRegsValid = true;
 602   }
 603   // Move backward just before the "Before" position.
 604   while (LiveRegPos != Before) {
 605     --LiveRegPos;
 606     LiveRegs.stepBackward(*LiveRegPos);
 607   }
 608 }
 609
 610 static bool ContainsReg(const ArrayRef<std::pair<unsigned, bool>> &Regs,
 611                         unsigned Reg) {
 612   for (const std::pair<unsigned, bool> &R : Regs)
 613     if (R.first == Reg)
 614       return true;
 615   return false;
 616 }
 617
 618 /// Create and insert a LDM or STM with Base as base register and registers in
 619 /// Regs as the register operands that would be loaded / stored.  It returns
 620 /// true if the transformation is done.
 621 MachineInstr *ARMLoadStoreOpt::CreateLoadStoreMulti(
 622     MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
 623     int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
 624     ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
 625     ArrayRef<std::pair<unsigned, bool>> Regs) {
 626   unsigned NumRegs = Regs.size();
 627   assert(NumRegs > 1);
 628
 629   // For Thumb1 targets, it might be necessary to clobber the CPSR to merge.
 630   // Compute liveness information for that register to make the decision.
 631   bool SafeToClobberCPSR = !isThumb1 ||
 632     (MBB.computeRegisterLiveness(TRI, ARM::CPSR, InsertBefore, 20) ==
 633      MachineBasicBlock::LQR_Dead);
 634
 635   bool Writeback = isThumb1; // Thumb1 LDM/STM have base reg writeback.
 636
 637   // Exception: If the base register is in the input reglist, Thumb1 LDM is
 638   // non-writeback.
 639   // It's also not possible to merge an STR of the base register in Thumb1.
 640   if (isThumb1 && ContainsReg(Regs, Base)) {
 641     assert(Base != ARM::SP && "Thumb1 does not allow SP in register list");
 642     if (Opcode == ARM::tLDRi)
 643       Writeback = false;
 644     else if (Opcode == ARM::tSTRi)
 645       return nullptr;
 646   }
 647
 648   ARM_AM::AMSubMode Mode = ARM_AM::ia;
 649   // VFP and Thumb2 do not support IB or DA modes. Thumb1 only supports IA.
 650   bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
 651   bool haveIBAndDA = isNotVFP && !isThumb2 && !isThumb1;
 652
 653   if (Offset == 4 && haveIBAndDA) {
 654     Mode = ARM_AM::ib;
 655   } else if (Offset == -4 * (int)NumRegs + 4 && haveIBAndDA) {
 656     Mode = ARM_AM::da;
 657   } else if (Offset == -4 * (int)NumRegs && isNotVFP && !isThumb1) {
 658     // VLDM/VSTM do not support DB mode without also updating the base reg.
 659     Mode = ARM_AM::db;
 660   } else if (Offset != 0 || Opcode == ARM::tLDRspi || Opcode == ARM::tSTRspi) {
 661     // Check if this is a supported opcode before inserting instructions to
 662     // calculate a new base register.
 663     if (!getLoadStoreMultipleOpcode(Opcode, Mode)) return nullptr;
 664
 665     // If starting offset isn't zero, insert a MI to materialize a new base.
 666     // But only do so if it is cost effective, i.e. merging more than two
 667     // loads / stores.
 668     if (NumRegs <= 2)
 669       return nullptr;
 670
 671     // On Thumb1, it's not worth materializing a new base register without
 672     // clobbering the CPSR (i.e. not using ADDS/SUBS).
 673     if (!SafeToClobberCPSR)
 674       return nullptr;
 675
 676     unsigned NewBase;
 677     if (isi32Load(Opcode)) {
 678       // If it is a load, then just use one of the destination registers
 679       // as the new base. Will no longer be writeback in Thumb1.
 680       NewBase = Regs[NumRegs-1].first;
 681       Writeback = false;
 682     } else {
 683       // Find a free register that we can use as scratch register.
 684       moveLiveRegsBefore(MBB, InsertBefore);
 685       // The merged instruction does not exist yet but will use several Regs if
 686       // it is a Store.
 687       if (!isLoadSingle(Opcode))
 688         for (const std::pair<unsigned, bool> &R : Regs)
 689           LiveRegs.addReg(R.first);
 690
 691       NewBase = findFreeReg(isThumb1 ? ARM::tGPRRegClass : ARM::GPRRegClass);
 692       if (NewBase == 0)
 693         return nullptr;
 694     }
 695
 696     int BaseOpc =
 697       isThumb2 ? ARM::t2ADDri :
 698       (isThumb1 && Base == ARM::SP) ? ARM::tADDrSPi :
 699       (isThumb1 && Offset < 8) ? ARM::tADDi3 :
 700       isThumb1 ? ARM::tADDi8  : ARM::ADDri;
 701
 702     if (Offset < 0) {
 703       Offset = - Offset;
 704       BaseOpc =
 705         isThumb2 ? ARM::t2SUBri :
 706         (isThumb1 && Offset < 8 && Base != ARM::SP) ? ARM::tSUBi3 :
 707         isThumb1 ? ARM::tSUBi8  : ARM::SUBri;
 708     }
 709
 710     if (!TL->isLegalAddImmediate(Offset))
 711       // FIXME: Try add with register operand?
 712       return nullptr; // Probably not worth it then.
 713
 714     // We can only append a kill flag to the add/sub input if the value is not
 715     // used in the register list of the stm as well.
 716     bool KillOldBase = BaseKill &&
 717       (!isi32Store(Opcode) || !ContainsReg(Regs, Base));
 718
 719     if (isThumb1) {
 720       // Thumb1: depending on immediate size, use either
 721       //   ADDS NewBase, Base, #imm3
 722       // or
 723       //   MOV  NewBase, Base
 724       //   ADDS NewBase, #imm8.
 725       if (Base != NewBase &&
 726           (BaseOpc == ARM::tADDi8 || BaseOpc == ARM::tSUBi8)) {
 727         // Need to insert a MOV to the new base first.
 728         if (isARMLowRegister(NewBase) && isARMLowRegister(Base) &&
 729             !STI->hasV6Ops()) {
 730           // thumbv4t doesn't have lo->lo copies, and we can't predicate tMOVSr
 731           if (Pred != ARMCC::AL)
 732             return nullptr;
 733           BuildMI(MBB, InsertBefore, DL, TII->get(ARM::tMOVSr), NewBase)
 734             .addReg(Base, getKillRegState(KillOldBase));
 735         } else
 736           BuildMI(MBB, InsertBefore, DL, TII->get(ARM::tMOVr), NewBase)
 737               .addReg(Base, getKillRegState(KillOldBase))
 738               .add(predOps(Pred, PredReg));
 739
 740         // The following ADDS/SUBS becomes an update.
 741         Base = NewBase;
 742         KillOldBase = true;
 743       }
 744       if (BaseOpc == ARM::tADDrSPi) {
 745         assert(Offset % 4 == 0 && "tADDrSPi offset is scaled by 4");
 746         BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
 747             .addReg(Base, getKillRegState(KillOldBase))
 748             .addImm(Offset / 4)
 749             .add(predOps(Pred, PredReg));
 750       } else
 751         BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
 752             .add(t1CondCodeOp(true))
 753             .addReg(Base, getKillRegState(KillOldBase))
 754             .addImm(Offset)
 755             .add(predOps(Pred, PredReg));
 756     } else {
 757       BuildMI(MBB, InsertBefore, DL, TII->get(BaseOpc), NewBase)
 758           .addReg(Base, getKillRegState(KillOldBase))
 759           .addImm(Offset)
 760           .add(predOps(Pred, PredReg))
 761           .add(condCodeOp());
 762     }
 763     Base = NewBase;
 764     BaseKill = true; // New base is always killed straight away.
 765   }
 766
 767   bool isDef = isLoadSingle(Opcode);
 768
 769   // Get LS multiple opcode. Note that for Thumb1 this might be an opcode with
 770   // base register writeback.
 771   Opcode = getLoadStoreMultipleOpcode(Opcode, Mode);
 772   if (!Opcode)
 773     return nullptr;
 774
 775   // Check if a Thumb1 LDM/STM merge is safe. This is the case if:
 776   // - There is no writeback (LDM of base register),
 777   // - the base register is killed by the merged instruction,
 778   // - or it's safe to overwrite the condition flags, i.e. to insert a SUBS
 779   //   to reset the base register.
 780   // Otherwise, don't merge.
 781   // It's safe to return here since the code to materialize a new base register
 782   // above is also conditional on SafeToClobberCPSR.
 783   if (isThumb1 && !SafeToClobberCPSR && Writeback && !BaseKill)
 784     return nullptr;
 785
 786   MachineInstrBuilder MIB;
 787
 788   if (Writeback) {
 789     assert(isThumb1 && "expected Writeback only inThumb1");
 790     if (Opcode == ARM::tLDMIA) {
 791       assert(!(ContainsReg(Regs, Base)) && "Thumb1 can't LDM ! with Base in Regs");
 792       // Update tLDMIA with writeback if necessary.
 793       Opcode = ARM::tLDMIA_UPD;
 794     }
 795
 796     MIB = BuildMI(MBB, InsertBefore, DL, TII->get(Opcode));
 797
 798     // Thumb1: we might need to set base writeback when building the MI.
 799     MIB.addReg(Base, getDefRegState(true))
 800        .addReg(Base, getKillRegState(BaseKill));
 801
 802     // The base isn't dead after a merged instruction with writeback.
 803     // Insert a sub instruction after the newly formed instruction to reset.
 804     if (!BaseKill)
 805       UpdateBaseRegUses(MBB, InsertBefore, DL, Base, NumRegs, Pred, PredReg);
 806   } else {
 807     // No writeback, simply build the MachineInstr.
 808     MIB = BuildMI(MBB, InsertBefore, DL, TII->get(Opcode));
 809     MIB.addReg(Base, getKillRegState(BaseKill));
 810   }
 811
 812   MIB.addImm(Pred).addReg(PredReg);
 813
 814   for (const std::pair<unsigned, bool> &R : Regs)
 815     MIB.addReg(R.first, getDefRegState(isDef) | getKillRegState(R.second));
 816
 817   return MIB.getInstr();
 818 }
 819
 820 MachineInstr *ARMLoadStoreOpt::CreateLoadStoreDouble(
 821     MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
 822     int Offset, unsigned Base, bool BaseKill, unsigned Opcode,
 823     ARMCC::CondCodes Pred, unsigned PredReg, const DebugLoc &DL,
 824     ArrayRef<std::pair<unsigned, bool>> Regs) const {
 825   bool IsLoad = isi32Load(Opcode);
 826   assert((IsLoad || isi32Store(Opcode)) && "Must have integer load or store");
 827   unsigned LoadStoreOpcode = IsLoad ? ARM::t2LDRDi8 : ARM::t2STRDi8;
 828
 829   assert(Regs.size() == 2);
 830   MachineInstrBuilder MIB = BuildMI(MBB, InsertBefore, DL,
 831                                     TII->get(LoadStoreOpcode));
 832   if (IsLoad) {
 833     MIB.addReg(Regs[0].first, RegState::Define)
 834        .addReg(Regs[1].first, RegState::Define);
 835   } else {
 836     MIB.addReg(Regs[0].first, getKillRegState(Regs[0].second))
 837        .addReg(Regs[1].first, getKillRegState(Regs[1].second));
 838   }
 839   MIB.addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
 840   return MIB.getInstr();
 841 }
 842
 843 /// Call MergeOps and update MemOps and merges accordingly on success.
 844 MachineInstr *ARMLoadStoreOpt::MergeOpsUpdate(const MergeCandidate &Cand) {
 845   const MachineInstr *First = Cand.Instrs.front();
 846   unsigned Opcode = First->getOpcode();
 847   bool IsLoad = isLoadSingle(Opcode);
 848   SmallVector<std::pair<unsigned, bool>, 8> Regs;
 849   SmallVector<unsigned, 4> ImpDefs;
 850   DenseSet<unsigned> KilledRegs;
 851   DenseSet<unsigned> UsedRegs;
 852   // Determine list of registers and list of implicit super-register defs.
 853   for (const MachineInstr *MI : Cand.Instrs) {
 854     const MachineOperand &MO = getLoadStoreRegOp(*MI);
 855     unsigned Reg = MO.getReg();
 856     bool IsKill = MO.isKill();
 857     if (IsKill)
 858       KilledRegs.insert(Reg);
 859     Regs.push_back(std::make_pair(Reg, IsKill));
 860     UsedRegs.insert(Reg);
 861
 862     if (IsLoad) {
 863       // Collect any implicit defs of super-registers, after merging we can't
 864       // be sure anymore that we properly preserved these live ranges and must
 865       // removed these implicit operands.
 866       for (const MachineOperand &MO : MI->implicit_operands()) {
 867         if (!MO.isReg() || !MO.isDef() || MO.isDead())
 868           continue;
 869         assert(MO.isImplicit());
 870         unsigned DefReg = MO.getReg();
 871
 872         if (is_contained(ImpDefs, DefReg))
 873           continue;
 874         // We can ignore cases where the super-reg is read and written.
 875         if (MI->readsRegister(DefReg))
 876           continue;
 877         ImpDefs.push_back(DefReg);
 878       }
 879     }
 880   }
 881
 882   // Attempt the merge.
 883   using iterator = MachineBasicBlock::iterator;
 884
 885   MachineInstr *LatestMI = Cand.Instrs[Cand.LatestMIIdx];
 886   iterator InsertBefore = std::next(iterator(LatestMI));
 887   MachineBasicBlock &MBB = *LatestMI->getParent();
 888   unsigned Offset = getMemoryOpOffset(*First);
 889   unsigned Base = getLoadStoreBaseOp(*First).getReg();
 890   bool BaseKill = LatestMI->killsRegister(Base);
 891   unsigned PredReg = 0;
 892   ARMCC::CondCodes Pred = getInstrPredicate(*First, PredReg);
 893   DebugLoc DL = First->getDebugLoc();
 894   MachineInstr *Merged = nullptr;
 895   if (Cand.CanMergeToLSDouble)
 896     Merged = CreateLoadStoreDouble(MBB, InsertBefore, Offset, Base, BaseKill,
 897                                    Opcode, Pred, PredReg, DL, Regs);
 898   if (!Merged && Cand.CanMergeToLSMulti)
 899     Merged = CreateLoadStoreMulti(MBB, InsertBefore, Offset, Base, BaseKill,
 900                                   Opcode, Pred, PredReg, DL, Regs);
 901   if (!Merged)
 902     return nullptr;
 903
 904   // Determine earliest instruction that will get removed. We then keep an
 905   // iterator just above it so the following erases don't invalidated it.
 906   iterator EarliestI(Cand.Instrs[Cand.EarliestMIIdx]);
 907   bool EarliestAtBegin = false;
 908   if (EarliestI == MBB.begin()) {
 909     EarliestAtBegin = true;
 910   } else {
 911     EarliestI = std::prev(EarliestI);
 912   }
 913
 914   // Remove instructions which have been merged.
 915   for (MachineInstr *MI : Cand.Instrs)
 916     MBB.erase(MI);
 917
 918   // Determine range between the earliest removed instruction and the new one.
 919   if (EarliestAtBegin)
 920     EarliestI = MBB.begin();
 921   else
 922     EarliestI = std::next(EarliestI);
 923   auto FixupRange = make_range(EarliestI, iterator(Merged));
 924
 925   if (isLoadSingle(Opcode)) {
 926     // If the previous loads defined a super-reg, then we have to mark earlier
 927     // operands undef; Replicate the super-reg def on the merged instruction.
 928     for (MachineInstr &MI : FixupRange) {
 929       for (unsigned &ImpDefReg : ImpDefs) {
 930         for (MachineOperand &MO : MI.implicit_operands()) {
 931           if (!MO.isReg() || MO.getReg() != ImpDefReg)
 932             continue;
 933           if (MO.readsReg())
 934             MO.setIsUndef();
 935           else if (MO.isDef())
 936             ImpDefReg = 0;
 937         }
 938       }
 939     }
 940
 941     MachineInstrBuilder MIB(*Merged->getParent()->getParent(), Merged);
 942     for (unsigned ImpDef : ImpDefs)
 943       MIB.addReg(ImpDef, RegState::ImplicitDefine);
 944   } else {
 945     // Remove kill flags: We are possibly storing the values later now.
 946     assert(isi32Store(Opcode) || Opcode == ARM::VSTRS || Opcode == ARM::VSTRD);
 947     for (MachineInstr &MI : FixupRange) {
 948       for (MachineOperand &MO : MI.uses()) {
 949         if (!MO.isReg() || !MO.isKill())
 950           continue;
 951         if (UsedRegs.count(MO.getReg()))
 952           MO.setIsKill(false);
 953       }
 954     }
 955     assert(ImpDefs.empty());
 956   }
 957
 958   return Merged;
 959 }
 960
 961 static bool isValidLSDoubleOffset(int Offset) {
 962   unsigned Value = abs(Offset);
 963   // t2LDRDi8/t2STRDi8 supports an 8 bit immediate which is internally
 964   // multiplied by 4.
 965   return (Value % 4) == 0 && Value < 1024;
 966 }
 967
 968 /// Return true for loads/stores that can be combined to a double/multi
 969 /// operation without increasing the requirements for alignment.
 970 static bool mayCombineMisaligned(const TargetSubtargetInfo &STI,
 971                                  const MachineInstr &MI) {
 972   // vldr/vstr trap on misaligned pointers anyway, forming vldm makes no
 973   // difference.
 974   unsigned Opcode = MI.getOpcode();
 975   if (!isi32Load(Opcode) && !isi32Store(Opcode))
 976     return true;
 977
 978   // Stack pointer alignment is out of the programmers control so we can trust
 979   // SP-relative loads/stores.
 980   if (getLoadStoreBaseOp(MI).getReg() == ARM::SP &&
 981       STI.getFrameLowering()->getTransientStackAlignment() >= 4)
 982     return true;
 983   return false;
 984 }
 985
 986 /// Find candidates for load/store multiple merge in list of MemOpQueueEntries.
 987 void ARMLoadStoreOpt::FormCandidates(const MemOpQueue &MemOps) {
 988   const MachineInstr *FirstMI = MemOps[0].MI;
 989   unsigned Opcode = FirstMI->getOpcode();
 990   bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
 991   unsigned Size = getLSMultipleTransferSize(FirstMI);
 992
 993   unsigned SIndex = 0;
 994   unsigned EIndex = MemOps.size();
 995   do {
 996     // Look at the first instruction.
 997     const MachineInstr *MI = MemOps[SIndex].MI;
 998     int Offset = MemOps[SIndex].Offset;
 999     const MachineOperand &PMO = getLoadStoreRegOp(*MI);
1000     unsigned PReg = PMO.getReg();
1001     unsigned PRegNum = PMO.isUndef() ? std::numeric_limits<unsigned>::max()
1002                                      : TRI->getEncodingValue(PReg);
1003     unsigned Latest = SIndex;
1004     unsigned Earliest = SIndex;
1005     unsigned Count = 1;
1006     bool CanMergeToLSDouble =
1007       STI->isThumb2() && isNotVFP && isValidLSDoubleOffset(Offset);
1008     // ARM errata 602117: LDRD with base in list may result in incorrect base
1009     // register when interrupted or faulted.
1010     if (STI->isCortexM3() && isi32Load(Opcode) &&
1011         PReg == getLoadStoreBaseOp(*MI).getReg())
1012       CanMergeToLSDouble = false;
1013
1014     bool CanMergeToLSMulti = true;
1015     // On swift vldm/vstm starting with an odd register number as that needs
1016     // more uops than single vldrs.
1017     if (STI->hasSlowOddRegister() && !isNotVFP && (PRegNum % 2) == 1)
1018       CanMergeToLSMulti = false;
1019
1020     // LDRD/STRD do not allow SP/PC. LDM/STM do not support it or have it
1021     // deprecated; LDM to PC is fine but cannot happen here.
1022     if (PReg == ARM::SP || PReg == ARM::PC)
1023       CanMergeToLSMulti = CanMergeToLSDouble = false;
1024
1025     // Should we be conservative?
1026     if (AssumeMisalignedLoadStores && !mayCombineMisaligned(*STI, *MI))
1027       CanMergeToLSMulti = CanMergeToLSDouble = false;
1028
1029     // vldm / vstm limit are 32 for S variants, 16 for D variants.
1030     unsigned Limit;
1031     switch (Opcode) {
1032     default:
1033       Limit = UINT_MAX;
1034       break;
1035     case ARM::VLDRD:
1036     case ARM::VSTRD:
1037       Limit = 16;
1038       break;
1039     }
1040
1041     // Merge following instructions where possible.
1042     for (unsigned I = SIndex+1; I < EIndex; ++I, ++Count) {
1043       int NewOffset = MemOps[I].Offset;
1044       if (NewOffset != Offset + (int)Size)
1045         break;
1046       const MachineOperand &MO = getLoadStoreRegOp(*MemOps[I].MI);
1047       unsigned Reg = MO.getReg();
1048       if (Reg == ARM::SP || Reg == ARM::PC)
1049         break;
1050       if (Count == Limit)
1051         break;
1052
1053       // See if the current load/store may be part of a multi load/store.
1054       unsigned RegNum = MO.isUndef() ? std::numeric_limits<unsigned>::max()
1055                                      : TRI->getEncodingValue(Reg);
1056       bool PartOfLSMulti = CanMergeToLSMulti;
1057       if (PartOfLSMulti) {
1058         // Register numbers must be in ascending order.
1059         if (RegNum <= PRegNum)
1060           PartOfLSMulti = false;
1061         // For VFP / NEON load/store multiples, the registers must be
1062         // consecutive and within the limit on the number of registers per
1063         // instruction.
1064         else if (!isNotVFP && RegNum != PRegNum+1)
1065           PartOfLSMulti = false;
1066       }
1067       // See if the current load/store may be part of a double load/store.
1068       bool PartOfLSDouble = CanMergeToLSDouble && Count <= 1;
1069
1070       if (!PartOfLSMulti && !PartOfLSDouble)
1071         break;
1072       CanMergeToLSMulti &= PartOfLSMulti;
1073       CanMergeToLSDouble &= PartOfLSDouble;
1074       // Track MemOp with latest and earliest position (Positions are
1075       // counted in reverse).
1076       unsigned Position = MemOps[I].Position;
1077       if (Position < MemOps[Latest].Position)
1078         Latest = I;
1079       else if (Position > MemOps[Earliest].Position)
1080         Earliest = I;
1081       // Prepare for next MemOp.
1082       Offset += Size;
1083       PRegNum = RegNum;
1084     }
1085
1086     // Form a candidate from the Ops collected so far.
1087     MergeCandidate *Candidate = new(Allocator.Allocate()) MergeCandidate;
1088     for (unsigned C = SIndex, CE = SIndex + Count; C < CE; ++C)
1089       Candidate->Instrs.push_back(MemOps[C].MI);
1090     Candidate->LatestMIIdx = Latest - SIndex;
1091     Candidate->EarliestMIIdx = Earliest - SIndex;
1092     Candidate->InsertPos = MemOps[Latest].Position;
1093     if (Count == 1)
1094       CanMergeToLSMulti = CanMergeToLSDouble = false;
1095     Candidate->CanMergeToLSMulti = CanMergeToLSMulti;
1096     Candidate->CanMergeToLSDouble = CanMergeToLSDouble;
1097     Candidates.push_back(Candidate);
1098     // Continue after the chain.
1099     SIndex += Count;
1100   } while (SIndex < EIndex);
1101 }
1102
1103 static unsigned getUpdatingLSMultipleOpcode(unsigned Opc,
1104                                             ARM_AM::AMSubMode Mode) {
1105   switch (Opc) {
1106   default: llvm_unreachable("Unhandled opcode!");
1107   case ARM::LDMIA:
1108   case ARM::LDMDA:
1109   case ARM::LDMDB:
1110   case ARM::LDMIB:
1111     switch (Mode) {
1112     default: llvm_unreachable("Unhandled submode!");
1113     case ARM_AM::ia: return ARM::LDMIA_UPD;
1114     case ARM_AM::ib: return ARM::LDMIB_UPD;
1115     case ARM_AM::da: return ARM::LDMDA_UPD;
1116     case ARM_AM::db: return ARM::LDMDB_UPD;
1117     }
1118   case ARM::STMIA:
1119   case ARM::STMDA:
1120   case ARM::STMDB:
1121   case ARM::STMIB:
1122     switch (Mode) {
1123     default: llvm_unreachable("Unhandled submode!");
1124     case ARM_AM::ia: return ARM::STMIA_UPD;
1125     case ARM_AM::ib: return ARM::STMIB_UPD;
1126     case ARM_AM::da: return ARM::STMDA_UPD;
1127     case ARM_AM::db: return ARM::STMDB_UPD;
1128     }
1129   case ARM::t2LDMIA:
1130   case ARM::t2LDMDB:
1131     switch (Mode) {
1132     default: llvm_unreachable("Unhandled submode!");
1133     case ARM_AM::ia: return ARM::t2LDMIA_UPD;
1134     case ARM_AM::db: return ARM::t2LDMDB_UPD;
1135     }
1136   case ARM::t2STMIA:
1137   case ARM::t2STMDB:
1138     switch (Mode) {
1139     default: llvm_unreachable("Unhandled submode!");
1140     case ARM_AM::ia: return ARM::t2STMIA_UPD;
1141     case ARM_AM::db: return ARM::t2STMDB_UPD;
1142     }
1143   case ARM::VLDMSIA:
1144     switch (Mode) {
1145     default: llvm_unreachable("Unhandled submode!");
1146     case ARM_AM::ia: return ARM::VLDMSIA_UPD;
1147     case ARM_AM::db: return ARM::VLDMSDB_UPD;
1148     }
1149   case ARM::VLDMDIA:
1150     switch (Mode) {
1151     default: llvm_unreachable("Unhandled submode!");
1152     case ARM_AM::ia: return ARM::VLDMDIA_UPD;
1153     case ARM_AM::db: return ARM::VLDMDDB_UPD;
1154     }
1155   case ARM::VSTMSIA:
1156     switch (Mode) {
1157     default: llvm_unreachable("Unhandled submode!");
1158     case ARM_AM::ia: return ARM::VSTMSIA_UPD;
1159     case ARM_AM::db: return ARM::VSTMSDB_UPD;
1160     }
1161   case ARM::VSTMDIA:
1162     switch (Mode) {
1163     default: llvm_unreachable("Unhandled submode!");
1164     case ARM_AM::ia: return ARM::VSTMDIA_UPD;
1165     case ARM_AM::db: return ARM::VSTMDDB_UPD;
1166     }
1167   }
1168 }
1169
1170 /// Check if the given instruction increments or decrements a register and
1171 /// return the amount it is incremented/decremented. Returns 0 if the CPSR flags
1172 /// generated by the instruction are possibly read as well.
1173 static int isIncrementOrDecrement(const MachineInstr &MI, unsigned Reg,
1174                                   ARMCC::CondCodes Pred, unsigned PredReg) {
1175   bool CheckCPSRDef;
1176   int Scale;
1177   switch (MI.getOpcode()) {
1178   case ARM::tADDi8:  Scale =  4; CheckCPSRDef = true; break;
1179   case ARM::tSUBi8:  Scale = -4; CheckCPSRDef = true; break;
1180   case ARM::t2SUBri:
1181   case ARM::SUBri:   Scale = -1; CheckCPSRDef = true; break;
1182   case ARM::t2ADDri:
1183   case ARM::ADDri:   Scale =  1; CheckCPSRDef = true; break;
1184   case ARM::tADDspi: Scale =  4; CheckCPSRDef = false; break;
1185   case ARM::tSUBspi: Scale = -4; CheckCPSRDef = false; break;
1186   default: return 0;
1187   }
1188
1189   unsigned MIPredReg;
1190   if (MI.getOperand(0).getReg() != Reg ||
1191       MI.getOperand(1).getReg() != Reg ||
1192       getInstrPredicate(MI, MIPredReg) != Pred ||
1193       MIPredReg != PredReg)
1194     return 0;
1195
1196   if (CheckCPSRDef && definesCPSR(MI))
1197     return 0;
1198   return MI.getOperand(2).getImm() * Scale;
1199 }
1200
1201 /// Searches for an increment or decrement of \p Reg before \p MBBI.
1202 static MachineBasicBlock::iterator
1203 findIncDecBefore(MachineBasicBlock::iterator MBBI, unsigned Reg,
1204                  ARMCC::CondCodes Pred, unsigned PredReg, int &Offset) {
1205   Offset = 0;
1206   MachineBasicBlock &MBB = *MBBI->getParent();
1207   MachineBasicBlock::iterator BeginMBBI = MBB.begin();
1208   MachineBasicBlock::iterator EndMBBI = MBB.end();
1209   if (MBBI == BeginMBBI)
1210     return EndMBBI;
1211
1212   // Skip debug values.
1213   MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
1214   while (PrevMBBI->isDebugInstr() && PrevMBBI != BeginMBBI)
1215     --PrevMBBI;
1216
1217   Offset = isIncrementOrDecrement(*PrevMBBI, Reg, Pred, PredReg);
1218   return Offset == 0 ? EndMBBI : PrevMBBI;
1219 }
1220
1221 /// Searches for a increment or decrement of \p Reg after \p MBBI.
1222 static MachineBasicBlock::iterator
1223 findIncDecAfter(MachineBasicBlock::iterator MBBI, unsigned Reg,
1224                 ARMCC::CondCodes Pred, unsigned PredReg, int &Offset) {
1225   Offset = 0;
1226   MachineBasicBlock &MBB = *MBBI->getParent();
1227   MachineBasicBlock::iterator EndMBBI = MBB.end();
1228   MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
1229   // Skip debug values.
1230   while (NextMBBI != EndMBBI && NextMBBI->isDebugInstr())
1231     ++NextMBBI;
1232   if (NextMBBI == EndMBBI)
1233     return EndMBBI;
1234
1235   Offset = isIncrementOrDecrement(*NextMBBI, Reg, Pred, PredReg);
1236   return Offset == 0 ? EndMBBI : NextMBBI;
1237 }
1238
1239 /// Fold proceeding/trailing inc/dec of base register into the
1240 /// LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
1241 ///
1242 /// stmia rn, <ra, rb, rc>
1243 /// rn := rn + 4 * 3;
1244 /// =>
1245 /// stmia rn!, <ra, rb, rc>
1246 ///
1247 /// rn := rn - 4 * 3;
1248 /// ldmia rn, <ra, rb, rc>
1249 /// =>
1250 /// ldmdb rn!, <ra, rb, rc>
1251 bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineInstr *MI) {
1252   // Thumb1 is already using updating loads/stores.
1253   if (isThumb1) return false;
1254
1255   const MachineOperand &BaseOP = MI->getOperand(0);
1256   unsigned Base = BaseOP.getReg();
1257   bool BaseKill = BaseOP.isKill();
1258   unsigned PredReg = 0;
1259   ARMCC::CondCodes Pred = getInstrPredicate(*MI, PredReg);
1260   unsigned Opcode = MI->getOpcode();
1261   DebugLoc DL = MI->getDebugLoc();
1262
1263   // Can't use an updating ld/st if the base register is also a dest
1264   // register. e.g. ldmdb r0!, {r0, r1, r2}. The behavior is undefined.
1265   for (unsigned i = 2, e = MI->getNumOperands(); i != e; ++i)
1266     if (MI->getOperand(i).getReg() == Base)
1267       return false;
1268
1269   int Bytes = getLSMultipleTransferSize(MI);
1270   MachineBasicBlock &MBB = *MI->getParent();
1271   MachineBasicBlock::iterator MBBI(MI);
1272   int Offset;
1273   MachineBasicBlock::iterator MergeInstr
1274     = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
1275   ARM_AM::AMSubMode Mode = getLoadStoreMultipleSubMode(Opcode);
1276   if (Mode == ARM_AM::ia && Offset == -Bytes) {
1277     Mode = ARM_AM::db;
1278   } else if (Mode == ARM_AM::ib && Offset == -Bytes) {
1279     Mode = ARM_AM::da;
1280   } else {
1281     MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
1282     if (((Mode != ARM_AM::ia && Mode != ARM_AM::ib) || Offset != Bytes) &&
1283         ((Mode != ARM_AM::da && Mode != ARM_AM::db) || Offset != -Bytes)) {
1284
1285       // We couldn't find an inc/dec to merge. But if the base is dead, we
1286       // can still change to a writeback form as that will save us 2 bytes
1287       // of code size. It can create WAW hazards though, so only do it if
1288       // we're minimizing code size.
1289       if (!STI->optForMinSize() || !BaseKill)
1290         return false;
1291
1292       bool HighRegsUsed = false;
1293       for (unsigned i = 2, e = MI->getNumOperands(); i != e; ++i)
1294         if (MI->getOperand(i).getReg() >= ARM::R8) {
1295           HighRegsUsed = true;
1296           break;
1297         }
1298
1299       if (!HighRegsUsed)
1300         MergeInstr = MBB.end();
1301       else
1302         return false;
1303     }
1304   }
1305   if (MergeInstr != MBB.end())
1306     MBB.erase(MergeInstr);
1307
1308   unsigned NewOpc = getUpdatingLSMultipleOpcode(Opcode, Mode);
1309   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
1310     .addReg(Base, getDefRegState(true)) // WB base register
1311     .addReg(Base, getKillRegState(BaseKill))
1312     .addImm(Pred).addReg(PredReg);
1313
1314   // Transfer the rest of operands.
1315   for (unsigned OpNum = 3, e = MI->getNumOperands(); OpNum != e; ++OpNum)
1316     MIB.add(MI->getOperand(OpNum));
1317
1318   // Transfer memoperands.
1319   MIB.setMemRefs(MI->memoperands());
1320
1321   MBB.erase(MBBI);
1322   return true;
1323 }
1324
1325 static unsigned getPreIndexedLoadStoreOpcode(unsigned Opc,
1326                                              ARM_AM::AddrOpc Mode) {
1327   switch (Opc) {
1328   case ARM::LDRi12:
1329     return ARM::LDR_PRE_IMM;
1330   case ARM::STRi12:
1331     return ARM::STR_PRE_IMM;
1332   case ARM::VLDRS:
1333     return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
1334   case ARM::VLDRD:
1335     return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
1336   case ARM::VSTRS:
1337     return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
1338   case ARM::VSTRD:
1339     return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
1340   case ARM::t2LDRi8:
1341   case ARM::t2LDRi12:
1342     return ARM::t2LDR_PRE;
1343   case ARM::t2STRi8:
1344   case ARM::t2STRi12:
1345     return ARM::t2STR_PRE;
1346   default: llvm_unreachable("Unhandled opcode!");
1347   }
1348 }
1349
1350 static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc,
1351                                               ARM_AM::AddrOpc Mode) {
1352   switch (Opc) {
1353   case ARM::LDRi12:
1354     return ARM::LDR_POST_IMM;
1355   case ARM::STRi12:
1356     return ARM::STR_POST_IMM;
1357   case ARM::VLDRS:
1358     return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
1359   case ARM::VLDRD:
1360     return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
1361   case ARM::VSTRS:
1362     return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
1363   case ARM::VSTRD:
1364     return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
1365   case ARM::t2LDRi8:
1366   case ARM::t2LDRi12:
1367     return ARM::t2LDR_POST;
1368   case ARM::t2STRi8:
1369   case ARM::t2STRi12:
1370     return ARM::t2STR_POST;
1371   default: llvm_unreachable("Unhandled opcode!");
1372   }
1373 }
1374
1375 /// Fold proceeding/trailing inc/dec of base register into the
1376 /// LDR/STR/FLD{D|S}/FST{D|S} op when possible:
1377 bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineInstr *MI) {
1378   // Thumb1 doesn't have updating LDR/STR.
1379   // FIXME: Use LDM/STM with single register instead.
1380   if (isThumb1) return false;
1381
1382   unsigned Base = getLoadStoreBaseOp(*MI).getReg();
1383   bool BaseKill = getLoadStoreBaseOp(*MI).isKill();
1384   unsigned Opcode = MI->getOpcode();
1385   DebugLoc DL = MI->getDebugLoc();
1386   bool isAM5 = (Opcode == ARM::VLDRD || Opcode == ARM::VLDRS ||
1387                 Opcode == ARM::VSTRD || Opcode == ARM::VSTRS);
1388   bool isAM2 = (Opcode == ARM::LDRi12 || Opcode == ARM::STRi12);
1389   if (isi32Load(Opcode) || isi32Store(Opcode))
1390     if (MI->getOperand(2).getImm() != 0)
1391       return false;
1392   if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0)
1393     return false;
1394
1395   // Can't do the merge if the destination register is the same as the would-be
1396   // writeback register.
1397   if (MI->getOperand(0).getReg() == Base)
1398     return false;
1399
1400   unsigned PredReg = 0;
1401   ARMCC::CondCodes Pred = getInstrPredicate(*MI, PredReg);
1402   int Bytes = getLSMultipleTransferSize(MI);
1403   MachineBasicBlock &MBB = *MI->getParent();
1404   MachineBasicBlock::iterator MBBI(MI);
1405   int Offset;
1406   MachineBasicBlock::iterator MergeInstr
1407     = findIncDecBefore(MBBI, Base, Pred, PredReg, Offset);
1408   unsigned NewOpc;
1409   if (!isAM5 && Offset == Bytes) {
1410     NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
1411   } else if (Offset == -Bytes) {
1412     NewOpc = getPreIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
1413   } else {
1414     MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
1415     if (Offset == Bytes) {
1416       NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::add);
1417     } else if (!isAM5 && Offset == -Bytes) {
1418       NewOpc = getPostIndexedLoadStoreOpcode(Opcode, ARM_AM::sub);
1419     } else
1420       return false;
1421   }
1422   MBB.erase(MergeInstr);
1423
1424   ARM_AM::AddrOpc AddSub = Offset < 0 ? ARM_AM::sub : ARM_AM::add;
1425
1426   bool isLd = isLoadSingle(Opcode);
1427   if (isAM5) {
1428     // VLDM[SD]_UPD, VSTM[SD]_UPD
1429     // (There are no base-updating versions of VLDR/VSTR instructions, but the
1430     // updating load/store-multiple instructions can be used with only one
1431     // register.)
1432     MachineOperand &MO = MI->getOperand(0);
1433     BuildMI(MBB, MBBI, DL, TII->get(NewOpc))
1434       .addReg(Base, getDefRegState(true)) // WB base register
1435       .addReg(Base, getKillRegState(isLd ? BaseKill : false))
1436       .addImm(Pred).addReg(PredReg)
1437       .addReg(MO.getReg(), (isLd ? getDefRegState(true) :
1438                             getKillRegState(MO.isKill())));
1439   } else if (isLd) {
1440     if (isAM2) {
1441       // LDR_PRE, LDR_POST
1442       if (NewOpc == ARM::LDR_PRE_IMM || NewOpc == ARM::LDRB_PRE_IMM) {
1443         BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
1444           .addReg(Base, RegState::Define)
1445           .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
1446       } else {
1447         int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
1448         BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
1449             .addReg(Base, RegState::Define)
1450             .addReg(Base)
1451             .addReg(0)
1452             .addImm(Imm)
1453             .add(predOps(Pred, PredReg));
1454       }
1455     } else {
1456       // t2LDR_PRE, t2LDR_POST
1457       BuildMI(MBB, MBBI, DL, TII->get(NewOpc), MI->getOperand(0).getReg())
1458           .addReg(Base, RegState::Define)
1459           .addReg(Base)
1460           .addImm(Offset)
1461           .add(predOps(Pred, PredReg));
1462     }
1463   } else {
1464     MachineOperand &MO = MI->getOperand(0);
1465     // FIXME: post-indexed stores use am2offset_imm, which still encodes
1466     // the vestigal zero-reg offset register. When that's fixed, this clause
1467     // can be removed entirely.
1468     if (isAM2 && NewOpc == ARM::STR_POST_IMM) {
1469       int Imm = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
1470       // STR_PRE, STR_POST
1471       BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
1472           .addReg(MO.getReg(), getKillRegState(MO.isKill()))
1473           .addReg(Base)
1474           .addReg(0)
1475           .addImm(Imm)
1476           .add(predOps(Pred, PredReg));
1477     } else {
1478       // t2STR_PRE, t2STR_POST
1479       BuildMI(MBB, MBBI, DL, TII->get(NewOpc), Base)
1480           .addReg(MO.getReg(), getKillRegState(MO.isKill()))
1481           .addReg(Base)
1482           .addImm(Offset)
1483           .add(predOps(Pred, PredReg));
1484     }
1485   }
1486   MBB.erase(MBBI);
1487
1488   return true;
1489 }
1490
1491 bool ARMLoadStoreOpt::MergeBaseUpdateLSDouble(MachineInstr &MI) const {
1492   unsigned Opcode = MI.getOpcode();
1493   assert((Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) &&
1494          "Must have t2STRDi8 or t2LDRDi8");
1495   if (MI.getOperand(3).getImm() != 0)
1496     return false;
1497
1498   // Behaviour for writeback is undefined if base register is the same as one
1499   // of the others.
1500   const MachineOperand &BaseOp = MI.getOperand(2);
1501   unsigned Base = BaseOp.getReg();
1502   const MachineOperand &Reg0Op = MI.getOperand(0);
1503   const MachineOperand &Reg1Op = MI.getOperand(1);
1504   if (Reg0Op.getReg() == Base || Reg1Op.getReg() == Base)
1505     return false;
1506
1507   unsigned PredReg;
1508   ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
1509   MachineBasicBlock::iterator MBBI(MI);
1510   MachineBasicBlock &MBB = *MI.getParent();
1511   int Offset;
1512   MachineBasicBlock::iterator MergeInstr = findIncDecBefore(MBBI, Base, Pred,
1513                                                             PredReg, Offset);
1514   unsigned NewOpc;
1515   if (Offset == 8 || Offset == -8) {
1516     NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_PRE : ARM::t2STRD_PRE;
1517   } else {
1518     MergeInstr = findIncDecAfter(MBBI, Base, Pred, PredReg, Offset);
1519     if (Offset == 8 || Offset == -8) {
1520       NewOpc = Opcode == ARM::t2LDRDi8 ? ARM::t2LDRD_POST : ARM::t2STRD_POST;
1521     } else
1522       return false;
1523   }
1524   MBB.erase(MergeInstr);
1525
1526   DebugLoc DL = MI.getDebugLoc();
1527   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, DL, TII->get(NewOpc));
1528   if (NewOpc == ARM::t2LDRD_PRE || NewOpc == ARM::t2LDRD_POST) {
1529     MIB.add(Reg0Op).add(Reg1Op).addReg(BaseOp.getReg(), RegState::Define);
1530   } else {
1531     assert(NewOpc == ARM::t2STRD_PRE || NewOpc == ARM::t2STRD_POST);
1532     MIB.addReg(BaseOp.getReg(), RegState::Define).add(Reg0Op).add(Reg1Op);
1533   }
1534   MIB.addReg(BaseOp.getReg(), RegState::Kill)
1535      .addImm(Offset).addImm(Pred).addReg(PredReg);
1536   assert(TII->get(Opcode).getNumOperands() == 6 &&
1537          TII->get(NewOpc).getNumOperands() == 7 &&
1538          "Unexpected number of operands in Opcode specification.");
1539
1540   // Transfer implicit operands.
1541   for (const MachineOperand &MO : MI.implicit_operands())
1542     MIB.add(MO);
1543   MIB.setMemRefs(MI.memoperands());
1544
1545   MBB.erase(MBBI);
1546   return true;
1547 }
1548
1549 /// Returns true if instruction is a memory operation that this pass is capable
1550 /// of operating on.
1551 static bool isMemoryOp(const MachineInstr &MI) {
1552   unsigned Opcode = MI.getOpcode();
1553   switch (Opcode) {
1554   case ARM::VLDRS:
1555   case ARM::VSTRS:
1556   case ARM::VLDRD:
1557   case ARM::VSTRD:
1558   case ARM::LDRi12:
1559   case ARM::STRi12:
1560   case ARM::tLDRi:
1561   case ARM::tSTRi:
1562   case ARM::tLDRspi:
1563   case ARM::tSTRspi:
1564   case ARM::t2LDRi8:
1565   case ARM::t2LDRi12:
1566   case ARM::t2STRi8:
1567   case ARM::t2STRi12:
1568     break;
1569   default:
1570     return false;
1571   }
1572   if (!MI.getOperand(1).isReg())
1573     return false;
1574
1575   // When no memory operands are present, conservatively assume unaligned,
1576   // volatile, unfoldable.
1577   if (!MI.hasOneMemOperand())
1578     return false;
1579
1580   const MachineMemOperand &MMO = **MI.memoperands_begin();
1581
1582   // Don't touch volatile memory accesses - we may be changing their order.
1583   if (MMO.isVolatile())
1584     return false;
1585
1586   // Unaligned ldr/str is emulated by some kernels, but unaligned ldm/stm is
1587   // not.
1588   if (MMO.getAlignment() < 4)
1589     return false;
1590
1591   // str <undef> could probably be eliminated entirely, but for now we just want
1592   // to avoid making a mess of it.
1593   // FIXME: Use str <undef> as a wildcard to enable better stm folding.
1594   if (MI.getOperand(0).isReg() && MI.getOperand(0).isUndef())
1595     return false;
1596
1597   // Likewise don't mess with references to undefined addresses.
1598   if (MI.getOperand(1).isUndef())
1599     return false;
1600
1601   return true;
1602 }
1603
1604 static void InsertLDR_STR(MachineBasicBlock &MBB,
1605                           MachineBasicBlock::iterator &MBBI, int Offset,
1606                           bool isDef, unsigned NewOpc, unsigned Reg,
1607                           bool RegDeadKill, bool RegUndef, unsigned BaseReg,
1608                           bool BaseKill, bool BaseUndef, ARMCC::CondCodes Pred,
1609                           unsigned PredReg, const TargetInstrInfo *TII) {
1610   if (isDef) {
1611     MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
1612                                       TII->get(NewOpc))
1613       .addReg(Reg, getDefRegState(true) | getDeadRegState(RegDeadKill))
1614       .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef));
1615     MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
1616   } else {
1617     MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
1618                                       TII->get(NewOpc))
1619       .addReg(Reg, getKillRegState(RegDeadKill) | getUndefRegState(RegUndef))
1620       .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef));
1621     MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
1622   }
1623 }
1624
1625 bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
1626                                           MachineBasicBlock::iterator &MBBI) {
1627   MachineInstr *MI = &*MBBI;
1628   unsigned Opcode = MI->getOpcode();
1629   // FIXME: Code/comments below check Opcode == t2STRDi8, but this check returns
1630   // if we see this opcode.
1631   if (Opcode != ARM::LDRD && Opcode != ARM::STRD && Opcode != ARM::t2LDRDi8)
1632     return false;
1633
1634   const MachineOperand &BaseOp = MI->getOperand(2);
1635   unsigned BaseReg = BaseOp.getReg();
1636   unsigned EvenReg = MI->getOperand(0).getReg();
1637   unsigned OddReg  = MI->getOperand(1).getReg();
1638   unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
1639   unsigned OddRegNum  = TRI->getDwarfRegNum(OddReg, false);
1640
1641   // ARM errata 602117: LDRD with base in list may result in incorrect base
1642   // register when interrupted or faulted.
1643   bool Errata602117 = EvenReg == BaseReg &&
1644     (Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8) && STI->isCortexM3();
1645   // ARM LDRD/STRD needs consecutive registers.
1646   bool NonConsecutiveRegs = (Opcode == ARM::LDRD || Opcode == ARM::STRD) &&
1647     (EvenRegNum % 2 != 0 || EvenRegNum + 1 != OddRegNum);
1648
1649   if (!Errata602117 && !NonConsecutiveRegs)
1650     return false;
1651
1652   bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
1653   bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
1654   bool EvenDeadKill = isLd ?
1655     MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
1656   bool EvenUndef = MI->getOperand(0).isUndef();
1657   bool OddDeadKill  = isLd ?
1658     MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
1659   bool OddUndef = MI->getOperand(1).isUndef();
1660   bool BaseKill = BaseOp.isKill();
1661   bool BaseUndef = BaseOp.isUndef();
1662   assert((isT2 || MI->getOperand(3).getReg() == ARM::NoRegister) &&
1663          "register offset not handled below");
1664   int OffImm = getMemoryOpOffset(*MI);
1665   unsigned PredReg = 0;
1666   ARMCC::CondCodes Pred = getInstrPredicate(*MI, PredReg);
1667
1668   if (OddRegNum > EvenRegNum && OffImm == 0) {
1669     // Ascending register numbers and no offset. It's safe to change it to a
1670     // ldm or stm.
1671     unsigned NewOpc = (isLd)
1672       ? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
1673       : (isT2 ? ARM::t2STMIA : ARM::STMIA);
1674     if (isLd) {
1675       BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
1676         .addReg(BaseReg, getKillRegState(BaseKill))
1677         .addImm(Pred).addReg(PredReg)
1678         .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
1679         .addReg(OddReg,  getDefRegState(isLd) | getDeadRegState(OddDeadKill));
1680       ++NumLDRD2LDM;
1681     } else {
1682       BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
1683         .addReg(BaseReg, getKillRegState(BaseKill))
1684         .addImm(Pred).addReg(PredReg)
1685         .addReg(EvenReg,
1686                 getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
1687         .addReg(OddReg,
1688                 getKillRegState(OddDeadKill)  | getUndefRegState(OddUndef));
1689       ++NumSTRD2STM;
1690     }
1691   } else {
1692     // Split into two instructions.
1693     unsigned NewOpc = (isLd)
1694       ? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
1695       : (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
1696     // Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
1697     // so adjust and use t2LDRi12 here for that.
1698     unsigned NewOpc2 = (isLd)
1699       ? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
1700       : (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
1701     // If this is a load, make sure the first load does not clobber the base
1702     // register before the second load reads it.
1703     if (isLd && TRI->regsOverlap(EvenReg, BaseReg)) {
1704       assert(!TRI->regsOverlap(OddReg, BaseReg));
1705       InsertLDR_STR(MBB, MBBI, OffImm + 4, isLd, NewOpc2, OddReg, OddDeadKill,
1706                     false, BaseReg, false, BaseUndef, Pred, PredReg, TII);
1707       InsertLDR_STR(MBB, MBBI, OffImm, isLd, NewOpc, EvenReg, EvenDeadKill,
1708                     false, BaseReg, BaseKill, BaseUndef, Pred, PredReg, TII);
1709     } else {
1710       if (OddReg == EvenReg && EvenDeadKill) {
1711         // If the two source operands are the same, the kill marker is
1712         // probably on the first one. e.g.
1713         // t2STRDi8 killed %r5, %r5, killed %r9, 0, 14, %reg0
1714         EvenDeadKill = false;
1715         OddDeadKill = true;
1716       }
1717       // Never kill the base register in the first instruction.
1718       if (EvenReg == BaseReg)
1719         EvenDeadKill = false;
1720       InsertLDR_STR(MBB, MBBI, OffImm, isLd, NewOpc, EvenReg, EvenDeadKill,
1721                     EvenUndef, BaseReg, false, BaseUndef, Pred, PredReg, TII);
1722       InsertLDR_STR(MBB, MBBI, OffImm + 4, isLd, NewOpc2, OddReg, OddDeadKill,
1723                     OddUndef, BaseReg, BaseKill, BaseUndef, Pred, PredReg, TII);
1724     }
1725     if (isLd)
1726       ++NumLDRD2LDR;
1727     else
1728       ++NumSTRD2STR;
1729   }
1730
1731   MBBI = MBB.erase(MBBI);
1732   return true;
1733 }
1734
1735 /// An optimization pass to turn multiple LDR / STR ops of the same base and
1736 /// incrementing offset into LDM / STM ops.
1737 bool ARMLoadStoreOpt::LoadStoreMultipleOpti(MachineBasicBlock &MBB) {
1738   MemOpQueue MemOps;
1739   unsigned CurrBase = 0;
1740   unsigned CurrOpc = ~0u;
1741   ARMCC::CondCodes CurrPred = ARMCC::AL;
1742   unsigned Position = 0;
1743   assert(Candidates.size() == 0);
1744   assert(MergeBaseCandidates.size() == 0);
1745   LiveRegsValid = false;
1746
1747   for (MachineBasicBlock::iterator I = MBB.end(), MBBI; I != MBB.begin();
1748        I = MBBI) {
1749     // The instruction in front of the iterator is the one we look at.
1750     MBBI = std::prev(I);
1751     if (FixInvalidRegPairOp(MBB, MBBI))
1752       continue;
1753     ++Position;
1754
1755     if (isMemoryOp(*MBBI)) {
1756       unsigned Opcode = MBBI->getOpcode();
1757       const MachineOperand &MO = MBBI->getOperand(0);
1758       unsigned Reg = MO.getReg();
1759       unsigned Base = getLoadStoreBaseOp(*MBBI).getReg();
1760       unsigned PredReg = 0;
1761       ARMCC::CondCodes Pred = getInstrPredicate(*MBBI, PredReg);
1762       int Offset = getMemoryOpOffset(*MBBI);
1763       if (CurrBase == 0) {
1764         // Start of a new chain.
1765         CurrBase = Base;
1766         CurrOpc  = Opcode;
1767         CurrPred = Pred;
1768         MemOps.push_back(MemOpQueueEntry(*MBBI, Offset, Position));
1769         continue;
1770       }
1771       // Note: No need to match PredReg in the next if.
1772       if (CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred) {
1773         // Watch out for:
1774         //   r4 := ldr [r0, #8]
1775         //   r4 := ldr [r0, #4]
1776         // or
1777         //   r0 := ldr [r0]
1778         // If a load overrides the base register or a register loaded by
1779         // another load in our chain, we cannot take this instruction.
1780         bool Overlap = false;
1781         if (isLoadSingle(Opcode)) {
1782           Overlap = (Base == Reg);
1783           if (!Overlap) {
1784             for (const MemOpQueueEntry &E : MemOps) {
1785               if (TRI->regsOverlap(Reg, E.MI->getOperand(0).getReg())) {
1786                 Overlap = true;
1787                 break;
1788               }
1789             }
1790           }
1791         }
1792
1793         if (!Overlap) {
1794           // Check offset and sort memory operation into the current chain.
1795           if (Offset > MemOps.back().Offset) {
1796             MemOps.push_back(MemOpQueueEntry(*MBBI, Offset, Position));
1797             continue;
1798           } else {
1799             MemOpQueue::iterator MI, ME;
1800             for (MI = MemOps.begin(), ME = MemOps.end(); MI != ME; ++MI) {
1801               if (Offset < MI->Offset) {
1802                 // Found a place to insert.
1803                 break;
1804               }
1805               if (Offset == MI->Offset) {
1806                 // Collision, abort.
1807                 MI = ME;
1808                 break;
1809               }
1810             }
1811             if (MI != MemOps.end()) {
1812               MemOps.insert(MI, MemOpQueueEntry(*MBBI, Offset, Position));
1813               continue;
1814             }
1815           }
1816         }
1817       }
1818
1819       // Don't advance the iterator; The op will start a new chain next.
1820       MBBI = I;
1821       --Position;
1822       // Fallthrough to look into existing chain.
1823     } else if (MBBI->isDebugInstr()) {
1824       continue;
1825     } else if (MBBI->getOpcode() == ARM::t2LDRDi8 ||
1826                MBBI->getOpcode() == ARM::t2STRDi8) {
1827       // ARMPreAllocLoadStoreOpt has already formed some LDRD/STRD instructions
1828       // remember them because we may still be able to merge add/sub into them.
1829       MergeBaseCandidates.push_back(&*MBBI);
1830     }
1831
1832     // If we are here then the chain is broken; Extract candidates for a merge.
1833     if (MemOps.size() > 0) {
1834       FormCandidates(MemOps);
1835       // Reset for the next chain.
1836       CurrBase = 0;
1837       CurrOpc = ~0u;
1838       CurrPred = ARMCC::AL;
1839       MemOps.clear();
1840     }
1841   }
1842   if (MemOps.size() > 0)
1843     FormCandidates(MemOps);
1844
1845   // Sort candidates so they get processed from end to begin of the basic
1846   // block later; This is necessary for liveness calculation.
1847   auto LessThan = [](const MergeCandidate* M0, const MergeCandidate *M1) {
1848     return M0->InsertPos < M1->InsertPos;
1849   };
1850   llvm::sort(Candidates, LessThan);
1851
1852   // Go through list of candidates and merge.
1853   bool Changed = false;
1854   for (const MergeCandidate *Candidate : Candidates) {
1855     if (Candidate->CanMergeToLSMulti || Candidate->CanMergeToLSDouble) {
1856       MachineInstr *Merged = MergeOpsUpdate(*Candidate);
1857       // Merge preceding/trailing base inc/dec into the merged op.
1858       if (Merged) {
1859         Changed = true;
1860         unsigned Opcode = Merged->getOpcode();
1861         if (Opcode == ARM::t2STRDi8 || Opcode == ARM::t2LDRDi8)
1862           MergeBaseUpdateLSDouble(*Merged);
1863         else
1864           MergeBaseUpdateLSMultiple(Merged);
1865       } else {
1866         for (MachineInstr *MI : Candidate->Instrs) {
1867           if (MergeBaseUpdateLoadStore(MI))
1868             Changed = true;
1869         }
1870       }
1871     } else {
1872       assert(Candidate->Instrs.size() == 1);
1873       if (MergeBaseUpdateLoadStore(Candidate->Instrs.front()))
1874         Changed = true;
1875     }
1876   }
1877   Candidates.clear();
1878   // Try to fold add/sub into the LDRD/STRD formed by ARMPreAllocLoadStoreOpt.
1879   for (MachineInstr *MI : MergeBaseCandidates)
1880     MergeBaseUpdateLSDouble(*MI);
1881   MergeBaseCandidates.clear();
1882
1883   return Changed;
1884 }
1885
1886 /// If this is a exit BB, try merging the return ops ("bx lr" and "mov pc, lr")
1887 /// into the preceding stack restore so it directly restore the value of LR
1888 /// into pc.
1889 ///   ldmfd sp!, {..., lr}
1890 ///   bx lr
1891 /// or
1892 ///   ldmfd sp!, {..., lr}
1893 ///   mov pc, lr
1894 /// =>
1895 ///   ldmfd sp!, {..., pc}
1896 bool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) {
1897   // Thumb1 LDM doesn't allow high registers.
1898   if (isThumb1) return false;
1899   if (MBB.empty()) return false;
1900
1901   MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
1902   if (MBBI != MBB.begin() && MBBI != MBB.end() &&
1903       (MBBI->getOpcode() == ARM::BX_RET ||
1904        MBBI->getOpcode() == ARM::tBX_RET ||
1905        MBBI->getOpcode() == ARM::MOVPCLR)) {
1906     MachineBasicBlock::iterator PrevI = std::prev(MBBI);
1907     // Ignore any debug instructions.
1908     while (PrevI->isDebugInstr() && PrevI != MBB.begin())
1909       --PrevI;
1910     MachineInstr &PrevMI = *PrevI;
1911     unsigned Opcode = PrevMI.getOpcode();
1912     if (Opcode == ARM::LDMIA_UPD || Opcode == ARM::LDMDA_UPD ||
1913         Opcode == ARM::LDMDB_UPD || Opcode == ARM::LDMIB_UPD ||
1914         Opcode == ARM::t2LDMIA_UPD || Opcode == ARM::t2LDMDB_UPD) {
1915       MachineOperand &MO = PrevMI.getOperand(PrevMI.getNumOperands() - 1);
1916       if (MO.getReg() != ARM::LR)
1917         return false;
1918       unsigned NewOpc = (isThumb2 ? ARM::t2LDMIA_RET : ARM::LDMIA_RET);
1919       assert(((isThumb2 && Opcode == ARM::t2LDMIA_UPD) ||
1920               Opcode == ARM::LDMIA_UPD) && "Unsupported multiple load-return!");
1921       PrevMI.setDesc(TII->get(NewOpc));
1922       MO.setReg(ARM::PC);
1923       PrevMI.copyImplicitOps(*MBB.getParent(), *MBBI);
1924       MBB.erase(MBBI);
1925       // We now restore LR into PC so it is not live-out of the return block
1926       // anymore: Clear the CSI Restored bit.
1927       MachineFrameInfo &MFI = MBB.getParent()->getFrameInfo();
1928       // CSI should be fixed after PrologEpilog Insertion
1929       assert(MFI.isCalleeSavedInfoValid() && "CSI should be valid");
1930       for (CalleeSavedInfo &Info : MFI.getCalleeSavedInfo()) {
1931         if (Info.getReg() == ARM::LR) {
1932           Info.setRestored(false);
1933           break;
1934         }
1935       }
1936       return true;
1937     }
1938   }
1939   return false;
1940 }
1941
1942 bool ARMLoadStoreOpt::CombineMovBx(MachineBasicBlock &MBB) {
1943   MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator();
1944   if (MBBI == MBB.begin() || MBBI == MBB.end() ||
1945       MBBI->getOpcode() != ARM::tBX_RET)
1946     return false;
1947
1948   MachineBasicBlock::iterator Prev = MBBI;
1949   --Prev;
1950   if (Prev->getOpcode() != ARM::tMOVr || !Prev->definesRegister(ARM::LR))
1951     return false;
1952
1953   for (auto Use : Prev->uses())
1954     if (Use.isKill()) {
1955       assert(STI->hasV4TOps());
1956       BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(ARM::tBX))
1957           .addReg(Use.getReg(), RegState::Kill)
1958           .add(predOps(ARMCC::AL))
1959           .copyImplicitOps(*MBBI);
1960       MBB.erase(MBBI);
1961       MBB.erase(Prev);
1962       return true;
1963     }
1964
1965   llvm_unreachable("tMOVr doesn't kill a reg before tBX_RET?");
1966 }
1967
1968 bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
1969   if (skipFunction(Fn.getFunction()))
1970     return false;
1971
1972   MF = &Fn;
1973   STI = &static_cast<const ARMSubtarget &>(Fn.getSubtarget());
1974   TL = STI->getTargetLowering();
1975   AFI = Fn.getInfo<ARMFunctionInfo>();
1976   TII = STI->getInstrInfo();
1977   TRI = STI->getRegisterInfo();
1978
1979   RegClassInfoValid = false;
1980   isThumb2 = AFI->isThumb2Function();
1981   isThumb1 = AFI->isThumbFunction() && !isThumb2;
1982
1983   bool Modified = false;
1984   for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
1985        ++MFI) {
1986     MachineBasicBlock &MBB = *MFI;
1987     Modified |= LoadStoreMultipleOpti(MBB);
1988     if (STI->hasV5TOps())
1989       Modified |= MergeReturnIntoLDM(MBB);
1990     if (isThumb1)
1991       Modified |= CombineMovBx(MBB);
1992   }
1993
1994   Allocator.DestroyAll();
1995   return Modified;
1996 }
1997
1998 #define ARM_PREALLOC_LOAD_STORE_OPT_NAME                                       \
1999   "ARM pre- register allocation load / store optimization pass"
2000
2001 namespace {
2002
2003   /// Pre- register allocation pass that move load / stores from consecutive
2004   /// locations close to make it more likely they will be combined later.
2005   struct ARMPreAllocLoadStoreOpt : public MachineFunctionPass{
2006     static char ID;
2007
2008     AliasAnalysis *AA;
2009     const DataLayout *TD;
2010     const TargetInstrInfo *TII;
2011     const TargetRegisterInfo *TRI;
2012     const ARMSubtarget *STI;
2013     MachineRegisterInfo *MRI;
2014     MachineFunction *MF;
2015
2016     ARMPreAllocLoadStoreOpt() : MachineFunctionPass(ID) {}
2017
2018     bool runOnMachineFunction(MachineFunction &Fn) override;
2019
2020     StringRef getPassName() const override {
2021       return ARM_PREALLOC_LOAD_STORE_OPT_NAME;
2022     }
2023
2024     void getAnalysisUsage(AnalysisUsage &AU) const override {
2025       AU.addRequired<AAResultsWrapperPass>();
2026       MachineFunctionPass::getAnalysisUsage(AU);
2027     }
2028
2029   private:
2030     bool CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1, DebugLoc &dl,
2031                           unsigned &NewOpc, unsigned &EvenReg,
2032                           unsigned &OddReg, unsigned &BaseReg,
2033                           int &Offset,
2034                           unsigned &PredReg, ARMCC::CondCodes &Pred,
2035                           bool &isT2);
2036     bool RescheduleOps(MachineBasicBlock *MBB,
2037                        SmallVectorImpl<MachineInstr *> &Ops,
2038                        unsigned Base, bool isLd,
2039                        DenseMap<MachineInstr*, unsigned> &MI2LocMap);
2040     bool RescheduleLoadStoreInstrs(MachineBasicBlock *MBB);
2041   };
2042
2043 } // end anonymous namespace
2044
2045 char ARMPreAllocLoadStoreOpt::ID = 0;
2046
2047 INITIALIZE_PASS(ARMPreAllocLoadStoreOpt, "arm-prera-ldst-opt",
2048                 ARM_PREALLOC_LOAD_STORE_OPT_NAME, false, false)
2049
2050 // Limit the number of instructions to be rescheduled.
2051 // FIXME: tune this limit, and/or come up with some better heuristics.
2052 static cl::opt<unsigned> InstReorderLimit("arm-prera-ldst-opt-reorder-limit",
2053                                           cl::init(8), cl::Hidden);
2054
2055 bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
2056   if (AssumeMisalignedLoadStores || skipFunction(Fn.getFunction()))
2057     return false;
2058
2059   TD = &Fn.getDataLayout();
2060   STI = &static_cast<const ARMSubtarget &>(Fn.getSubtarget());
2061   TII = STI->getInstrInfo();
2062   TRI = STI->getRegisterInfo();
2063   MRI = &Fn.getRegInfo();
2064   MF  = &Fn;
2065   AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
2066
2067   bool Modified = false;
2068   for (MachineBasicBlock &MFI : Fn)
2069     Modified |= RescheduleLoadStoreInstrs(&MFI);
2070
2071   return Modified;
2072 }
2073
2074 static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base,
2075                                       MachineBasicBlock::iterator I,
2076                                       MachineBasicBlock::iterator E,
2077                                       SmallPtrSetImpl<MachineInstr*> &MemOps,
2078                                       SmallSet<unsigned, 4> &MemRegs,
2079                                       const TargetRegisterInfo *TRI,
2080                                       AliasAnalysis *AA) {
2081   // Are there stores / loads / calls between them?
2082   SmallSet<unsigned, 4> AddedRegPressure;
2083   while (++I != E) {
2084     if (I->isDebugInstr() || MemOps.count(&*I))
2085       continue;
2086     if (I->isCall() || I->isTerminator() || I->hasUnmodeledSideEffects())
2087       return false;
2088     if (I->mayStore() || (!isLd && I->mayLoad()))
2089       for (MachineInstr *MemOp : MemOps)
2090         if (I->mayAlias(AA, *MemOp, /*UseTBAA*/ false))
2091           return false;
2092     for (unsigned j = 0, NumOps = I->getNumOperands(); j != NumOps; ++j) {
2093       MachineOperand &MO = I->getOperand(j);
2094       if (!MO.isReg())
2095         continue;
2096       unsigned Reg = MO.getReg();
2097       if (MO.isDef() && TRI->regsOverlap(Reg, Base))
2098         return false;
2099       if (Reg != Base && !MemRegs.count(Reg))
2100         AddedRegPressure.insert(Reg);
2101     }
2102   }
2103
2104   // Estimate register pressure increase due to the transformation.
2105   if (MemRegs.size() <= 4)
2106     // Ok if we are moving small number of instructions.
2107     return true;
2108   return AddedRegPressure.size() <= MemRegs.size() * 2;
2109 }
2110
2111 bool
2112 ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1,
2113                                           DebugLoc &dl, unsigned &NewOpc,
2114                                           unsigned &FirstReg,
2115                                           unsigned &SecondReg,
2116                                           unsigned &BaseReg, int &Offset,
2117                                           unsigned &PredReg,
2118                                           ARMCC::CondCodes &Pred,
2119                                           bool &isT2) {
2120   // Make sure we're allowed to generate LDRD/STRD.
2121   if (!STI->hasV5TEOps())
2122     return false;
2123
2124   // FIXME: VLDRS / VSTRS -> VLDRD / VSTRD
2125   unsigned Scale = 1;
2126   unsigned Opcode = Op0->getOpcode();
2127   if (Opcode == ARM::LDRi12) {
2128     NewOpc = ARM::LDRD;
2129   } else if (Opcode == ARM::STRi12) {
2130     NewOpc = ARM::STRD;
2131   } else if (Opcode == ARM::t2LDRi8 || Opcode == ARM::t2LDRi12) {
2132     NewOpc = ARM::t2LDRDi8;
2133     Scale = 4;
2134     isT2 = true;
2135   } else if (Opcode == ARM::t2STRi8 || Opcode == ARM::t2STRi12) {
2136     NewOpc = ARM::t2STRDi8;
2137     Scale = 4;
2138     isT2 = true;
2139   } else {
2140     return false;
2141   }
2142
2143   // Make sure the base address satisfies i64 ld / st alignment requirement.
2144   // At the moment, we ignore the memoryoperand's value.
2145   // If we want to use AliasAnalysis, we should check it accordingly.
2146   if (!Op0->hasOneMemOperand() ||
2147       (*Op0->memoperands_begin())->isVolatile())
2148     return false;
2149
2150   unsigned Align = (*Op0->memoperands_begin())->getAlignment();
2151   const Function &Func = MF->getFunction();
2152   unsigned ReqAlign = STI->hasV6Ops()
2153     ? TD->getABITypeAlignment(Type::getInt64Ty(Func.getContext()))
2154     : 8;  // Pre-v6 need 8-byte align
2155   if (Align < ReqAlign)
2156     return false;
2157
2158   // Then make sure the immediate offset fits.
2159   int OffImm = getMemoryOpOffset(*Op0);
2160   if (isT2) {
2161     int Limit = (1 << 8) * Scale;
2162     if (OffImm >= Limit || (OffImm <= -Limit) || (OffImm & (Scale-1)))
2163       return false;
2164     Offset = OffImm;
2165   } else {
2166     ARM_AM::AddrOpc AddSub = ARM_AM::add;
2167     if (OffImm < 0) {
2168       AddSub = ARM_AM::sub;
2169       OffImm = - OffImm;
2170     }
2171     int Limit = (1 << 8) * Scale;
2172     if (OffImm >= Limit || (OffImm & (Scale-1)))
2173       return false;
2174     Offset = ARM_AM::getAM3Opc(AddSub, OffImm);
2175   }
2176   FirstReg = Op0->getOperand(0).getReg();
2177   SecondReg = Op1->getOperand(0).getReg();
2178   if (FirstReg == SecondReg)
2179     return false;
2180   BaseReg = Op0->getOperand(1).getReg();
2181   Pred = getInstrPredicate(*Op0, PredReg);
2182   dl = Op0->getDebugLoc();
2183   return true;
2184 }
2185
2186 bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
2187                                  SmallVectorImpl<MachineInstr *> &Ops,
2188                                  unsigned Base, bool isLd,
2189                                  DenseMap<MachineInstr*, unsigned> &MI2LocMap) {
2190   bool RetVal = false;
2191
2192   // Sort by offset (in reverse order).
2193   llvm::sort(Ops, [](const MachineInstr *LHS, const MachineInstr *RHS) {
2194     int LOffset = getMemoryOpOffset(*LHS);
2195     int ROffset = getMemoryOpOffset(*RHS);
2196     assert(LHS == RHS || LOffset != ROffset);
2197     return LOffset > ROffset;
2198   });
2199
2200   // The loads / stores of the same base are in order. Scan them from first to
2201   // last and check for the following:
2202   // 1. Any def of base.
2203   // 2. Any gaps.
2204   while (Ops.size() > 1) {
2205     unsigned FirstLoc = ~0U;
2206     unsigned LastLoc = 0;
2207     MachineInstr *FirstOp = nullptr;
2208     MachineInstr *LastOp = nullptr;
2209     int LastOffset = 0;
2210     unsigned LastOpcode = 0;
2211     unsigned LastBytes = 0;
2212     unsigned NumMove = 0;
2213     for (int i = Ops.size() - 1; i >= 0; --i) {
2214       // Make sure each operation has the same kind.
2215       MachineInstr *Op = Ops[i];
2216       unsigned LSMOpcode
2217         = getLoadStoreMultipleOpcode(Op->getOpcode(), ARM_AM::ia);
2218       if (LastOpcode && LSMOpcode != LastOpcode)
2219         break;
2220
2221       // Check that we have a continuous set of offsets.
2222       int Offset = getMemoryOpOffset(*Op);
2223       unsigned Bytes = getLSMultipleTransferSize(Op);
2224       if (LastBytes) {
2225         if (Bytes != LastBytes || Offset != (LastOffset + (int)Bytes))
2226           break;
2227       }
2228
2229       // Don't try to reschedule too many instructions.
2230       if (NumMove == InstReorderLimit)
2231         break;
2232
2233       // Found a mergable instruction; save information about it.
2234       ++NumMove;
2235       LastOffset = Offset;
2236       LastBytes = Bytes;
2237       LastOpcode = LSMOpcode;
2238
2239       unsigned Loc = MI2LocMap[Op];
2240       if (Loc <= FirstLoc) {
2241         FirstLoc = Loc;
2242         FirstOp = Op;
2243       }
2244       if (Loc >= LastLoc) {
2245         LastLoc = Loc;
2246         LastOp = Op;
2247       }
2248     }
2249
2250     if (NumMove <= 1)
2251       Ops.pop_back();
2252     else {
2253       SmallPtrSet<MachineInstr*, 4> MemOps;
2254       SmallSet<unsigned, 4> MemRegs;
2255       for (size_t i = Ops.size() - NumMove, e = Ops.size(); i != e; ++i) {
2256         MemOps.insert(Ops[i]);
2257         MemRegs.insert(Ops[i]->getOperand(0).getReg());
2258       }
2259
2260       // Be conservative, if the instructions are too far apart, don't
2261       // move them. We want to limit the increase of register pressure.
2262       bool DoMove = (LastLoc - FirstLoc) <= NumMove*4; // FIXME: Tune this.
2263       if (DoMove)
2264         DoMove = IsSafeAndProfitableToMove(isLd, Base, FirstOp, LastOp,
2265                                            MemOps, MemRegs, TRI, AA);
2266       if (!DoMove) {
2267         for (unsigned i = 0; i != NumMove; ++i)
2268           Ops.pop_back();
2269       } else {
2270         // This is the new location for the loads / stores.
2271         MachineBasicBlock::iterator InsertPos = isLd ? FirstOp : LastOp;
2272         while (InsertPos != MBB->end() &&
2273                (MemOps.count(&*InsertPos) || InsertPos->isDebugInstr()))
2274           ++InsertPos;
2275
2276         // If we are moving a pair of loads / stores, see if it makes sense
2277         // to try to allocate a pair of registers that can form register pairs.
2278         MachineInstr *Op0 = Ops.back();
2279         MachineInstr *Op1 = Ops[Ops.size()-2];
2280         unsigned FirstReg = 0, SecondReg = 0;
2281         unsigned BaseReg = 0, PredReg = 0;
2282         ARMCC::CondCodes Pred = ARMCC::AL;
2283         bool isT2 = false;
2284         unsigned NewOpc = 0;
2285         int Offset = 0;
2286         DebugLoc dl;
2287         if (NumMove == 2 && CanFormLdStDWord(Op0, Op1, dl, NewOpc,
2288                                              FirstReg, SecondReg, BaseReg,
2289                                              Offset, PredReg, Pred, isT2)) {
2290           Ops.pop_back();
2291           Ops.pop_back();
2292
2293           const MCInstrDesc &MCID = TII->get(NewOpc);
2294           const TargetRegisterClass *TRC = TII->getRegClass(MCID, 0, TRI, *MF);
2295           MRI->constrainRegClass(FirstReg, TRC);
2296           MRI->constrainRegClass(SecondReg, TRC);
2297
2298           // Form the pair instruction.
2299           if (isLd) {
2300             MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
2301               .addReg(FirstReg, RegState::Define)
2302               .addReg(SecondReg, RegState::Define)
2303               .addReg(BaseReg);
2304             // FIXME: We're converting from LDRi12 to an insn that still
2305             // uses addrmode2, so we need an explicit offset reg. It should
2306             // always by reg0 since we're transforming LDRi12s.
2307             if (!isT2)
2308               MIB.addReg(0);
2309             MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
2310             MIB.cloneMergedMemRefs({Op0, Op1});
2311             LLVM_DEBUG(dbgs() << "Formed " << *MIB << "\n");
2312             ++NumLDRDFormed;
2313           } else {
2314             MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
2315               .addReg(FirstReg)
2316               .addReg(SecondReg)
2317               .addReg(BaseReg);
2318             // FIXME: We're converting from LDRi12 to an insn that still
2319             // uses addrmode2, so we need an explicit offset reg. It should
2320             // always by reg0 since we're transforming STRi12s.
2321             if (!isT2)
2322               MIB.addReg(0);
2323             MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
2324             MIB.cloneMergedMemRefs({Op0, Op1});
2325             LLVM_DEBUG(dbgs() << "Formed " << *MIB << "\n");
2326             ++NumSTRDFormed;
2327           }
2328           MBB->erase(Op0);
2329           MBB->erase(Op1);
2330
2331           if (!isT2) {
2332             // Add register allocation hints to form register pairs.
2333             MRI->setRegAllocationHint(FirstReg, ARMRI::RegPairEven, SecondReg);
2334             MRI->setRegAllocationHint(SecondReg,  ARMRI::RegPairOdd, FirstReg);
2335           }
2336         } else {
2337           for (unsigned i = 0; i != NumMove; ++i) {
2338             MachineInstr *Op = Ops.back();
2339             Ops.pop_back();
2340             MBB->splice(InsertPos, MBB, Op);
2341           }
2342         }
2343
2344         NumLdStMoved += NumMove;
2345         RetVal = true;
2346       }
2347     }
2348   }
2349
2350   return RetVal;
2351 }
2352
2353 bool
2354 ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
2355   bool RetVal = false;
2356
2357   DenseMap<MachineInstr*, unsigned> MI2LocMap;
2358   using MapIt = DenseMap<unsigned, SmallVector<MachineInstr *, 4>>::iterator;
2359   using Base2InstMap = DenseMap<unsigned, SmallVector<MachineInstr *, 4>>;
2360   using BaseVec = SmallVector<unsigned, 4>;
2361   Base2InstMap Base2LdsMap;
2362   Base2InstMap Base2StsMap;
2363   BaseVec LdBases;
2364   BaseVec StBases;
2365
2366   unsigned Loc = 0;
2367   MachineBasicBlock::iterator MBBI = MBB->begin();
2368   MachineBasicBlock::iterator E = MBB->end();
2369   while (MBBI != E) {
2370     for (; MBBI != E; ++MBBI) {
2371       MachineInstr &MI = *MBBI;
2372       if (MI.isCall() || MI.isTerminator()) {
2373         // Stop at barriers.
2374         ++MBBI;
2375         break;
2376       }
2377
2378       if (!MI.isDebugInstr())
2379         MI2LocMap[&MI] = ++Loc;
2380
2381       if (!isMemoryOp(MI))
2382         continue;
2383       unsigned PredReg = 0;
2384       if (getInstrPredicate(MI, PredReg) != ARMCC::AL)
2385         continue;
2386
2387       int Opc = MI.getOpcode();
2388       bool isLd = isLoadSingle(Opc);
2389       unsigned Base = MI.getOperand(1).getReg();
2390       int Offset = getMemoryOpOffset(MI);
2391       bool StopHere = false;
2392       auto FindBases = [&] (Base2InstMap &Base2Ops, BaseVec &Bases) {
2393         MapIt BI = Base2Ops.find(Base);
2394         if (BI == Base2Ops.end()) {
2395           Base2Ops[Base].push_back(&MI);
2396           Bases.push_back(Base);
2397           return;
2398         }
2399         for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
2400           if (Offset == getMemoryOpOffset(*BI->second[i])) {
2401             StopHere = true;
2402             break;
2403           }
2404         }
2405         if (!StopHere)
2406           BI->second.push_back(&MI);
2407       };
2408
2409       if (isLd)
2410         FindBases(Base2LdsMap, LdBases);
2411       else
2412         FindBases(Base2StsMap, StBases);
2413
2414       if (StopHere) {
2415         // Found a duplicate (a base+offset combination that's seen earlier).
2416         // Backtrack.
2417         --Loc;
2418         break;
2419       }
2420     }
2421
2422     // Re-schedule loads.
2423     for (unsigned i = 0, e = LdBases.size(); i != e; ++i) {
2424       unsigned Base = LdBases[i];
2425       SmallVectorImpl<MachineInstr *> &Lds = Base2LdsMap[Base];
2426       if (Lds.size() > 1)
2427         RetVal |= RescheduleOps(MBB, Lds, Base, true, MI2LocMap);
2428     }
2429
2430     // Re-schedule stores.
2431     for (unsigned i = 0, e = StBases.size(); i != e; ++i) {
2432       unsigned Base = StBases[i];
2433       SmallVectorImpl<MachineInstr *> &Sts = Base2StsMap[Base];
2434       if (Sts.size() > 1)
2435         RetVal |= RescheduleOps(MBB, Sts, Base, false, MI2LocMap);
2436     }
2437
2438     if (MBBI != E) {
2439       Base2LdsMap.clear();
2440       Base2StsMap.clear();
2441       LdBases.clear();
2442       StBases.clear();
2443     }
2444   }
2445
2446   return RetVal;
2447 }
2448
2449 /// Returns an instance of the load / store optimization pass.
2450 FunctionPass *llvm::createARMLoadStoreOptimizationPass(bool PreAlloc) {
2451   if (PreAlloc)
2452     return new ARMPreAllocLoadStoreOpt();
2453   return new ARMLoadStoreOpt();
2454 }