llvm/lib/Target/ARM/ARMBaseInstrInfo.h

   1 //===-- ARMBaseInstrInfo.h - ARM Base Instruction Information ---*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the Base ARM implementation of the TargetInstrInfo class.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef LLVM_LIB_TARGET_ARM_ARMBASEINSTRINFO_H
  14 #define LLVM_LIB_TARGET_ARM_ARMBASEINSTRINFO_H
  15
  16 #include "ARMBaseRegisterInfo.h"
  17 #include "MCTargetDesc/ARMBaseInfo.h"
  18 #include "MCTargetDesc/ARMMCTargetDesc.h"
  19 #include "llvm/ADT/DenseMap.h"
  20 #include "llvm/ADT/SmallSet.h"
  21 #include "llvm/CodeGen/MachineBasicBlock.h"
  22 #include "llvm/CodeGen/MachineInstr.h"
  23 #include "llvm/CodeGen/MachineInstrBuilder.h"
  24 #include "llvm/CodeGen/MachineOperand.h"
  25 #include "llvm/CodeGen/MachineRegisterInfo.h"
  26 #include "llvm/CodeGen/Register.h"
  27 #include "llvm/CodeGen/TargetInstrInfo.h"
  28 #include "llvm/IR/IntrinsicInst.h"
  29 #include "llvm/IR/IntrinsicsARM.h"
  30 #include "llvm/Support/ErrorHandling.h"
  31 #include <array>
  32 #include <cstdint>
  33
  34 #define GET_INSTRINFO_HEADER
  35 #include "ARMGenInstrInfo.inc"
  36
  37 namespace llvm {
  38
  39 class ARMBaseRegisterInfo;
  40 class ARMSubtarget;
  41
  42 class ARMBaseInstrInfo : public ARMGenInstrInfo {
  43   const ARMSubtarget &Subtarget;
  44
  45 protected:
  46   // Can be only subclassed.
  47   explicit ARMBaseInstrInfo(const ARMSubtarget &STI);
  48
  49   void expandLoadStackGuardBase(MachineBasicBlock::iterator MI,
  50                                 unsigned LoadImmOpc, unsigned LoadOpc) const;
  51
  52   /// Build the equivalent inputs of a REG_SEQUENCE for the given \p MI
  53   /// and \p DefIdx.
  54   /// \p [out] InputRegs of the equivalent REG_SEQUENCE. Each element of
  55   /// the list is modeled as <Reg:SubReg, SubIdx>.
  56   /// E.g., REG_SEQUENCE %1:sub1, sub0, %2, sub1 would produce
  57   /// two elements:
  58   /// - %1:sub1, sub0
  59   /// - %2<:0>, sub1
  60   ///
  61   /// \returns true if it is possible to build such an input sequence
  62   /// with the pair \p MI, \p DefIdx. False otherwise.
  63   ///
  64   /// \pre MI.isRegSequenceLike().
  65   bool getRegSequenceLikeInputs(
  66       const MachineInstr &MI, unsigned DefIdx,
  67       SmallVectorImpl<RegSubRegPairAndIdx> &InputRegs) const override;
  68
  69   /// Build the equivalent inputs of a EXTRACT_SUBREG for the given \p MI
  70   /// and \p DefIdx.
  71   /// \p [out] InputReg of the equivalent EXTRACT_SUBREG.
  72   /// E.g., EXTRACT_SUBREG %1:sub1, sub0, sub1 would produce:
  73   /// - %1:sub1, sub0
  74   ///
  75   /// \returns true if it is possible to build such an input sequence
  76   /// with the pair \p MI, \p DefIdx. False otherwise.
  77   ///
  78   /// \pre MI.isExtractSubregLike().
  79   bool getExtractSubregLikeInputs(const MachineInstr &MI, unsigned DefIdx,
  80                                   RegSubRegPairAndIdx &InputReg) const override;
  81
  82   /// Build the equivalent inputs of a INSERT_SUBREG for the given \p MI
  83   /// and \p DefIdx.
  84   /// \p [out] BaseReg and \p [out] InsertedReg contain
  85   /// the equivalent inputs of INSERT_SUBREG.
  86   /// E.g., INSERT_SUBREG %0:sub0, %1:sub1, sub3 would produce:
  87   /// - BaseReg: %0:sub0
  88   /// - InsertedReg: %1:sub1, sub3
  89   ///
  90   /// \returns true if it is possible to build such an input sequence
  91   /// with the pair \p MI, \p DefIdx. False otherwise.
  92   ///
  93   /// \pre MI.isInsertSubregLike().
  94   bool
  95   getInsertSubregLikeInputs(const MachineInstr &MI, unsigned DefIdx,
  96                             RegSubRegPair &BaseReg,
  97                             RegSubRegPairAndIdx &InsertedReg) const override;
  98
  99   /// Commutes the operands in the given instruction.
 100   /// The commutable operands are specified by their indices OpIdx1 and OpIdx2.
 101   ///
 102   /// Do not call this method for a non-commutable instruction or for
 103   /// non-commutable pair of operand indices OpIdx1 and OpIdx2.
 104   /// Even though the instruction is commutable, the method may still
 105   /// fail to commute the operands, null pointer is returned in such cases.
 106   MachineInstr *commuteInstructionImpl(MachineInstr &MI, bool NewMI,
 107                                        unsigned OpIdx1,
 108                                        unsigned OpIdx2) const override;
 109   /// If the specific machine instruction is an instruction that moves/copies
 110   /// value from one register to another register return destination and source
 111   /// registers as machine operands.
 112   std::optional<DestSourcePair>
 113   isCopyInstrImpl(const MachineInstr &MI) const override;
 114
 115   /// Specialization of \ref TargetInstrInfo::describeLoadedValue, used to
 116   /// enhance debug entry value descriptions for ARM targets.
 117   std::optional<ParamLoadedValue>
 118   describeLoadedValue(const MachineInstr &MI, Register Reg) const override;
 119
 120 public:
 121   // Return whether the target has an explicit NOP encoding.
 122   bool hasNOP() const;
 123
 124   // Return the non-pre/post incrementing version of 'Opc'. Return 0
 125   // if there is not such an opcode.
 126   virtual unsigned getUnindexedOpcode(unsigned Opc) const = 0;
 127
 128   MachineInstr *convertToThreeAddress(MachineInstr &MI, LiveVariables *LV,
 129                                       LiveIntervals *LIS) const override;
 130
 131   virtual const ARMBaseRegisterInfo &getRegisterInfo() const = 0;
 132   const ARMSubtarget &getSubtarget() const { return Subtarget; }
 133
 134   ScheduleHazardRecognizer *
 135   CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI,
 136                                const ScheduleDAG *DAG) const override;
 137
 138   ScheduleHazardRecognizer *
 139   CreateTargetMIHazardRecognizer(const InstrItineraryData *II,
 140                                  const ScheduleDAGMI *DAG) const override;
 141
 142   ScheduleHazardRecognizer *
 143   CreateTargetPostRAHazardRecognizer(const InstrItineraryData *II,
 144                                      const ScheduleDAG *DAG) const override;
 145
 146   // Branch analysis.
 147   bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
 148                      MachineBasicBlock *&FBB,
 149                      SmallVectorImpl<MachineOperand> &Cond,
 150                      bool AllowModify = false) const override;
 151   unsigned removeBranch(MachineBasicBlock &MBB,
 152                         int *BytesRemoved = nullptr) const override;
 153   unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 154                         MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
 155                         const DebugLoc &DL,
 156                         int *BytesAdded = nullptr) const override;
 157
 158   bool
 159   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
 160
 161   // Predication support.
 162   bool isPredicated(const MachineInstr &MI) const override;
 163
 164   // MIR printer helper function to annotate Operands with a comment.
 165   std::string
 166   createMIROperandComment(const MachineInstr &MI, const MachineOperand &Op,
 167                           unsigned OpIdx,
 168                           const TargetRegisterInfo *TRI) const override;
 169
 170   ARMCC::CondCodes getPredicate(const MachineInstr &MI) const {
 171     int PIdx = MI.findFirstPredOperandIdx();
 172     return PIdx != -1 ? (ARMCC::CondCodes)MI.getOperand(PIdx).getImm()
 173                       : ARMCC::AL;
 174   }
 175
 176   bool PredicateInstruction(MachineInstr &MI,
 177                             ArrayRef<MachineOperand> Pred) const override;
 178
 179   bool SubsumesPredicate(ArrayRef<MachineOperand> Pred1,
 180                          ArrayRef<MachineOperand> Pred2) const override;
 181
 182   bool ClobbersPredicate(MachineInstr &MI, std::vector<MachineOperand> &Pred,
 183                          bool SkipDead) const override;
 184
 185   bool isPredicable(const MachineInstr &MI) const override;
 186
 187   // CPSR defined in instruction
 188   static bool isCPSRDefined(const MachineInstr &MI);
 189
 190   /// GetInstSize - Returns the size of the specified MachineInstr.
 191   ///
 192   unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
 193
 194   Register isLoadFromStackSlot(const MachineInstr &MI,
 195                                int &FrameIndex) const override;
 196   Register isStoreToStackSlot(const MachineInstr &MI,
 197                               int &FrameIndex) const override;
 198   Register isLoadFromStackSlotPostFE(const MachineInstr &MI,
 199                                      int &FrameIndex) const override;
 200   Register isStoreToStackSlotPostFE(const MachineInstr &MI,
 201                                     int &FrameIndex) const override;
 202
 203   void copyToCPSR(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 204                   MCRegister SrcReg, bool KillSrc,
 205                   const ARMSubtarget &Subtarget) const;
 206   void copyFromCPSR(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 207                     MCRegister DestReg, bool KillSrc,
 208                     const ARMSubtarget &Subtarget) const;
 209
 210   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 211                    const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
 212                    bool KillSrc, bool RenamableDest = false,
 213                    bool RenamableSrc = false) const override;
 214
 215   void storeRegToStackSlot(MachineBasicBlock &MBB,
 216                            MachineBasicBlock::iterator MBBI, Register SrcReg,
 217                            bool isKill, int FrameIndex,
 218                            const TargetRegisterClass *RC,
 219                            const TargetRegisterInfo *TRI,
 220                            Register VReg) const override;
 221
 222   void loadRegFromStackSlot(MachineBasicBlock &MBB,
 223                             MachineBasicBlock::iterator MBBI, Register DestReg,
 224                             int FrameIndex, const TargetRegisterClass *RC,
 225                             const TargetRegisterInfo *TRI,
 226                             Register VReg) const override;
 227
 228   bool expandPostRAPseudo(MachineInstr &MI) const override;
 229
 230   bool shouldSink(const MachineInstr &MI) const override;
 231
 232   void reMaterialize(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
 233                      Register DestReg, unsigned SubIdx,
 234                      const MachineInstr &Orig,
 235                      const TargetRegisterInfo &TRI) const override;
 236
 237   MachineInstr &
 238   duplicate(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore,
 239             const MachineInstr &Orig) const override;
 240
 241   const MachineInstrBuilder &AddDReg(MachineInstrBuilder &MIB, unsigned Reg,
 242                                      unsigned SubIdx, unsigned State,
 243                                      const TargetRegisterInfo *TRI) const;
 244
 245   bool produceSameValue(const MachineInstr &MI0, const MachineInstr &MI1,
 246                         const MachineRegisterInfo *MRI) const override;
 247
 248   /// areLoadsFromSameBasePtr - This is used by the pre-regalloc scheduler to
 249   /// determine if two loads are loading from the same base address. It should
 250   /// only return true if the base pointers are the same and the only
 251   /// differences between the two addresses is the offset. It also returns the
 252   /// offsets by reference.
 253   bool areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2, int64_t &Offset1,
 254                                int64_t &Offset2) const override;
 255
 256   /// shouldScheduleLoadsNear - This is a used by the pre-regalloc scheduler to
 257   /// determine (in conjunction with areLoadsFromSameBasePtr) if two loads
 258   /// should be scheduled togther. On some targets if two loads are loading from
 259   /// addresses in the same cache line, it's better if they are scheduled
 260   /// together. This function takes two integers that represent the load offsets
 261   /// from the common base address. It returns true if it decides it's desirable
 262   /// to schedule the two loads together. "NumLoads" is the number of loads that
 263   /// have already been scheduled after Load1.
 264   bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2,
 265                                int64_t Offset1, int64_t Offset2,
 266                                unsigned NumLoads) const override;
 267
 268   bool isSchedulingBoundary(const MachineInstr &MI,
 269                             const MachineBasicBlock *MBB,
 270                             const MachineFunction &MF) const override;
 271
 272   bool isProfitableToIfCvt(MachineBasicBlock &MBB,
 273                            unsigned NumCycles, unsigned ExtraPredCycles,
 274                            BranchProbability Probability) const override;
 275
 276   bool isProfitableToIfCvt(MachineBasicBlock &TMBB, unsigned NumT,
 277                            unsigned ExtraT, MachineBasicBlock &FMBB,
 278                            unsigned NumF, unsigned ExtraF,
 279                            BranchProbability Probability) const override;
 280
 281   bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB, unsigned NumCycles,
 282                                  BranchProbability Probability) const override {
 283     return NumCycles == 1;
 284   }
 285
 286   unsigned extraSizeToPredicateInstructions(const MachineFunction &MF,
 287                                             unsigned NumInsts) const override;
 288   unsigned predictBranchSizeForIfCvt(MachineInstr &MI) const override;
 289
 290   bool isProfitableToUnpredicate(MachineBasicBlock &TMBB,
 291                                  MachineBasicBlock &FMBB) const override;
 292
 293   /// analyzeCompare - For a comparison instruction, return the source registers
 294   /// in SrcReg and SrcReg2 if having two register operands, and the value it
 295   /// compares against in CmpValue. Return true if the comparison instruction
 296   /// can be analyzed.
 297   bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
 298                       Register &SrcReg2, int64_t &CmpMask,
 299                       int64_t &CmpValue) const override;
 300
 301   /// optimizeCompareInstr - Convert the instruction to set the zero flag so
 302   /// that we can remove a "comparison with zero"; Remove a redundant CMP
 303   /// instruction if the flags can be updated in the same way by an earlier
 304   /// instruction such as SUB.
 305   bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
 306                             Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
 307                             const MachineRegisterInfo *MRI) const override;
 308
 309   bool analyzeSelect(const MachineInstr &MI,
 310                      SmallVectorImpl<MachineOperand> &Cond, unsigned &TrueOp,
 311                      unsigned &FalseOp, bool &Optimizable) const override;
 312
 313   MachineInstr *optimizeSelect(MachineInstr &MI,
 314                                SmallPtrSetImpl<MachineInstr *> &SeenMIs,
 315                                bool) const override;
 316
 317   /// foldImmediate - 'Reg' is known to be defined by a move immediate
 318   /// instruction, try to fold the immediate into the use instruction.
 319   bool foldImmediate(MachineInstr &UseMI, MachineInstr &DefMI, Register Reg,
 320                      MachineRegisterInfo *MRI) const override;
 321
 322   unsigned getNumMicroOps(const InstrItineraryData *ItinData,
 323                           const MachineInstr &MI) const override;
 324
 325   std::optional<unsigned> getOperandLatency(const InstrItineraryData *ItinData,
 326                                             const MachineInstr &DefMI,
 327                                             unsigned DefIdx,
 328                                             const MachineInstr &UseMI,
 329                                             unsigned UseIdx) const override;
 330   std::optional<unsigned> getOperandLatency(const InstrItineraryData *ItinData,
 331                                             SDNode *DefNode, unsigned DefIdx,
 332                                             SDNode *UseNode,
 333                                             unsigned UseIdx) const override;
 334
 335   /// VFP/NEON execution domains.
 336   std::pair<uint16_t, uint16_t>
 337   getExecutionDomain(const MachineInstr &MI) const override;
 338   void setExecutionDomain(MachineInstr &MI, unsigned Domain) const override;
 339
 340   unsigned
 341   getPartialRegUpdateClearance(const MachineInstr &, unsigned,
 342                                const TargetRegisterInfo *) const override;
 343   void breakPartialRegDependency(MachineInstr &, unsigned,
 344                                  const TargetRegisterInfo *TRI) const override;
 345
 346   /// Get the number of addresses by LDM or VLDM or zero for unknown.
 347   unsigned getNumLDMAddresses(const MachineInstr &MI) const;
 348
 349   std::pair<unsigned, unsigned>
 350   decomposeMachineOperandsTargetFlags(unsigned TF) const override;
 351   ArrayRef<std::pair<unsigned, const char *>>
 352   getSerializableDirectMachineOperandTargetFlags() const override;
 353   ArrayRef<std::pair<unsigned, const char *>>
 354   getSerializableBitmaskMachineOperandTargetFlags() const override;
 355
 356   /// ARM supports the MachineOutliner.
 357   bool isFunctionSafeToOutlineFrom(MachineFunction &MF,
 358                                    bool OutlineFromLinkOnceODRs) const override;
 359   std::optional<std::unique_ptr<outliner::OutlinedFunction>>
 360   getOutliningCandidateInfo(
 361       const MachineModuleInfo &MMI,
 362       std::vector<outliner::Candidate> &RepeatedSequenceLocs,
 363       unsigned MinRepeats) const override;
 364   void mergeOutliningCandidateAttributes(
 365       Function &F, std::vector<outliner::Candidate> &Candidates) const override;
 366   outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
 367                                            MachineBasicBlock::iterator &MIT,
 368                                            unsigned Flags) const override;
 369   bool isMBBSafeToOutlineFrom(MachineBasicBlock &MBB,
 370                               unsigned &Flags) const override;
 371   void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,
 372                           const outliner::OutlinedFunction &OF) const override;
 373   MachineBasicBlock::iterator
 374   insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
 375                      MachineBasicBlock::iterator &It, MachineFunction &MF,
 376                      outliner::Candidate &C) const override;
 377
 378   /// Enable outlining by default at -Oz.
 379   bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
 380
 381   bool isUnspillableTerminatorImpl(const MachineInstr *MI) const override {
 382     return MI->getOpcode() == ARM::t2LoopEndDec ||
 383            MI->getOpcode() == ARM::t2DoLoopStartTP ||
 384            MI->getOpcode() == ARM::t2WhileLoopStartLR ||
 385            MI->getOpcode() == ARM::t2WhileLoopStartTP;
 386   }
 387
 388   /// Analyze loop L, which must be a single-basic-block loop, and if the
 389   /// conditions can be understood enough produce a PipelinerLoopInfo object.
 390   std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
 391   analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 392
 393 private:
 394   /// Returns an unused general-purpose register which can be used for
 395   /// constructing an outlined call if one exists. Returns 0 otherwise.
 396   Register findRegisterToSaveLRTo(outliner::Candidate &C) const;
 397
 398   /// Adds an instruction which saves the link register on top of the stack into
 399   /// the MachineBasicBlock \p MBB at position \p It. If \p Auth is true,
 400   /// compute and store an authentication code alongiside the link register.
 401   /// If \p CFI is true, emit CFI instructions.
 402   void saveLROnStack(MachineBasicBlock &MBB, MachineBasicBlock::iterator It,
 403                      bool CFI, bool Auth) const;
 404
 405   /// Adds an instruction which restores the link register from the top the
 406   /// stack into the MachineBasicBlock \p MBB at position \p It. If \p Auth is
 407   /// true, restore an authentication code and authenticate LR.
 408   /// If \p CFI is true, emit CFI instructions.
 409   void restoreLRFromStack(MachineBasicBlock &MBB,
 410                           MachineBasicBlock::iterator It, bool CFI,
 411                           bool Auth) const;
 412
 413   /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
 414   /// for the case when the LR is saved in the register \p Reg.
 415   void emitCFIForLRSaveToReg(MachineBasicBlock &MBB,
 416                              MachineBasicBlock::iterator It,
 417                              Register Reg) const;
 418
 419   /// Emit CFI instructions into the MachineBasicBlock \p MBB at position \p It,
 420   /// after the LR is was restored from a register.
 421   void emitCFIForLRRestoreFromReg(MachineBasicBlock &MBB,
 422                                   MachineBasicBlock::iterator It) const;
 423   /// \brief Sets the offsets on outlined instructions in \p MBB which use SP
 424   /// so that they will be valid post-outlining.
 425   ///
 426   /// \param MBB A \p MachineBasicBlock in an outlined function.
 427   void fixupPostOutline(MachineBasicBlock &MBB) const;
 428
 429   /// Returns true if the machine instruction offset can handle the stack fixup
 430   /// and updates it if requested.
 431   bool checkAndUpdateStackOffset(MachineInstr *MI, int64_t Fixup,
 432                                  bool Updt) const;
 433
 434   unsigned getInstBundleLength(const MachineInstr &MI) const;
 435
 436   std::optional<unsigned> getVLDMDefCycle(const InstrItineraryData *ItinData,
 437                                           const MCInstrDesc &DefMCID,
 438                                           unsigned DefClass, unsigned DefIdx,
 439                                           unsigned DefAlign) const;
 440   std::optional<unsigned> getLDMDefCycle(const InstrItineraryData *ItinData,
 441                                          const MCInstrDesc &DefMCID,
 442                                          unsigned DefClass, unsigned DefIdx,
 443                                          unsigned DefAlign) const;
 444   std::optional<unsigned> getVSTMUseCycle(const InstrItineraryData *ItinData,
 445                                           const MCInstrDesc &UseMCID,
 446                                           unsigned UseClass, unsigned UseIdx,
 447                                           unsigned UseAlign) const;
 448   std::optional<unsigned> getSTMUseCycle(const InstrItineraryData *ItinData,
 449                                          const MCInstrDesc &UseMCID,
 450                                          unsigned UseClass, unsigned UseIdx,
 451                                          unsigned UseAlign) const;
 452   std::optional<unsigned> getOperandLatency(const InstrItineraryData *ItinData,
 453                                             const MCInstrDesc &DefMCID,
 454                                             unsigned DefIdx, unsigned DefAlign,
 455                                             const MCInstrDesc &UseMCID,
 456                                             unsigned UseIdx,
 457                                             unsigned UseAlign) const;
 458
 459   std::optional<unsigned> getOperandLatencyImpl(
 460       const InstrItineraryData *ItinData, const MachineInstr &DefMI,
 461       unsigned DefIdx, const MCInstrDesc &DefMCID, unsigned DefAdj,
 462       const MachineOperand &DefMO, unsigned Reg, const MachineInstr &UseMI,
 463       unsigned UseIdx, const MCInstrDesc &UseMCID, unsigned UseAdj) const;
 464
 465   unsigned getPredicationCost(const MachineInstr &MI) const override;
 466
 467   unsigned getInstrLatency(const InstrItineraryData *ItinData,
 468                            const MachineInstr &MI,
 469                            unsigned *PredCost = nullptr) const override;
 470
 471   unsigned getInstrLatency(const InstrItineraryData *ItinData,
 472                            SDNode *Node) const override;
 473
 474   bool hasHighOperandLatency(const TargetSchedModel &SchedModel,
 475                              const MachineRegisterInfo *MRI,
 476                              const MachineInstr &DefMI, unsigned DefIdx,
 477                              const MachineInstr &UseMI,
 478                              unsigned UseIdx) const override;
 479   bool hasLowDefLatency(const TargetSchedModel &SchedModel,
 480                         const MachineInstr &DefMI,
 481                         unsigned DefIdx) const override;
 482
 483   /// verifyInstruction - Perform target specific instruction verification.
 484   bool verifyInstruction(const MachineInstr &MI,
 485                          StringRef &ErrInfo) const override;
 486
 487   virtual void expandLoadStackGuard(MachineBasicBlock::iterator MI) const = 0;
 488
 489   void expandMEMCPY(MachineBasicBlock::iterator) const;
 490
 491   /// Identify instructions that can be folded into a MOVCC instruction, and
 492   /// return the defining instruction.
 493   MachineInstr *canFoldIntoMOVCC(Register Reg, const MachineRegisterInfo &MRI,
 494                                  const TargetInstrInfo *TII) const;
 495
 496   bool isReallyTriviallyReMaterializable(const MachineInstr &MI) const override;
 497
 498 private:
 499   /// Modeling special VFP / NEON fp MLA / MLS hazards.
 500
 501   /// MLxEntryMap - Map fp MLA / MLS to the corresponding entry in the internal
 502   /// MLx table.
 503   DenseMap<unsigned, unsigned> MLxEntryMap;
 504
 505   /// MLxHazardOpcodes - Set of add / sub and multiply opcodes that would cause
 506   /// stalls when scheduled together with fp MLA / MLS opcodes.
 507   SmallSet<unsigned, 16> MLxHazardOpcodes;
 508
 509 public:
 510   /// isFpMLxInstruction - Return true if the specified opcode is a fp MLA / MLS
 511   /// instruction.
 512   bool isFpMLxInstruction(unsigned Opcode) const {
 513     return MLxEntryMap.count(Opcode);
 514   }
 515
 516   /// isFpMLxInstruction - This version also returns the multiply opcode and the
 517   /// addition / subtraction opcode to expand to. Return true for 'HasLane' for
 518   /// the MLX instructions with an extra lane operand.
 519   bool isFpMLxInstruction(unsigned Opcode, unsigned &MulOpc,
 520                           unsigned &AddSubOpc, bool &NegAcc,
 521                           bool &HasLane) const;
 522
 523   /// canCauseFpMLxStall - Return true if an instruction of the specified opcode
 524   /// will cause stalls when scheduled after (within 4-cycle window) a fp
 525   /// MLA / MLS instruction.
 526   bool canCauseFpMLxStall(unsigned Opcode) const {
 527     return MLxHazardOpcodes.count(Opcode);
 528   }
 529
 530   /// Returns true if the instruction has a shift by immediate that can be
 531   /// executed in one cycle less.
 532   bool isSwiftFastImmShift(const MachineInstr *MI) const;
 533
 534   /// Returns predicate register associated with the given frame instruction.
 535   unsigned getFramePred(const MachineInstr &MI) const {
 536     assert(isFrameInstr(MI));
 537     // Operands of ADJCALLSTACKDOWN/ADJCALLSTACKUP:
 538     // - argument declared in the pattern:
 539     // 0 - frame size
 540     // 1 - arg of CALLSEQ_START/CALLSEQ_END
 541     // 2 - predicate code (like ARMCC::AL)
 542     // - added by predOps:
 543     // 3 - predicate reg
 544     return MI.getOperand(3).getReg();
 545   }
 546
 547   std::optional<RegImmPair> isAddImmediate(const MachineInstr &MI,
 548                                            Register Reg) const override;
 549 };
 550
 551 /// Get the operands corresponding to the given \p Pred value. By default, the
 552 /// predicate register is assumed to be 0 (no register), but you can pass in a
 553 /// \p PredReg if that is not the case.
 554 static inline std::array<MachineOperand, 2> predOps(ARMCC::CondCodes Pred,
 555                                                     unsigned PredReg = 0) {
 556   return {{MachineOperand::CreateImm(static_cast<int64_t>(Pred)),
 557            MachineOperand::CreateReg(PredReg, false)}};
 558 }
 559
 560 /// Get the operand corresponding to the conditional code result. By default,
 561 /// this is 0 (no register).
 562 static inline MachineOperand condCodeOp(unsigned CCReg = 0) {
 563   return MachineOperand::CreateReg(CCReg, false);
 564 }
 565
 566 /// Get the operand corresponding to the conditional code result for Thumb1.
 567 /// This operand will always refer to CPSR and it will have the Define flag set.
 568 /// You can optionally set the Dead flag by means of \p isDead.
 569 static inline MachineOperand t1CondCodeOp(bool isDead = false) {
 570   return MachineOperand::CreateReg(ARM::CPSR,
 571                                    /*Define*/ true, /*Implicit*/ false,
 572                                    /*Kill*/ false, isDead);
 573 }
 574
 575 static inline
 576 bool isUncondBranchOpcode(int Opc) {
 577   return Opc == ARM::B || Opc == ARM::tB || Opc == ARM::t2B;
 578 }
 579
 580 // This table shows the VPT instruction variants, i.e. the different
 581 // mask field encodings, see also B5.6. Predication/conditional execution in
 582 // the ArmARM.
 583 static inline bool isVPTOpcode(int Opc) {
 584   return Opc == ARM::MVE_VPTv16i8 || Opc == ARM::MVE_VPTv16u8 ||
 585          Opc == ARM::MVE_VPTv16s8 || Opc == ARM::MVE_VPTv8i16 ||
 586          Opc == ARM::MVE_VPTv8u16 || Opc == ARM::MVE_VPTv8s16 ||
 587          Opc == ARM::MVE_VPTv4i32 || Opc == ARM::MVE_VPTv4u32 ||
 588          Opc == ARM::MVE_VPTv4s32 || Opc == ARM::MVE_VPTv4f32 ||
 589          Opc == ARM::MVE_VPTv8f16 || Opc == ARM::MVE_VPTv16i8r ||
 590          Opc == ARM::MVE_VPTv16u8r || Opc == ARM::MVE_VPTv16s8r ||
 591          Opc == ARM::MVE_VPTv8i16r || Opc == ARM::MVE_VPTv8u16r ||
 592          Opc == ARM::MVE_VPTv8s16r || Opc == ARM::MVE_VPTv4i32r ||
 593          Opc == ARM::MVE_VPTv4u32r || Opc == ARM::MVE_VPTv4s32r ||
 594          Opc == ARM::MVE_VPTv4f32r || Opc == ARM::MVE_VPTv8f16r ||
 595          Opc == ARM::MVE_VPST;
 596 }
 597
 598 static inline
 599 unsigned VCMPOpcodeToVPT(unsigned Opcode) {
 600   switch (Opcode) {
 601   default:
 602     return 0;
 603   case ARM::MVE_VCMPf32:
 604     return ARM::MVE_VPTv4f32;
 605   case ARM::MVE_VCMPf16:
 606     return ARM::MVE_VPTv8f16;
 607   case ARM::MVE_VCMPi8:
 608     return ARM::MVE_VPTv16i8;
 609   case ARM::MVE_VCMPi16:
 610     return ARM::MVE_VPTv8i16;
 611   case ARM::MVE_VCMPi32:
 612     return ARM::MVE_VPTv4i32;
 613   case ARM::MVE_VCMPu8:
 614     return ARM::MVE_VPTv16u8;
 615   case ARM::MVE_VCMPu16:
 616     return ARM::MVE_VPTv8u16;
 617   case ARM::MVE_VCMPu32:
 618     return ARM::MVE_VPTv4u32;
 619   case ARM::MVE_VCMPs8:
 620     return ARM::MVE_VPTv16s8;
 621   case ARM::MVE_VCMPs16:
 622     return ARM::MVE_VPTv8s16;
 623   case ARM::MVE_VCMPs32:
 624     return ARM::MVE_VPTv4s32;
 625
 626   case ARM::MVE_VCMPf32r:
 627     return ARM::MVE_VPTv4f32r;
 628   case ARM::MVE_VCMPf16r:
 629     return ARM::MVE_VPTv8f16r;
 630   case ARM::MVE_VCMPi8r:
 631     return ARM::MVE_VPTv16i8r;
 632   case ARM::MVE_VCMPi16r:
 633     return ARM::MVE_VPTv8i16r;
 634   case ARM::MVE_VCMPi32r:
 635     return ARM::MVE_VPTv4i32r;
 636   case ARM::MVE_VCMPu8r:
 637     return ARM::MVE_VPTv16u8r;
 638   case ARM::MVE_VCMPu16r:
 639     return ARM::MVE_VPTv8u16r;
 640   case ARM::MVE_VCMPu32r:
 641     return ARM::MVE_VPTv4u32r;
 642   case ARM::MVE_VCMPs8r:
 643     return ARM::MVE_VPTv16s8r;
 644   case ARM::MVE_VCMPs16r:
 645     return ARM::MVE_VPTv8s16r;
 646   case ARM::MVE_VCMPs32r:
 647     return ARM::MVE_VPTv4s32r;
 648   }
 649 }
 650
 651 static inline
 652 bool isCondBranchOpcode(int Opc) {
 653   return Opc == ARM::Bcc || Opc == ARM::tBcc || Opc == ARM::t2Bcc;
 654 }
 655
 656 static inline bool isJumpTableBranchOpcode(int Opc) {
 657   return Opc == ARM::BR_JTr || Opc == ARM::BR_JTm_i12 ||
 658          Opc == ARM::BR_JTm_rs || Opc == ARM::BR_JTadd || Opc == ARM::tBR_JTr ||
 659          Opc == ARM::t2BR_JT;
 660 }
 661
 662 static inline
 663 bool isIndirectBranchOpcode(int Opc) {
 664   return Opc == ARM::BX || Opc == ARM::MOVPCRX || Opc == ARM::tBRIND;
 665 }
 666
 667 static inline bool isIndirectCall(const MachineInstr &MI) {
 668   int Opc = MI.getOpcode();
 669   switch (Opc) {
 670     // indirect calls:
 671   case ARM::BLX:
 672   case ARM::BLX_noip:
 673   case ARM::BLX_pred:
 674   case ARM::BLX_pred_noip:
 675   case ARM::BX_CALL:
 676   case ARM::BMOVPCRX_CALL:
 677   case ARM::TCRETURNri:
 678   case ARM::TCRETURNrinotr12:
 679   case ARM::TAILJMPr:
 680   case ARM::TAILJMPr4:
 681   case ARM::tBLXr:
 682   case ARM::tBLXr_noip:
 683   case ARM::tBLXNSr:
 684   case ARM::tBLXNS_CALL:
 685   case ARM::tBX_CALL:
 686   case ARM::tTAILJMPr:
 687     assert(MI.isCall(MachineInstr::IgnoreBundle));
 688     return true;
 689     // direct calls:
 690   case ARM::BL:
 691   case ARM::BL_pred:
 692   case ARM::BMOVPCB_CALL:
 693   case ARM::BL_PUSHLR:
 694   case ARM::BLXi:
 695   case ARM::TCRETURNdi:
 696   case ARM::TAILJMPd:
 697   case ARM::SVC:
 698   case ARM::HVC:
 699   case ARM::TPsoft:
 700   case ARM::tTAILJMPd:
 701   case ARM::t2SMC:
 702   case ARM::t2HVC:
 703   case ARM::tBL:
 704   case ARM::tBLXi:
 705   case ARM::tBL_PUSHLR:
 706   case ARM::tTAILJMPdND:
 707   case ARM::tSVC:
 708   case ARM::tTPsoft:
 709     assert(MI.isCall(MachineInstr::IgnoreBundle));
 710     return false;
 711   }
 712   assert(!MI.isCall(MachineInstr::IgnoreBundle));
 713   return false;
 714 }
 715
 716 static inline bool isIndirectControlFlowNotComingBack(const MachineInstr &MI) {
 717   int opc = MI.getOpcode();
 718   return MI.isReturn() || isIndirectBranchOpcode(MI.getOpcode()) ||
 719          isJumpTableBranchOpcode(opc);
 720 }
 721
 722 static inline bool isSpeculationBarrierEndBBOpcode(int Opc) {
 723   return Opc == ARM::SpeculationBarrierISBDSBEndBB ||
 724          Opc == ARM::SpeculationBarrierSBEndBB ||
 725          Opc == ARM::t2SpeculationBarrierISBDSBEndBB ||
 726          Opc == ARM::t2SpeculationBarrierSBEndBB;
 727 }
 728
 729 static inline bool isPopOpcode(int Opc) {
 730   return Opc == ARM::tPOP_RET || Opc == ARM::LDMIA_RET ||
 731          Opc == ARM::t2LDMIA_RET || Opc == ARM::tPOP || Opc == ARM::LDMIA_UPD ||
 732          Opc == ARM::t2LDMIA_UPD || Opc == ARM::VLDMDIA_UPD;
 733 }
 734
 735 static inline bool isPushOpcode(int Opc) {
 736   return Opc == ARM::tPUSH || Opc == ARM::t2STMDB_UPD ||
 737          Opc == ARM::STMDB_UPD || Opc == ARM::VSTMDDB_UPD;
 738 }
 739
 740 static inline bool isSubImmOpcode(int Opc) {
 741   return Opc == ARM::SUBri ||
 742          Opc == ARM::tSUBi3 || Opc == ARM::tSUBi8 ||
 743          Opc == ARM::tSUBSi3 || Opc == ARM::tSUBSi8 ||
 744          Opc == ARM::t2SUBri || Opc == ARM::t2SUBri12 || Opc == ARM::t2SUBSri;
 745 }
 746
 747 static inline bool isMovRegOpcode(int Opc) {
 748   return Opc == ARM::MOVr || Opc == ARM::tMOVr || Opc == ARM::t2MOVr;
 749 }
 750 /// isValidCoprocessorNumber - decide whether an explicit coprocessor
 751 /// number is legal in generic instructions like CDP. The answer can
 752 /// vary with the subtarget.
 753 static inline bool isValidCoprocessorNumber(unsigned Num,
 754                                             const FeatureBitset& featureBits) {
 755   // In Armv7 and Armv8-M CP10 and CP11 clash with VFP/NEON, however, the
 756   // coprocessor is still valid for CDP/MCR/MRC and friends. Allowing it is
 757   // useful for code which is shared with older architectures which do not know
 758   // the new VFP/NEON mnemonics.
 759
 760   // Armv8-A disallows everything *other* than 111x (CP14 and CP15).
 761   if (featureBits[ARM::HasV8Ops] && (Num & 0xE) != 0xE)
 762     return false;
 763
 764   // Armv8.1-M disallows 100x (CP8,CP9) and 111x (CP14,CP15)
 765   // which clash with MVE.
 766   if (featureBits[ARM::HasV8_1MMainlineOps] &&
 767       ((Num & 0xE) == 0x8 || (Num & 0xE) == 0xE))
 768     return false;
 769
 770   return true;
 771 }
 772
 773 static inline bool isSEHInstruction(const MachineInstr &MI) {
 774   unsigned Opc = MI.getOpcode();
 775   switch (Opc) {
 776   case ARM::SEH_StackAlloc:
 777   case ARM::SEH_SaveRegs:
 778   case ARM::SEH_SaveRegs_Ret:
 779   case ARM::SEH_SaveSP:
 780   case ARM::SEH_SaveFRegs:
 781   case ARM::SEH_SaveLR:
 782   case ARM::SEH_Nop:
 783   case ARM::SEH_Nop_Ret:
 784   case ARM::SEH_PrologEnd:
 785   case ARM::SEH_EpilogStart:
 786   case ARM::SEH_EpilogEnd:
 787     return true;
 788   default:
 789     return false;
 790   }
 791 }
 792
 793 /// getInstrPredicate - If instruction is predicated, returns its predicate
 794 /// condition, otherwise returns AL. It also returns the condition code
 795 /// register by reference.
 796 ARMCC::CondCodes getInstrPredicate(const MachineInstr &MI, Register &PredReg);
 797
 798 unsigned getMatchingCondBranchOpcode(unsigned Opc);
 799
 800 /// Map pseudo instructions that imply an 'S' bit onto real opcodes. Whether
 801 /// the instruction is encoded with an 'S' bit is determined by the optional
 802 /// CPSR def operand.
 803 unsigned convertAddSubFlagsOpcode(unsigned OldOpc);
 804
 805 /// emitARMRegPlusImmediate / emitT2RegPlusImmediate - Emits a series of
 806 /// instructions to materializea destreg = basereg + immediate in ARM / Thumb2
 807 /// code.
 808 void emitARMRegPlusImmediate(MachineBasicBlock &MBB,
 809                              MachineBasicBlock::iterator &MBBI,
 810                              const DebugLoc &dl, Register DestReg,
 811                              Register BaseReg, int NumBytes,
 812                              ARMCC::CondCodes Pred, Register PredReg,
 813                              const ARMBaseInstrInfo &TII, unsigned MIFlags = 0);
 814
 815 void emitT2RegPlusImmediate(MachineBasicBlock &MBB,
 816                             MachineBasicBlock::iterator &MBBI,
 817                             const DebugLoc &dl, Register DestReg,
 818                             Register BaseReg, int NumBytes,
 819                             ARMCC::CondCodes Pred, Register PredReg,
 820                             const ARMBaseInstrInfo &TII, unsigned MIFlags = 0);
 821 void emitThumbRegPlusImmediate(MachineBasicBlock &MBB,
 822                                MachineBasicBlock::iterator &MBBI,
 823                                const DebugLoc &dl, Register DestReg,
 824                                Register BaseReg, int NumBytes,
 825                                const TargetInstrInfo &TII,
 826                                const ARMBaseRegisterInfo &MRI,
 827                                unsigned MIFlags = 0);
 828
 829 /// Tries to add registers to the reglist of a given base-updating
 830 /// push/pop instruction to adjust the stack by an additional
 831 /// NumBytes. This can save a few bytes per function in code-size, but
 832 /// obviously generates more memory traffic. As such, it only takes
 833 /// effect in functions being optimised for size.
 834 bool tryFoldSPUpdateIntoPushPop(const ARMSubtarget &Subtarget,
 835                                 MachineFunction &MF, MachineInstr *MI,
 836                                 unsigned NumBytes);
 837
 838 /// rewriteARMFrameIndex / rewriteT2FrameIndex -
 839 /// Rewrite MI to access 'Offset' bytes from the FP. Return false if the
 840 /// offset could not be handled directly in MI, and return the left-over
 841 /// portion by reference.
 842 bool rewriteARMFrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
 843                           Register FrameReg, int &Offset,
 844                           const ARMBaseInstrInfo &TII);
 845
 846 bool rewriteT2FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
 847                          Register FrameReg, int &Offset,
 848                          const ARMBaseInstrInfo &TII,
 849                          const TargetRegisterInfo *TRI);
 850
 851 /// Return true if Reg is defd between From and To
 852 bool registerDefinedBetween(unsigned Reg, MachineBasicBlock::iterator From,
 853                             MachineBasicBlock::iterator To,
 854                             const TargetRegisterInfo *TRI);
 855
 856 /// Search backwards from a tBcc to find a tCMPi8 against 0, meaning
 857 /// we can convert them to a tCBZ or tCBNZ. Return nullptr if not found.
 858 MachineInstr *findCMPToFoldIntoCBZ(MachineInstr *Br,
 859                                    const TargetRegisterInfo *TRI);
 860
 861 void addUnpredicatedMveVpredNOp(MachineInstrBuilder &MIB);
 862 void addUnpredicatedMveVpredROp(MachineInstrBuilder &MIB, Register DestReg);
 863
 864 void addPredicatedMveVpredNOp(MachineInstrBuilder &MIB, unsigned Cond);
 865 void addPredicatedMveVpredROp(MachineInstrBuilder &MIB, unsigned Cond,
 866                               unsigned Inactive);
 867
 868 /// Returns the number of instructions required to materialize the given
 869 /// constant in a register, or 3 if a literal pool load is needed.
 870 /// If ForCodesize is specified, an approximate cost in bytes is returned.
 871 unsigned ConstantMaterializationCost(unsigned Val,
 872                                      const ARMSubtarget *Subtarget,
 873                                      bool ForCodesize = false);
 874
 875 /// Returns true if Val1 has a lower Constant Materialization Cost than Val2.
 876 /// Uses the cost from ConstantMaterializationCost, first with ForCodesize as
 877 /// specified. If the scores are equal, return the comparison for !ForCodesize.
 878 bool HasLowerConstantMaterializationCost(unsigned Val1, unsigned Val2,
 879                                          const ARMSubtarget *Subtarget,
 880                                          bool ForCodesize = false);
 881
 882 // Return the immediate if this is ADDri or SUBri, scaled as appropriate.
 883 // Returns 0 for unknown instructions.
 884 inline int getAddSubImmediate(MachineInstr &MI) {
 885   int Scale = 1;
 886   unsigned ImmOp;
 887   switch (MI.getOpcode()) {
 888   case ARM::t2ADDri:
 889     ImmOp = 2;
 890     break;
 891   case ARM::t2SUBri:
 892   case ARM::t2SUBri12:
 893     ImmOp = 2;
 894     Scale = -1;
 895     break;
 896   case ARM::tSUBi3:
 897   case ARM::tSUBi8:
 898     ImmOp = 3;
 899     Scale = -1;
 900     break;
 901   default:
 902     return 0;
 903   }
 904   return Scale * MI.getOperand(ImmOp).getImm();
 905 }
 906
 907 // Given a memory access Opcode, check that the give Imm would be a valid Offset
 908 // for this instruction using its addressing mode.
 909 inline bool isLegalAddressImm(unsigned Opcode, int Imm,
 910                               const TargetInstrInfo *TII) {
 911   const MCInstrDesc &Desc = TII->get(Opcode);
 912   unsigned AddrMode = (Desc.TSFlags & ARMII::AddrModeMask);
 913   switch (AddrMode) {
 914   case ARMII::AddrModeT2_i7:
 915     return std::abs(Imm) < ((1 << 7) * 1);
 916   case ARMII::AddrModeT2_i7s2:
 917     return std::abs(Imm) < ((1 << 7) * 2) && Imm % 2 == 0;
 918   case ARMII::AddrModeT2_i7s4:
 919     return std::abs(Imm) < ((1 << 7) * 4) && Imm % 4 == 0;
 920   case ARMII::AddrModeT2_i8:
 921     return std::abs(Imm) < ((1 << 8) * 1);
 922   case ARMII::AddrModeT2_i8pos:
 923     return Imm >= 0 && Imm < ((1 << 8) * 1);
 924   case ARMII::AddrModeT2_i8neg:
 925     return Imm < 0 && -Imm < ((1 << 8) * 1);
 926   case ARMII::AddrModeT2_i8s4:
 927     return std::abs(Imm) < ((1 << 8) * 4) && Imm % 4 == 0;
 928   case ARMII::AddrModeT2_i12:
 929     return Imm >= 0 && Imm < ((1 << 12) * 1);
 930   case ARMII::AddrMode2:
 931     return std::abs(Imm) < ((1 << 12) * 1);
 932   default:
 933     llvm_unreachable("Unhandled Addressing mode");
 934   }
 935 }
 936
 937 // Return true if the given intrinsic is a gather
 938 inline bool isGather(IntrinsicInst *IntInst) {
 939   if (IntInst == nullptr)
 940     return false;
 941   unsigned IntrinsicID = IntInst->getIntrinsicID();
 942   return (IntrinsicID == Intrinsic::masked_gather ||
 943           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base ||
 944           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base_predicated ||
 945           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base_wb ||
 946           IntrinsicID == Intrinsic::arm_mve_vldr_gather_base_wb_predicated ||
 947           IntrinsicID == Intrinsic::arm_mve_vldr_gather_offset ||
 948           IntrinsicID == Intrinsic::arm_mve_vldr_gather_offset_predicated);
 949 }
 950
 951 // Return true if the given intrinsic is a scatter
 952 inline bool isScatter(IntrinsicInst *IntInst) {
 953   if (IntInst == nullptr)
 954     return false;
 955   unsigned IntrinsicID = IntInst->getIntrinsicID();
 956   return (IntrinsicID == Intrinsic::masked_scatter ||
 957           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base ||
 958           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base_predicated ||
 959           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base_wb ||
 960           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_base_wb_predicated ||
 961           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_offset ||
 962           IntrinsicID == Intrinsic::arm_mve_vstr_scatter_offset_predicated);
 963 }
 964
 965 // Return true if the given intrinsic is a gather or scatter
 966 inline bool isGatherScatter(IntrinsicInst *IntInst) {
 967   if (IntInst == nullptr)
 968     return false;
 969   return isGather(IntInst) || isScatter(IntInst);
 970 }
 971
 972 unsigned getBLXOpcode(const MachineFunction &MF);
 973 unsigned gettBLXrOpcode(const MachineFunction &MF);
 974 unsigned getBLXpredOpcode(const MachineFunction &MF);
 975
 976 inline bool isMVEVectorInstruction(const MachineInstr *MI) {
 977   // This attempts to remove non-mve instructions (scalar shifts), which
 978   // are just DPU CX instruction.
 979   switch (MI->getOpcode()) {
 980   case ARM::MVE_SQSHL:
 981   case ARM::MVE_SRSHR:
 982   case ARM::MVE_UQSHL:
 983   case ARM::MVE_URSHR:
 984   case ARM::MVE_SQRSHR:
 985   case ARM::MVE_UQRSHL:
 986   case ARM::MVE_ASRLr:
 987   case ARM::MVE_ASRLi:
 988   case ARM::MVE_LSLLr:
 989   case ARM::MVE_LSLLi:
 990   case ARM::MVE_LSRL:
 991   case ARM::MVE_SQRSHRL:
 992   case ARM::MVE_SQSHLL:
 993   case ARM::MVE_SRSHRL:
 994   case ARM::MVE_UQRSHLL:
 995   case ARM::MVE_UQSHLL:
 996   case ARM::MVE_URSHRL:
 997     return false;
 998   }
 999   const MCInstrDesc &MCID = MI->getDesc();
1000   uint64_t Flags = MCID.TSFlags;
1001   return (Flags & ARMII::DomainMask) == ARMII::DomainMVE;
1002 }
1003
1004 } // end namespace llvm
1005
1006 #endif // LLVM_LIB_TARGET_ARM_ARMBASEINSTRINFO_H