llvm/lib/Target/AArch64/AArch64InstrInfo.h

   1 //===- AArch64InstrInfo.h - AArch64 Instruction Information -----*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file contains the AArch64 implementation of the TargetInstrInfo class.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H
  14 #define LLVM_LIB_TARGET_AARCH64_AARCH64INSTRINFO_H
  15
  16 #include "AArch64.h"
  17 #include "AArch64RegisterInfo.h"
  18 #include "llvm/CodeGen/TargetInstrInfo.h"
  19 #include "llvm/Support/TypeSize.h"
  20 #include <optional>
  21
  22 #define GET_INSTRINFO_HEADER
  23 #include "AArch64GenInstrInfo.inc"
  24
  25 namespace llvm {
  26
  27 class AArch64Subtarget;
  28
  29 static const MachineMemOperand::Flags MOSuppressPair =
  30     MachineMemOperand::MOTargetFlag1;
  31 static const MachineMemOperand::Flags MOStridedAccess =
  32     MachineMemOperand::MOTargetFlag2;
  33
  34 #define FALKOR_STRIDED_ACCESS_MD "falkor.strided.access"
  35
  36 // AArch64 MachineCombiner patterns
  37 enum AArch64MachineCombinerPattern : unsigned {
  38   // These are patterns used to reduce the length of dependence chain.
  39   SUBADD_OP1 = MachineCombinerPattern::TARGET_PATTERN_START,
  40   SUBADD_OP2,
  41
  42   // These are multiply-add patterns matched by the AArch64 machine combiner.
  43   MULADDW_OP1,
  44   MULADDW_OP2,
  45   MULSUBW_OP1,
  46   MULSUBW_OP2,
  47   MULADDWI_OP1,
  48   MULSUBWI_OP1,
  49   MULADDX_OP1,
  50   MULADDX_OP2,
  51   MULSUBX_OP1,
  52   MULSUBX_OP2,
  53   MULADDXI_OP1,
  54   MULSUBXI_OP1,
  55   // NEON integers vectors
  56   MULADDv8i8_OP1,
  57   MULADDv8i8_OP2,
  58   MULADDv16i8_OP1,
  59   MULADDv16i8_OP2,
  60   MULADDv4i16_OP1,
  61   MULADDv4i16_OP2,
  62   MULADDv8i16_OP1,
  63   MULADDv8i16_OP2,
  64   MULADDv2i32_OP1,
  65   MULADDv2i32_OP2,
  66   MULADDv4i32_OP1,
  67   MULADDv4i32_OP2,
  68
  69   MULSUBv8i8_OP1,
  70   MULSUBv8i8_OP2,
  71   MULSUBv16i8_OP1,
  72   MULSUBv16i8_OP2,
  73   MULSUBv4i16_OP1,
  74   MULSUBv4i16_OP2,
  75   MULSUBv8i16_OP1,
  76   MULSUBv8i16_OP2,
  77   MULSUBv2i32_OP1,
  78   MULSUBv2i32_OP2,
  79   MULSUBv4i32_OP1,
  80   MULSUBv4i32_OP2,
  81
  82   MULADDv4i16_indexed_OP1,
  83   MULADDv4i16_indexed_OP2,
  84   MULADDv8i16_indexed_OP1,
  85   MULADDv8i16_indexed_OP2,
  86   MULADDv2i32_indexed_OP1,
  87   MULADDv2i32_indexed_OP2,
  88   MULADDv4i32_indexed_OP1,
  89   MULADDv4i32_indexed_OP2,
  90
  91   MULSUBv4i16_indexed_OP1,
  92   MULSUBv4i16_indexed_OP2,
  93   MULSUBv8i16_indexed_OP1,
  94   MULSUBv8i16_indexed_OP2,
  95   MULSUBv2i32_indexed_OP1,
  96   MULSUBv2i32_indexed_OP2,
  97   MULSUBv4i32_indexed_OP1,
  98   MULSUBv4i32_indexed_OP2,
  99
 100   // Floating Point
 101   FMULADDH_OP1,
 102   FMULADDH_OP2,
 103   FMULSUBH_OP1,
 104   FMULSUBH_OP2,
 105   FMULADDS_OP1,
 106   FMULADDS_OP2,
 107   FMULSUBS_OP1,
 108   FMULSUBS_OP2,
 109   FMULADDD_OP1,
 110   FMULADDD_OP2,
 111   FMULSUBD_OP1,
 112   FMULSUBD_OP2,
 113   FNMULSUBH_OP1,
 114   FNMULSUBS_OP1,
 115   FNMULSUBD_OP1,
 116   FMLAv1i32_indexed_OP1,
 117   FMLAv1i32_indexed_OP2,
 118   FMLAv1i64_indexed_OP1,
 119   FMLAv1i64_indexed_OP2,
 120   FMLAv4f16_OP1,
 121   FMLAv4f16_OP2,
 122   FMLAv8f16_OP1,
 123   FMLAv8f16_OP2,
 124   FMLAv2f32_OP2,
 125   FMLAv2f32_OP1,
 126   FMLAv2f64_OP1,
 127   FMLAv2f64_OP2,
 128   FMLAv4i16_indexed_OP1,
 129   FMLAv4i16_indexed_OP2,
 130   FMLAv8i16_indexed_OP1,
 131   FMLAv8i16_indexed_OP2,
 132   FMLAv2i32_indexed_OP1,
 133   FMLAv2i32_indexed_OP2,
 134   FMLAv2i64_indexed_OP1,
 135   FMLAv2i64_indexed_OP2,
 136   FMLAv4f32_OP1,
 137   FMLAv4f32_OP2,
 138   FMLAv4i32_indexed_OP1,
 139   FMLAv4i32_indexed_OP2,
 140   FMLSv1i32_indexed_OP2,
 141   FMLSv1i64_indexed_OP2,
 142   FMLSv4f16_OP1,
 143   FMLSv4f16_OP2,
 144   FMLSv8f16_OP1,
 145   FMLSv8f16_OP2,
 146   FMLSv2f32_OP1,
 147   FMLSv2f32_OP2,
 148   FMLSv2f64_OP1,
 149   FMLSv2f64_OP2,
 150   FMLSv4i16_indexed_OP1,
 151   FMLSv4i16_indexed_OP2,
 152   FMLSv8i16_indexed_OP1,
 153   FMLSv8i16_indexed_OP2,
 154   FMLSv2i32_indexed_OP1,
 155   FMLSv2i32_indexed_OP2,
 156   FMLSv2i64_indexed_OP1,
 157   FMLSv2i64_indexed_OP2,
 158   FMLSv4f32_OP1,
 159   FMLSv4f32_OP2,
 160   FMLSv4i32_indexed_OP1,
 161   FMLSv4i32_indexed_OP2,
 162
 163   FMULv2i32_indexed_OP1,
 164   FMULv2i32_indexed_OP2,
 165   FMULv2i64_indexed_OP1,
 166   FMULv2i64_indexed_OP2,
 167   FMULv4i16_indexed_OP1,
 168   FMULv4i16_indexed_OP2,
 169   FMULv4i32_indexed_OP1,
 170   FMULv4i32_indexed_OP2,
 171   FMULv8i16_indexed_OP1,
 172   FMULv8i16_indexed_OP2,
 173
 174   FNMADD,
 175 };
 176 class AArch64InstrInfo final : public AArch64GenInstrInfo {
 177   const AArch64RegisterInfo RI;
 178   const AArch64Subtarget &Subtarget;
 179
 180 public:
 181   explicit AArch64InstrInfo(const AArch64Subtarget &STI);
 182
 183   /// getRegisterInfo - TargetInstrInfo is a superset of MRegister info.  As
 184   /// such, whenever a client has an instance of instruction info, it should
 185   /// always be able to get register info as well (through this method).
 186   const AArch64RegisterInfo &getRegisterInfo() const { return RI; }
 187
 188   unsigned getInstSizeInBytes(const MachineInstr &MI) const override;
 189
 190   bool isAsCheapAsAMove(const MachineInstr &MI) const override;
 191
 192   bool isCoalescableExtInstr(const MachineInstr &MI, Register &SrcReg,
 193                              Register &DstReg, unsigned &SubIdx) const override;
 194
 195   bool
 196   areMemAccessesTriviallyDisjoint(const MachineInstr &MIa,
 197                                   const MachineInstr &MIb) const override;
 198
 199   Register isLoadFromStackSlot(const MachineInstr &MI,
 200                                int &FrameIndex) const override;
 201   Register isStoreToStackSlot(const MachineInstr &MI,
 202                               int &FrameIndex) const override;
 203
 204   /// Does this instruction set its full destination register to zero?
 205   static bool isGPRZero(const MachineInstr &MI);
 206
 207   /// Does this instruction rename a GPR without modifying bits?
 208   static bool isGPRCopy(const MachineInstr &MI);
 209
 210   /// Does this instruction rename an FPR without modifying bits?
 211   static bool isFPRCopy(const MachineInstr &MI);
 212
 213   /// Return true if pairing the given load or store is hinted to be
 214   /// unprofitable.
 215   static bool isLdStPairSuppressed(const MachineInstr &MI);
 216
 217   /// Return true if the given load or store is a strided memory access.
 218   static bool isStridedAccess(const MachineInstr &MI);
 219
 220   /// Return true if it has an unscaled load/store offset.
 221   static bool hasUnscaledLdStOffset(unsigned Opc);
 222   static bool hasUnscaledLdStOffset(MachineInstr &MI) {
 223     return hasUnscaledLdStOffset(MI.getOpcode());
 224   }
 225
 226   /// Returns the unscaled load/store for the scaled load/store opcode,
 227   /// if there is a corresponding unscaled variant available.
 228   static std::optional<unsigned> getUnscaledLdSt(unsigned Opc);
 229
 230   /// Scaling factor for (scaled or unscaled) load or store.
 231   static int getMemScale(unsigned Opc);
 232   static int getMemScale(const MachineInstr &MI) {
 233     return getMemScale(MI.getOpcode());
 234   }
 235
 236   /// Returns whether the instruction is a pre-indexed load.
 237   static bool isPreLd(const MachineInstr &MI);
 238
 239   /// Returns whether the instruction is a pre-indexed store.
 240   static bool isPreSt(const MachineInstr &MI);
 241
 242   /// Returns whether the instruction is a pre-indexed load/store.
 243   static bool isPreLdSt(const MachineInstr &MI);
 244
 245   /// Returns whether the instruction is a paired load/store.
 246   static bool isPairedLdSt(const MachineInstr &MI);
 247
 248   /// Returns the base register operator of a load/store.
 249   static const MachineOperand &getLdStBaseOp(const MachineInstr &MI);
 250
 251   /// Returns the immediate offset operator of a load/store.
 252   static const MachineOperand &getLdStOffsetOp(const MachineInstr &MI);
 253
 254   /// Returns whether the physical register is FP or NEON.
 255   static bool isFpOrNEON(Register Reg);
 256
 257   /// Returns the shift amount operator of a load/store.
 258   static const MachineOperand &getLdStAmountOp(const MachineInstr &MI);
 259
 260   /// Returns whether the instruction is FP or NEON.
 261   static bool isFpOrNEON(const MachineInstr &MI);
 262
 263   /// Returns whether the instruction is in H form (16 bit operands)
 264   static bool isHForm(const MachineInstr &MI);
 265
 266   /// Returns whether the instruction is in Q form (128 bit operands)
 267   static bool isQForm(const MachineInstr &MI);
 268
 269   /// Returns whether the instruction can be compatible with non-zero BTYPE.
 270   static bool hasBTISemantics(const MachineInstr &MI);
 271
 272   /// Returns the index for the immediate for a given instruction.
 273   static unsigned getLoadStoreImmIdx(unsigned Opc);
 274
 275   /// Return true if pairing the given load or store may be paired with another.
 276   static bool isPairableLdStInst(const MachineInstr &MI);
 277
 278   /// Returns true if MI is one of the TCRETURN* instructions.
 279   static bool isTailCallReturnInst(const MachineInstr &MI);
 280
 281   /// Return the opcode that set flags when possible.  The caller is
 282   /// responsible for ensuring the opc has a flag setting equivalent.
 283   static unsigned convertToFlagSettingOpc(unsigned Opc);
 284
 285   /// Return true if this is a load/store that can be potentially paired/merged.
 286   bool isCandidateToMergeOrPair(const MachineInstr &MI) const;
 287
 288   /// Hint that pairing the given load or store is unprofitable.
 289   static void suppressLdStPair(MachineInstr &MI);
 290
 291   std::optional<ExtAddrMode>
 292   getAddrModeFromMemoryOp(const MachineInstr &MemI,
 293                           const TargetRegisterInfo *TRI) const override;
 294
 295   bool canFoldIntoAddrMode(const MachineInstr &MemI, Register Reg,
 296                            const MachineInstr &AddrI,
 297                            ExtAddrMode &AM) const override;
 298
 299   MachineInstr *emitLdStWithAddr(MachineInstr &MemI,
 300                                  const ExtAddrMode &AM) const override;
 301
 302   bool getMemOperandsWithOffsetWidth(
 303       const MachineInstr &MI, SmallVectorImpl<const MachineOperand *> &BaseOps,
 304       int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
 305       const TargetRegisterInfo *TRI) const override;
 306
 307   /// If \p OffsetIsScalable is set to 'true', the offset is scaled by `vscale`.
 308   /// This is true for some SVE instructions like ldr/str that have a
 309   /// 'reg + imm' addressing mode where the immediate is an index to the
 310   /// scalable vector located at 'reg + imm * vscale x #bytes'.
 311   bool getMemOperandWithOffsetWidth(const MachineInstr &MI,
 312                                     const MachineOperand *&BaseOp,
 313                                     int64_t &Offset, bool &OffsetIsScalable,
 314                                     TypeSize &Width,
 315                                     const TargetRegisterInfo *TRI) const;
 316
 317   /// Return the immediate offset of the base register in a load/store \p LdSt.
 318   MachineOperand &getMemOpBaseRegImmOfsOffsetOperand(MachineInstr &LdSt) const;
 319
 320   /// Returns true if opcode \p Opc is a memory operation. If it is, set
 321   /// \p Scale, \p Width, \p MinOffset, and \p MaxOffset accordingly.
 322   ///
 323   /// For unscaled instructions, \p Scale is set to 1. All values are in bytes.
 324   /// MinOffset/MaxOffset are the un-scaled limits of the immediate in the
 325   /// instruction, the actual offset limit is [MinOffset*Scale,
 326   /// MaxOffset*Scale].
 327   static bool getMemOpInfo(unsigned Opcode, TypeSize &Scale, TypeSize &Width,
 328                            int64_t &MinOffset, int64_t &MaxOffset);
 329
 330   bool shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
 331                            int64_t Offset1, bool OffsetIsScalable1,
 332                            ArrayRef<const MachineOperand *> BaseOps2,
 333                            int64_t Offset2, bool OffsetIsScalable2,
 334                            unsigned ClusterSize,
 335                            unsigned NumBytes) const override;
 336
 337   void copyPhysRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 338                         const DebugLoc &DL, MCRegister DestReg,
 339                         MCRegister SrcReg, bool KillSrc, unsigned Opcode,
 340                         llvm::ArrayRef<unsigned> Indices) const;
 341   void copyGPRRegTuple(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 342                        const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
 343                        bool KillSrc, unsigned Opcode, unsigned ZeroReg,
 344                        llvm::ArrayRef<unsigned> Indices) const;
 345   void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
 346                    const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg,
 347                    bool KillSrc, bool RenamableDest = false,
 348                    bool RenamableSrc = false) const override;
 349
 350   void storeRegToStackSlot(MachineBasicBlock &MBB,
 351                            MachineBasicBlock::iterator MBBI, Register SrcReg,
 352                            bool isKill, int FrameIndex,
 353                            const TargetRegisterClass *RC,
 354                            const TargetRegisterInfo *TRI,
 355                            Register VReg) const override;
 356
 357   void loadRegFromStackSlot(MachineBasicBlock &MBB,
 358                             MachineBasicBlock::iterator MBBI, Register DestReg,
 359                             int FrameIndex, const TargetRegisterClass *RC,
 360                             const TargetRegisterInfo *TRI,
 361                             Register VReg) const override;
 362
 363   // This tells target independent code that it is okay to pass instructions
 364   // with subreg operands to foldMemoryOperandImpl.
 365   bool isSubregFoldable() const override { return true; }
 366
 367   using TargetInstrInfo::foldMemoryOperandImpl;
 368   MachineInstr *
 369   foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
 370                         ArrayRef<unsigned> Ops,
 371                         MachineBasicBlock::iterator InsertPt, int FrameIndex,
 372                         LiveIntervals *LIS = nullptr,
 373                         VirtRegMap *VRM = nullptr) const override;
 374
 375   /// \returns true if a branch from an instruction with opcode \p BranchOpc
 376   ///  bytes is capable of jumping to a position \p BrOffset bytes away.
 377   bool isBranchOffsetInRange(unsigned BranchOpc,
 378                              int64_t BrOffset) const override;
 379
 380   MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override;
 381
 382   void insertIndirectBranch(MachineBasicBlock &MBB,
 383                             MachineBasicBlock &NewDestBB,
 384                             MachineBasicBlock &RestoreBB, const DebugLoc &DL,
 385                             int64_t BrOffset, RegScavenger *RS) const override;
 386
 387   bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB,
 388                      MachineBasicBlock *&FBB,
 389                      SmallVectorImpl<MachineOperand> &Cond,
 390                      bool AllowModify = false) const override;
 391   bool analyzeBranchPredicate(MachineBasicBlock &MBB,
 392                               MachineBranchPredicate &MBP,
 393                               bool AllowModify) const override;
 394   unsigned removeBranch(MachineBasicBlock &MBB,
 395                         int *BytesRemoved = nullptr) const override;
 396   unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB,
 397                         MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
 398                         const DebugLoc &DL,
 399                         int *BytesAdded = nullptr) const override;
 400
 401   std::unique_ptr<TargetInstrInfo::PipelinerLoopInfo>
 402   analyzeLoopForPipelining(MachineBasicBlock *LoopBB) const override;
 403
 404   bool
 405   reverseBranchCondition(SmallVectorImpl<MachineOperand> &Cond) const override;
 406   bool canInsertSelect(const MachineBasicBlock &, ArrayRef<MachineOperand> Cond,
 407                        Register, Register, Register, int &, int &,
 408                        int &) const override;
 409   void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
 410                     const DebugLoc &DL, Register DstReg,
 411                     ArrayRef<MachineOperand> Cond, Register TrueReg,
 412                     Register FalseReg) const override;
 413
 414   void insertNoop(MachineBasicBlock &MBB,
 415                   MachineBasicBlock::iterator MI) const override;
 416
 417   MCInst getNop() const override;
 418
 419   bool isSchedulingBoundary(const MachineInstr &MI,
 420                             const MachineBasicBlock *MBB,
 421                             const MachineFunction &MF) const override;
 422
 423   /// analyzeCompare - For a comparison instruction, return the source registers
 424   /// in SrcReg and SrcReg2, and the value it compares against in CmpValue.
 425   /// Return true if the comparison instruction can be analyzed.
 426   bool analyzeCompare(const MachineInstr &MI, Register &SrcReg,
 427                       Register &SrcReg2, int64_t &CmpMask,
 428                       int64_t &CmpValue) const override;
 429   /// optimizeCompareInstr - Convert the instruction supplying the argument to
 430   /// the comparison into one that sets the zero bit in the flags register.
 431   bool optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
 432                             Register SrcReg2, int64_t CmpMask, int64_t CmpValue,
 433                             const MachineRegisterInfo *MRI) const override;
 434   bool optimizeCondBranch(MachineInstr &MI) const override;
 435
 436   CombinerObjective getCombinerObjective(unsigned Pattern) const override;
 437   /// Return true when a code sequence can improve throughput. It
 438   /// should be called only for instructions in loops.
 439   /// \param Pattern - combiner pattern
 440   bool isThroughputPattern(unsigned Pattern) const override;
 441   /// Return true when there is potentially a faster code sequence
 442   /// for an instruction chain ending in ``Root``. All potential patterns are
 443   /// listed in the ``Patterns`` array.
 444   bool getMachineCombinerPatterns(MachineInstr &Root,
 445                                   SmallVectorImpl<unsigned> &Patterns,
 446                                   bool DoRegPressureReduce) const override;
 447   /// Return true when Inst is associative and commutative so that it can be
 448   /// reassociated. If Invert is true, then the inverse of Inst operation must
 449   /// be checked.
 450   bool isAssociativeAndCommutative(const MachineInstr &Inst,
 451                                    bool Invert) const override;
 452   /// When getMachineCombinerPatterns() finds patterns, this function generates
 453   /// the instructions that could replace the original code sequence
 454   void genAlternativeCodeSequence(
 455       MachineInstr &Root, unsigned Pattern,
 456       SmallVectorImpl<MachineInstr *> &InsInstrs,
 457       SmallVectorImpl<MachineInstr *> &DelInstrs,
 458       DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const override;
 459   /// AArch64 supports MachineCombiner.
 460   bool useMachineCombiner() const override;
 461
 462   bool expandPostRAPseudo(MachineInstr &MI) const override;
 463
 464   std::pair<unsigned, unsigned>
 465   decomposeMachineOperandsTargetFlags(unsigned TF) const override;
 466   ArrayRef<std::pair<unsigned, const char *>>
 467   getSerializableDirectMachineOperandTargetFlags() const override;
 468   ArrayRef<std::pair<unsigned, const char *>>
 469   getSerializableBitmaskMachineOperandTargetFlags() const override;
 470   ArrayRef<std::pair<MachineMemOperand::Flags, const char *>>
 471   getSerializableMachineMemOperandTargetFlags() const override;
 472
 473   bool isFunctionSafeToOutlineFrom(MachineFunction &MF,
 474                                    bool OutlineFromLinkOnceODRs) const override;
 475   std::optional<std::unique_ptr<outliner::OutlinedFunction>>
 476   getOutliningCandidateInfo(
 477       const MachineModuleInfo &MMI,
 478       std::vector<outliner::Candidate> &RepeatedSequenceLocs,
 479       unsigned MinRepeats) const override;
 480   void mergeOutliningCandidateAttributes(
 481       Function &F, std::vector<outliner::Candidate> &Candidates) const override;
 482   outliner::InstrType getOutliningTypeImpl(const MachineModuleInfo &MMI,
 483                                            MachineBasicBlock::iterator &MIT,
 484                                            unsigned Flags) const override;
 485   SmallVector<
 486       std::pair<MachineBasicBlock::iterator, MachineBasicBlock::iterator>>
 487   getOutlinableRanges(MachineBasicBlock &MBB, unsigned &Flags) const override;
 488   void buildOutlinedFrame(MachineBasicBlock &MBB, MachineFunction &MF,
 489                           const outliner::OutlinedFunction &OF) const override;
 490   MachineBasicBlock::iterator
 491   insertOutlinedCall(Module &M, MachineBasicBlock &MBB,
 492                      MachineBasicBlock::iterator &It, MachineFunction &MF,
 493                      outliner::Candidate &C) const override;
 494   bool shouldOutlineFromFunctionByDefault(MachineFunction &MF) const override;
 495
 496   void buildClearRegister(Register Reg, MachineBasicBlock &MBB,
 497                           MachineBasicBlock::iterator Iter, DebugLoc &DL,
 498                           bool AllowSideEffects = true) const override;
 499
 500   /// Returns the vector element size (B, H, S or D) of an SVE opcode.
 501   uint64_t getElementSizeForOpcode(unsigned Opc) const;
 502   /// Returns true if the opcode is for an SVE instruction that sets the
 503   /// condition codes as if it's results had been fed to a PTEST instruction
 504   /// along with the same general predicate.
 505   bool isPTestLikeOpcode(unsigned Opc) const;
 506   /// Returns true if the opcode is for an SVE WHILE## instruction.
 507   bool isWhileOpcode(unsigned Opc) const;
 508   /// Returns true if the instruction has a shift by immediate that can be
 509   /// executed in one cycle less.
 510   static bool isFalkorShiftExtFast(const MachineInstr &MI);
 511   /// Return true if the instructions is a SEH instruciton used for unwinding
 512   /// on Windows.
 513   static bool isSEHInstruction(const MachineInstr &MI);
 514
 515   std::optional<RegImmPair> isAddImmediate(const MachineInstr &MI,
 516                                            Register Reg) const override;
 517
 518   bool isFunctionSafeToSplit(const MachineFunction &MF) const override;
 519
 520   bool isMBBSafeToSplitToCold(const MachineBasicBlock &MBB) const override;
 521
 522   std::optional<ParamLoadedValue>
 523   describeLoadedValue(const MachineInstr &MI, Register Reg) const override;
 524
 525   unsigned int getTailDuplicateSize(CodeGenOptLevel OptLevel) const override;
 526
 527   bool isExtendLikelyToBeFolded(MachineInstr &ExtMI,
 528                                 MachineRegisterInfo &MRI) const override;
 529
 530   static void decomposeStackOffsetForFrameOffsets(const StackOffset &Offset,
 531                                                   int64_t &NumBytes,
 532                                                   int64_t &NumPredicateVectors,
 533                                                   int64_t &NumDataVectors);
 534   static void decomposeStackOffsetForDwarfOffsets(const StackOffset &Offset,
 535                                                   int64_t &ByteSized,
 536                                                   int64_t &VGSized);
 537
 538   // Return true if address of the form BaseReg + Scale * ScaledReg + Offset can
 539   // be used for a load/store of NumBytes. BaseReg is always present and
 540   // implicit.
 541   bool isLegalAddressingMode(unsigned NumBytes, int64_t Offset,
 542                              unsigned Scale) const;
 543
 544   // Decrement the SP, issuing probes along the way. `TargetReg` is the new top
 545   // of the stack. `FrameSetup` is passed as true, if the allocation is a part
 546   // of constructing the activation frame of a function.
 547   MachineBasicBlock::iterator probedStackAlloc(MachineBasicBlock::iterator MBBI,
 548                                                Register TargetReg,
 549                                                bool FrameSetup) const;
 550
 551 #define GET_INSTRINFO_HELPER_DECLS
 552 #include "AArch64GenInstrInfo.inc"
 553
 554 protected:
 555   /// If the specific machine instruction is an instruction that moves/copies
 556   /// value from one register to another register return destination and source
 557   /// registers as machine operands.
 558   std::optional<DestSourcePair>
 559   isCopyInstrImpl(const MachineInstr &MI) const override;
 560   std::optional<DestSourcePair>
 561   isCopyLikeInstrImpl(const MachineInstr &MI) const override;
 562
 563 private:
 564   unsigned getInstBundleLength(const MachineInstr &MI) const;
 565
 566   /// Sets the offsets on outlined instructions in \p MBB which use SP
 567   /// so that they will be valid post-outlining.
 568   ///
 569   /// \param MBB A \p MachineBasicBlock in an outlined function.
 570   void fixupPostOutline(MachineBasicBlock &MBB) const;
 571
 572   void instantiateCondBranch(MachineBasicBlock &MBB, const DebugLoc &DL,
 573                              MachineBasicBlock *TBB,
 574                              ArrayRef<MachineOperand> Cond) const;
 575   bool substituteCmpToZero(MachineInstr &CmpInstr, unsigned SrcReg,
 576                            const MachineRegisterInfo &MRI) const;
 577   bool removeCmpToZeroOrOne(MachineInstr &CmpInstr, unsigned SrcReg,
 578                             int CmpValue, const MachineRegisterInfo &MRI) const;
 579
 580   /// Returns an unused general-purpose register which can be used for
 581   /// constructing an outlined call if one exists. Returns 0 otherwise.
 582   Register findRegisterToSaveLRTo(outliner::Candidate &C) const;
 583
 584   /// Remove a ptest of a predicate-generating operation that already sets, or
 585   /// can be made to set, the condition codes in an identical manner
 586   bool optimizePTestInstr(MachineInstr *PTest, unsigned MaskReg,
 587                           unsigned PredReg,
 588                           const MachineRegisterInfo *MRI) const;
 589   std::optional<unsigned>
 590   canRemovePTestInstr(MachineInstr *PTest, MachineInstr *Mask,
 591                       MachineInstr *Pred, const MachineRegisterInfo *MRI) const;
 592
 593   /// verifyInstruction - Perform target specific instruction verification.
 594   bool verifyInstruction(const MachineInstr &MI,
 595                          StringRef &ErrInfo) const override;
 596 };
 597
 598 struct UsedNZCV {
 599   bool N = false;
 600   bool Z = false;
 601   bool C = false;
 602   bool V = false;
 603
 604   UsedNZCV() = default;
 605
 606   UsedNZCV &operator|=(const UsedNZCV &UsedFlags) {
 607     this->N |= UsedFlags.N;
 608     this->Z |= UsedFlags.Z;
 609     this->C |= UsedFlags.C;
 610     this->V |= UsedFlags.V;
 611     return *this;
 612   }
 613 };
 614
 615 /// \returns Conditions flags used after \p CmpInstr in its MachineBB if  NZCV
 616 /// flags are not alive in successors of the same \p CmpInstr and \p MI parent.
 617 /// \returns std::nullopt otherwise.
 618 ///
 619 /// Collect instructions using that flags in \p CCUseInstrs if provided.
 620 std::optional<UsedNZCV>
 621 examineCFlagsUse(MachineInstr &MI, MachineInstr &CmpInstr,
 622                  const TargetRegisterInfo &TRI,
 623                  SmallVectorImpl<MachineInstr *> *CCUseInstrs = nullptr);
 624
 625 /// Return true if there is an instruction /after/ \p DefMI and before \p UseMI
 626 /// which either reads or clobbers NZCV.
 627 bool isNZCVTouchedInInstructionRange(const MachineInstr &DefMI,
 628                                      const MachineInstr &UseMI,
 629                                      const TargetRegisterInfo *TRI);
 630
 631 MCCFIInstruction createDefCFA(const TargetRegisterInfo &TRI, unsigned FrameReg,
 632                               unsigned Reg, const StackOffset &Offset,
 633                               bool LastAdjustmentWasScalable = true);
 634 MCCFIInstruction createCFAOffset(const TargetRegisterInfo &MRI, unsigned Reg,
 635                                  const StackOffset &OffsetFromDefCFA);
 636
 637 /// emitFrameOffset - Emit instructions as needed to set DestReg to SrcReg
 638 /// plus Offset.  This is intended to be used from within the prolog/epilog
 639 /// insertion (PEI) pass, where a virtual scratch register may be allocated
 640 /// if necessary, to be replaced by the scavenger at the end of PEI.
 641 void emitFrameOffset(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
 642                      const DebugLoc &DL, unsigned DestReg, unsigned SrcReg,
 643                      StackOffset Offset, const TargetInstrInfo *TII,
 644                      MachineInstr::MIFlag = MachineInstr::NoFlags,
 645                      bool SetNZCV = false, bool NeedsWinCFI = false,
 646                      bool *HasWinCFI = nullptr, bool EmitCFAOffset = false,
 647                      StackOffset InitialOffset = {},
 648                      unsigned FrameReg = AArch64::SP);
 649
 650 /// rewriteAArch64FrameIndex - Rewrite MI to access 'Offset' bytes from the
 651 /// FP. Return false if the offset could not be handled directly in MI, and
 652 /// return the left-over portion by reference.
 653 bool rewriteAArch64FrameIndex(MachineInstr &MI, unsigned FrameRegIdx,
 654                               unsigned FrameReg, StackOffset &Offset,
 655                               const AArch64InstrInfo *TII);
 656
 657 /// Use to report the frame offset status in isAArch64FrameOffsetLegal.
 658 enum AArch64FrameOffsetStatus {
 659   AArch64FrameOffsetCannotUpdate = 0x0, ///< Offset cannot apply.
 660   AArch64FrameOffsetIsLegal = 0x1,      ///< Offset is legal.
 661   AArch64FrameOffsetCanUpdate = 0x2     ///< Offset can apply, at least partly.
 662 };
 663
 664 /// Check if the @p Offset is a valid frame offset for @p MI.
 665 /// The returned value reports the validity of the frame offset for @p MI.
 666 /// It uses the values defined by AArch64FrameOffsetStatus for that.
 667 /// If result == AArch64FrameOffsetCannotUpdate, @p MI cannot be updated to
 668 /// use an offset.eq
 669 /// If result & AArch64FrameOffsetIsLegal, @p Offset can completely be
 670 /// rewritten in @p MI.
 671 /// If result & AArch64FrameOffsetCanUpdate, @p Offset contains the
 672 /// amount that is off the limit of the legal offset.
 673 /// If set, @p OutUseUnscaledOp will contain the whether @p MI should be
 674 /// turned into an unscaled operator, which opcode is in @p OutUnscaledOp.
 675 /// If set, @p EmittableOffset contains the amount that can be set in @p MI
 676 /// (possibly with @p OutUnscaledOp if OutUseUnscaledOp is true) and that
 677 /// is a legal offset.
 678 int isAArch64FrameOffsetLegal(const MachineInstr &MI, StackOffset &Offset,
 679                               bool *OutUseUnscaledOp = nullptr,
 680                               unsigned *OutUnscaledOp = nullptr,
 681                               int64_t *EmittableOffset = nullptr);
 682
 683 static inline bool isUncondBranchOpcode(int Opc) { return Opc == AArch64::B; }
 684
 685 static inline bool isCondBranchOpcode(int Opc) {
 686   switch (Opc) {
 687   case AArch64::Bcc:
 688   case AArch64::CBZW:
 689   case AArch64::CBZX:
 690   case AArch64::CBNZW:
 691   case AArch64::CBNZX:
 692   case AArch64::TBZW:
 693   case AArch64::TBZX:
 694   case AArch64::TBNZW:
 695   case AArch64::TBNZX:
 696     return true;
 697   default:
 698     return false;
 699   }
 700 }
 701
 702 static inline bool isIndirectBranchOpcode(int Opc) {
 703   switch (Opc) {
 704   case AArch64::BR:
 705   case AArch64::BRAA:
 706   case AArch64::BRAB:
 707   case AArch64::BRAAZ:
 708   case AArch64::BRABZ:
 709     return true;
 710   }
 711   return false;
 712 }
 713
 714 static inline bool isPTrueOpcode(unsigned Opc) {
 715   switch (Opc) {
 716   case AArch64::PTRUE_B:
 717   case AArch64::PTRUE_H:
 718   case AArch64::PTRUE_S:
 719   case AArch64::PTRUE_D:
 720     return true;
 721   default:
 722     return false;
 723   }
 724 }
 725
 726 /// Return opcode to be used for indirect calls.
 727 unsigned getBLRCallOpcode(const MachineFunction &MF);
 728
 729 /// Return XPAC opcode to be used for a ptrauth strip using the given key.
 730 static inline unsigned getXPACOpcodeForKey(AArch64PACKey::ID K) {
 731   using namespace AArch64PACKey;
 732   switch (K) {
 733   case IA: case IB: return AArch64::XPACI;
 734   case DA: case DB: return AArch64::XPACD;
 735   }
 736   llvm_unreachable("Unhandled AArch64PACKey::ID enum");
 737 }
 738
 739 /// Return AUT opcode to be used for a ptrauth auth using the given key, or its
 740 /// AUT*Z variant that doesn't take a discriminator operand, using zero instead.
 741 static inline unsigned getAUTOpcodeForKey(AArch64PACKey::ID K, bool Zero) {
 742   using namespace AArch64PACKey;
 743   switch (K) {
 744   case IA: return Zero ? AArch64::AUTIZA : AArch64::AUTIA;
 745   case IB: return Zero ? AArch64::AUTIZB : AArch64::AUTIB;
 746   case DA: return Zero ? AArch64::AUTDZA : AArch64::AUTDA;
 747   case DB: return Zero ? AArch64::AUTDZB : AArch64::AUTDB;
 748   }
 749   llvm_unreachable("Unhandled AArch64PACKey::ID enum");
 750 }
 751
 752 /// Return PAC opcode to be used for a ptrauth sign using the given key, or its
 753 /// PAC*Z variant that doesn't take a discriminator operand, using zero instead.
 754 static inline unsigned getPACOpcodeForKey(AArch64PACKey::ID K, bool Zero) {
 755   using namespace AArch64PACKey;
 756   switch (K) {
 757   case IA: return Zero ? AArch64::PACIZA : AArch64::PACIA;
 758   case IB: return Zero ? AArch64::PACIZB : AArch64::PACIB;
 759   case DA: return Zero ? AArch64::PACDZA : AArch64::PACDA;
 760   case DB: return Zero ? AArch64::PACDZB : AArch64::PACDB;
 761   }
 762   llvm_unreachable("Unhandled AArch64PACKey::ID enum");
 763 }
 764
 765 // struct TSFlags {
 766 #define TSFLAG_ELEMENT_SIZE_TYPE(X)      (X)        // 3-bits
 767 #define TSFLAG_DESTRUCTIVE_INST_TYPE(X) ((X) << 3)  // 4-bits
 768 #define TSFLAG_FALSE_LANE_TYPE(X)       ((X) << 7)  // 2-bits
 769 #define TSFLAG_INSTR_FLAGS(X)           ((X) << 9)  // 2-bits
 770 #define TSFLAG_SME_MATRIX_TYPE(X)       ((X) << 11) // 3-bits
 771 // }
 772
 773 namespace AArch64 {
 774
 775 enum ElementSizeType {
 776   ElementSizeMask = TSFLAG_ELEMENT_SIZE_TYPE(0x7),
 777   ElementSizeNone = TSFLAG_ELEMENT_SIZE_TYPE(0x0),
 778   ElementSizeB    = TSFLAG_ELEMENT_SIZE_TYPE(0x1),
 779   ElementSizeH    = TSFLAG_ELEMENT_SIZE_TYPE(0x2),
 780   ElementSizeS    = TSFLAG_ELEMENT_SIZE_TYPE(0x3),
 781   ElementSizeD    = TSFLAG_ELEMENT_SIZE_TYPE(0x4),
 782 };
 783
 784 enum DestructiveInstType {
 785   DestructiveInstTypeMask       = TSFLAG_DESTRUCTIVE_INST_TYPE(0xf),
 786   NotDestructive                = TSFLAG_DESTRUCTIVE_INST_TYPE(0x0),
 787   DestructiveOther              = TSFLAG_DESTRUCTIVE_INST_TYPE(0x1),
 788   DestructiveUnary              = TSFLAG_DESTRUCTIVE_INST_TYPE(0x2),
 789   DestructiveBinaryImm          = TSFLAG_DESTRUCTIVE_INST_TYPE(0x3),
 790   DestructiveBinaryShImmUnpred  = TSFLAG_DESTRUCTIVE_INST_TYPE(0x4),
 791   DestructiveBinary             = TSFLAG_DESTRUCTIVE_INST_TYPE(0x5),
 792   DestructiveBinaryComm         = TSFLAG_DESTRUCTIVE_INST_TYPE(0x6),
 793   DestructiveBinaryCommWithRev  = TSFLAG_DESTRUCTIVE_INST_TYPE(0x7),
 794   DestructiveTernaryCommWithRev = TSFLAG_DESTRUCTIVE_INST_TYPE(0x8),
 795   DestructiveUnaryPassthru      = TSFLAG_DESTRUCTIVE_INST_TYPE(0x9),
 796 };
 797
 798 enum FalseLaneType {
 799   FalseLanesMask  = TSFLAG_FALSE_LANE_TYPE(0x3),
 800   FalseLanesZero  = TSFLAG_FALSE_LANE_TYPE(0x1),
 801   FalseLanesUndef = TSFLAG_FALSE_LANE_TYPE(0x2),
 802 };
 803
 804 // NOTE: This is a bit field.
 805 static const uint64_t InstrFlagIsWhile     = TSFLAG_INSTR_FLAGS(0x1);
 806 static const uint64_t InstrFlagIsPTestLike = TSFLAG_INSTR_FLAGS(0x2);
 807
 808 enum SMEMatrixType {
 809   SMEMatrixTypeMask = TSFLAG_SME_MATRIX_TYPE(0x7),
 810   SMEMatrixNone     = TSFLAG_SME_MATRIX_TYPE(0x0),
 811   SMEMatrixTileB    = TSFLAG_SME_MATRIX_TYPE(0x1),
 812   SMEMatrixTileH    = TSFLAG_SME_MATRIX_TYPE(0x2),
 813   SMEMatrixTileS    = TSFLAG_SME_MATRIX_TYPE(0x3),
 814   SMEMatrixTileD    = TSFLAG_SME_MATRIX_TYPE(0x4),
 815   SMEMatrixTileQ    = TSFLAG_SME_MATRIX_TYPE(0x5),
 816   SMEMatrixArray    = TSFLAG_SME_MATRIX_TYPE(0x6),
 817 };
 818
 819 #undef TSFLAG_ELEMENT_SIZE_TYPE
 820 #undef TSFLAG_DESTRUCTIVE_INST_TYPE
 821 #undef TSFLAG_FALSE_LANE_TYPE
 822 #undef TSFLAG_INSTR_FLAGS
 823 #undef TSFLAG_SME_MATRIX_TYPE
 824
 825 int getSVEPseudoMap(uint16_t Opcode);
 826 int getSVERevInstr(uint16_t Opcode);
 827 int getSVENonRevInstr(uint16_t Opcode);
 828
 829 int getSMEPseudoMap(uint16_t Opcode);
 830 }
 831
 832 } // end namespace llvm
 833
 834 #endif