llvm/lib/Target/AArch64/AArch64MacroFusion.cpp

   1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file contains the AArch64 implementation of the DAG scheduling
  10 ///  mutation to pair instructions back to back.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64MacroFusion.h"
  15 #include "AArch64Subtarget.h"
  16 #include "llvm/CodeGen/MacroFusion.h"
  17 #include "llvm/CodeGen/TargetInstrInfo.h"
  18
  19 using namespace llvm;
  20
  21 /// CMN, CMP, TST followed by Bcc
  22 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
  23                                 const MachineInstr &SecondMI, bool CmpOnly) {
  24   if (SecondMI.getOpcode() != AArch64::Bcc)
  25     return false;
  26
  27   // Assume the 1st instr to be a wildcard if it is unspecified.
  28   if (FirstMI == nullptr)
  29     return true;
  30
  31   // If we're in CmpOnly mode, we only fuse arithmetic instructions that
  32   // discard their result.
  33   if (CmpOnly && !(FirstMI->getOperand(0).getReg() == AArch64::XZR ||
  34                    FirstMI->getOperand(0).getReg() == AArch64::WZR)) {
  35     return false;
  36   }
  37
  38   switch (FirstMI->getOpcode()) {
  39   case AArch64::ADDSWri:
  40   case AArch64::ADDSWrr:
  41   case AArch64::ADDSXri:
  42   case AArch64::ADDSXrr:
  43   case AArch64::ANDSWri:
  44   case AArch64::ANDSWrr:
  45   case AArch64::ANDSXri:
  46   case AArch64::ANDSXrr:
  47   case AArch64::SUBSWri:
  48   case AArch64::SUBSWrr:
  49   case AArch64::SUBSXri:
  50   case AArch64::SUBSXrr:
  51   case AArch64::BICSWrr:
  52   case AArch64::BICSXrr:
  53     return true;
  54   case AArch64::ADDSWrs:
  55   case AArch64::ADDSXrs:
  56   case AArch64::ANDSWrs:
  57   case AArch64::ANDSXrs:
  58   case AArch64::SUBSWrs:
  59   case AArch64::SUBSXrs:
  60   case AArch64::BICSWrs:
  61   case AArch64::BICSXrs:
  62     // Shift value can be 0 making these behave like the "rr" variant...
  63     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
  64   }
  65
  66   return false;
  67 }
  68
  69 /// ALU operations followed by CBZ/CBNZ.
  70 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
  71                                 const MachineInstr &SecondMI) {
  72   if (SecondMI.getOpcode() != AArch64::CBZW &&
  73       SecondMI.getOpcode() != AArch64::CBZX &&
  74       SecondMI.getOpcode() != AArch64::CBNZW &&
  75       SecondMI.getOpcode() != AArch64::CBNZX)
  76     return false;
  77
  78   // Assume the 1st instr to be a wildcard if it is unspecified.
  79   if (FirstMI == nullptr)
  80     return true;
  81
  82   switch (FirstMI->getOpcode()) {
  83   case AArch64::ADDWri:
  84   case AArch64::ADDWrr:
  85   case AArch64::ADDXri:
  86   case AArch64::ADDXrr:
  87   case AArch64::ANDWri:
  88   case AArch64::ANDWrr:
  89   case AArch64::ANDXri:
  90   case AArch64::ANDXrr:
  91   case AArch64::EORWri:
  92   case AArch64::EORWrr:
  93   case AArch64::EORXri:
  94   case AArch64::EORXrr:
  95   case AArch64::ORRWri:
  96   case AArch64::ORRWrr:
  97   case AArch64::ORRXri:
  98   case AArch64::ORRXrr:
  99   case AArch64::SUBWri:
 100   case AArch64::SUBWrr:
 101   case AArch64::SUBXri:
 102   case AArch64::SUBXrr:
 103     return true;
 104   case AArch64::ADDWrs:
 105   case AArch64::ADDXrs:
 106   case AArch64::ANDWrs:
 107   case AArch64::ANDXrs:
 108   case AArch64::SUBWrs:
 109   case AArch64::SUBXrs:
 110   case AArch64::BICWrs:
 111   case AArch64::BICXrs:
 112     // Shift value can be 0 making these behave like the "rr" variant...
 113     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 114   }
 115
 116   return false;
 117 }
 118
 119 /// AES crypto encoding or decoding.
 120 static bool isAESPair(const MachineInstr *FirstMI,
 121                       const MachineInstr &SecondMI) {
 122   // Assume the 1st instr to be a wildcard if it is unspecified.
 123   switch (SecondMI.getOpcode()) {
 124   // AES encode.
 125   case AArch64::AESMCrr:
 126   case AArch64::AESMCrrTied:
 127     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
 128   // AES decode.
 129   case AArch64::AESIMCrr:
 130   case AArch64::AESIMCrrTied:
 131     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
 132   }
 133
 134   return false;
 135 }
 136
 137 /// AESE/AESD/PMULL + EOR.
 138 static bool isCryptoEORPair(const MachineInstr *FirstMI,
 139                             const MachineInstr &SecondMI) {
 140   if (SecondMI.getOpcode() != AArch64::EORv16i8)
 141     return false;
 142
 143   // Assume the 1st instr to be a wildcard if it is unspecified.
 144   if (FirstMI == nullptr)
 145     return true;
 146
 147   switch (FirstMI->getOpcode()) {
 148   case AArch64::AESErr:
 149   case AArch64::AESDrr:
 150   case AArch64::PMULLv16i8:
 151   case AArch64::PMULLv8i8:
 152   case AArch64::PMULLv1i64:
 153   case AArch64::PMULLv2i64:
 154     return true;
 155   }
 156
 157   return false;
 158 }
 159
 160 /// Literal generation.
 161 static bool isLiteralsPair(const MachineInstr *FirstMI,
 162                            const MachineInstr &SecondMI) {
 163   // Assume the 1st instr to be a wildcard if it is unspecified.
 164
 165   // PC relative address.
 166   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
 167       SecondMI.getOpcode() == AArch64::ADDXri)
 168     return true;
 169
 170   // 32 bit immediate.
 171   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
 172       (SecondMI.getOpcode() == AArch64::MOVKWi &&
 173        SecondMI.getOperand(3).getImm() == 16))
 174     return true;
 175
 176   // Lower half of 64 bit immediate.
 177   if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
 178      (SecondMI.getOpcode() == AArch64::MOVKXi &&
 179       SecondMI.getOperand(3).getImm() == 16))
 180     return true;
 181
 182   // Upper half of 64 bit immediate.
 183   if ((FirstMI == nullptr ||
 184        (FirstMI->getOpcode() == AArch64::MOVKXi &&
 185         FirstMI->getOperand(3).getImm() == 32)) &&
 186       (SecondMI.getOpcode() == AArch64::MOVKXi &&
 187        SecondMI.getOperand(3).getImm() == 48))
 188     return true;
 189
 190   return false;
 191 }
 192
 193 /// Fuse address generation and loads or stores.
 194 static bool isAddressLdStPair(const MachineInstr *FirstMI,
 195                               const MachineInstr &SecondMI) {
 196   switch (SecondMI.getOpcode()) {
 197   case AArch64::STRBBui:
 198   case AArch64::STRBui:
 199   case AArch64::STRDui:
 200   case AArch64::STRHHui:
 201   case AArch64::STRHui:
 202   case AArch64::STRQui:
 203   case AArch64::STRSui:
 204   case AArch64::STRWui:
 205   case AArch64::STRXui:
 206   case AArch64::LDRBBui:
 207   case AArch64::LDRBui:
 208   case AArch64::LDRDui:
 209   case AArch64::LDRHHui:
 210   case AArch64::LDRHui:
 211   case AArch64::LDRQui:
 212   case AArch64::LDRSui:
 213   case AArch64::LDRWui:
 214   case AArch64::LDRXui:
 215   case AArch64::LDRSBWui:
 216   case AArch64::LDRSBXui:
 217   case AArch64::LDRSHWui:
 218   case AArch64::LDRSHXui:
 219   case AArch64::LDRSWui:
 220     // Assume the 1st instr to be a wildcard if it is unspecified.
 221     if (FirstMI == nullptr)
 222       return true;
 223
 224    switch (FirstMI->getOpcode()) {
 225     case AArch64::ADR:
 226       return SecondMI.getOperand(2).getImm() == 0;
 227     case AArch64::ADRP:
 228       return true;
 229     }
 230   }
 231
 232   return false;
 233 }
 234
 235 /// Compare and conditional select.
 236 static bool isCCSelectPair(const MachineInstr *FirstMI,
 237                            const MachineInstr &SecondMI) {
 238   // 32 bits
 239   if (SecondMI.getOpcode() == AArch64::CSELWr) {
 240     // Assume the 1st instr to be a wildcard if it is unspecified.
 241     if (FirstMI == nullptr)
 242       return true;
 243
 244     if (FirstMI->definesRegister(AArch64::WZR))
 245       switch (FirstMI->getOpcode()) {
 246       case AArch64::SUBSWrs:
 247         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 248       case AArch64::SUBSWrx:
 249         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
 250       case AArch64::SUBSWrr:
 251       case AArch64::SUBSWri:
 252         return true;
 253       }
 254   }
 255
 256   // 64 bits
 257   if (SecondMI.getOpcode() == AArch64::CSELXr) {
 258     // Assume the 1st instr to be a wildcard if it is unspecified.
 259     if (FirstMI == nullptr)
 260       return true;
 261
 262     if (FirstMI->definesRegister(AArch64::XZR))
 263       switch (FirstMI->getOpcode()) {
 264       case AArch64::SUBSXrs:
 265         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 266       case AArch64::SUBSXrx:
 267       case AArch64::SUBSXrx64:
 268         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
 269       case AArch64::SUBSXrr:
 270       case AArch64::SUBSXri:
 271         return true;
 272       }
 273   }
 274
 275   return false;
 276 }
 277
 278 // Arithmetic and logic.
 279 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
 280                                   const MachineInstr &SecondMI) {
 281   if (AArch64InstrInfo::hasShiftedReg(SecondMI))
 282     return false;
 283
 284   switch (SecondMI.getOpcode()) {
 285   // Arithmetic
 286   case AArch64::ADDWrr:
 287   case AArch64::ADDXrr:
 288   case AArch64::SUBWrr:
 289   case AArch64::SUBXrr:
 290   case AArch64::ADDWrs:
 291   case AArch64::ADDXrs:
 292   case AArch64::SUBWrs:
 293   case AArch64::SUBXrs:
 294   // Logic
 295   case AArch64::ANDWrr:
 296   case AArch64::ANDXrr:
 297   case AArch64::BICWrr:
 298   case AArch64::BICXrr:
 299   case AArch64::EONWrr:
 300   case AArch64::EONXrr:
 301   case AArch64::EORWrr:
 302   case AArch64::EORXrr:
 303   case AArch64::ORNWrr:
 304   case AArch64::ORNXrr:
 305   case AArch64::ORRWrr:
 306   case AArch64::ORRXrr:
 307   case AArch64::ANDWrs:
 308   case AArch64::ANDXrs:
 309   case AArch64::BICWrs:
 310   case AArch64::BICXrs:
 311   case AArch64::EONWrs:
 312   case AArch64::EONXrs:
 313   case AArch64::EORWrs:
 314   case AArch64::EORXrs:
 315   case AArch64::ORNWrs:
 316   case AArch64::ORNXrs:
 317   case AArch64::ORRWrs:
 318   case AArch64::ORRXrs:
 319     // Assume the 1st instr to be a wildcard if it is unspecified.
 320     if (FirstMI == nullptr)
 321       return true;
 322
 323     // Arithmetic
 324     switch (FirstMI->getOpcode()) {
 325     case AArch64::ADDWrr:
 326     case AArch64::ADDXrr:
 327     case AArch64::ADDSWrr:
 328     case AArch64::ADDSXrr:
 329     case AArch64::SUBWrr:
 330     case AArch64::SUBXrr:
 331     case AArch64::SUBSWrr:
 332     case AArch64::SUBSXrr:
 333       return true;
 334     case AArch64::ADDWrs:
 335     case AArch64::ADDXrs:
 336     case AArch64::ADDSWrs:
 337     case AArch64::ADDSXrs:
 338     case AArch64::SUBWrs:
 339     case AArch64::SUBXrs:
 340     case AArch64::SUBSWrs:
 341     case AArch64::SUBSXrs:
 342       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 343     }
 344     break;
 345
 346   // Arithmetic, setting flags.
 347   case AArch64::ADDSWrr:
 348   case AArch64::ADDSXrr:
 349   case AArch64::SUBSWrr:
 350   case AArch64::SUBSXrr:
 351   case AArch64::ADDSWrs:
 352   case AArch64::ADDSXrs:
 353   case AArch64::SUBSWrs:
 354   case AArch64::SUBSXrs:
 355     // Assume the 1st instr to be a wildcard if it is unspecified.
 356     if (FirstMI == nullptr)
 357       return true;
 358
 359     // Arithmetic, not setting flags.
 360     switch (FirstMI->getOpcode()) {
 361     case AArch64::ADDWrr:
 362     case AArch64::ADDXrr:
 363     case AArch64::SUBWrr:
 364     case AArch64::SUBXrr:
 365       return true;
 366     case AArch64::ADDWrs:
 367     case AArch64::ADDXrs:
 368     case AArch64::SUBWrs:
 369     case AArch64::SUBXrs:
 370       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 371     }
 372     break;
 373   }
 374
 375   return false;
 376 }
 377
 378 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
 379 /// together. Given SecondMI, when FirstMI is unspecified, then check if
 380 /// SecondMI may be part of a fused pair at all.
 381 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
 382                                    const TargetSubtargetInfo &TSI,
 383                                    const MachineInstr *FirstMI,
 384                                    const MachineInstr &SecondMI) {
 385   const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
 386
 387   // All checking functions assume that the 1st instr is a wildcard if it is
 388   // unspecified.
 389   if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {
 390     bool CmpOnly = !ST.hasArithmeticBccFusion();
 391     if (isArithmeticBccPair(FirstMI, SecondMI, CmpOnly))
 392       return true;
 393   }
 394   if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
 395     return true;
 396   if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
 397     return true;
 398   if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
 399     return true;
 400   if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
 401     return true;
 402   if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
 403     return true;
 404   if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
 405     return true;
 406   if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
 407     return true;
 408
 409   return false;
 410 }
 411
 412 std::unique_ptr<ScheduleDAGMutation>
 413 llvm::createAArch64MacroFusionDAGMutation() {
 414   return createMacroFusionDAGMutation(shouldScheduleAdjacent);
 415 }