lib/Target/AArch64/AArch64MacroFusion.cpp

   1 //===- AArch64MacroFusion.cpp - AArch64 Macro Fusion ----------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file This file contains the AArch64 implementation of the DAG scheduling
  10 ///  mutation to pair instructions back to back.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "AArch64Subtarget.h"
  15 #include "llvm/CodeGen/MacroFusion.h"
  16 #include "llvm/CodeGen/TargetInstrInfo.h"
  17
  18 using namespace llvm;
  19
  20 namespace {
  21
  22 /// CMN, CMP, TST followed by Bcc
  23 static bool isArithmeticBccPair(const MachineInstr *FirstMI,
  24                                 const MachineInstr &SecondMI) {
  25   if (SecondMI.getOpcode() != AArch64::Bcc)
  26     return false;
  27
  28   // Assume the 1st instr to be a wildcard if it is unspecified.
  29   if (FirstMI == nullptr)
  30     return true;
  31
  32   switch (FirstMI->getOpcode()) {
  33   case AArch64::ADDSWri:
  34   case AArch64::ADDSWrr:
  35   case AArch64::ADDSXri:
  36   case AArch64::ADDSXrr:
  37   case AArch64::ANDSWri:
  38   case AArch64::ANDSWrr:
  39   case AArch64::ANDSXri:
  40   case AArch64::ANDSXrr:
  41   case AArch64::SUBSWri:
  42   case AArch64::SUBSWrr:
  43   case AArch64::SUBSXri:
  44   case AArch64::SUBSXrr:
  45   case AArch64::BICSWrr:
  46   case AArch64::BICSXrr:
  47     return true;
  48   case AArch64::ADDSWrs:
  49   case AArch64::ADDSXrs:
  50   case AArch64::ANDSWrs:
  51   case AArch64::ANDSXrs:
  52   case AArch64::SUBSWrs:
  53   case AArch64::SUBSXrs:
  54   case AArch64::BICSWrs:
  55   case AArch64::BICSXrs:
  56     // Shift value can be 0 making these behave like the "rr" variant...
  57     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
  58   }
  59
  60   return false;
  61 }
  62
  63 /// ALU operations followed by CBZ/CBNZ.
  64 static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
  65                                 const MachineInstr &SecondMI) {
  66   if (SecondMI.getOpcode() != AArch64::CBZW &&
  67       SecondMI.getOpcode() != AArch64::CBZX &&
  68       SecondMI.getOpcode() != AArch64::CBNZW &&
  69       SecondMI.getOpcode() != AArch64::CBNZX)
  70     return false;
  71
  72   // Assume the 1st instr to be a wildcard if it is unspecified.
  73   if (FirstMI == nullptr)
  74     return true;
  75
  76   switch (FirstMI->getOpcode()) {
  77   case AArch64::ADDWri:
  78   case AArch64::ADDWrr:
  79   case AArch64::ADDXri:
  80   case AArch64::ADDXrr:
  81   case AArch64::ANDWri:
  82   case AArch64::ANDWrr:
  83   case AArch64::ANDXri:
  84   case AArch64::ANDXrr:
  85   case AArch64::EORWri:
  86   case AArch64::EORWrr:
  87   case AArch64::EORXri:
  88   case AArch64::EORXrr:
  89   case AArch64::ORRWri:
  90   case AArch64::ORRWrr:
  91   case AArch64::ORRXri:
  92   case AArch64::ORRXrr:
  93   case AArch64::SUBWri:
  94   case AArch64::SUBWrr:
  95   case AArch64::SUBXri:
  96   case AArch64::SUBXrr:
  97     return true;
  98   case AArch64::ADDWrs:
  99   case AArch64::ADDXrs:
 100   case AArch64::ANDWrs:
 101   case AArch64::ANDXrs:
 102   case AArch64::SUBWrs:
 103   case AArch64::SUBXrs:
 104   case AArch64::BICWrs:
 105   case AArch64::BICXrs:
 106     // Shift value can be 0 making these behave like the "rr" variant...
 107     return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 108   }
 109
 110   return false;
 111 }
 112
 113 /// AES crypto encoding or decoding.
 114 static bool isAESPair(const MachineInstr *FirstMI,
 115                       const MachineInstr &SecondMI) {
 116   // Assume the 1st instr to be a wildcard if it is unspecified.
 117   switch (SecondMI.getOpcode()) {
 118   // AES encode.
 119   case AArch64::AESMCrr:
 120   case AArch64::AESMCrrTied:
 121     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
 122   // AES decode.
 123   case AArch64::AESIMCrr:
 124   case AArch64::AESIMCrrTied:
 125     return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
 126   }
 127
 128   return false;
 129 }
 130
 131 /// AESE/AESD/PMULL + EOR.
 132 static bool isCryptoEORPair(const MachineInstr *FirstMI,
 133                             const MachineInstr &SecondMI) {
 134   if (SecondMI.getOpcode() != AArch64::EORv16i8)
 135     return false;
 136
 137   // Assume the 1st instr to be a wildcard if it is unspecified.
 138   if (FirstMI == nullptr)
 139     return true;
 140
 141   switch (FirstMI->getOpcode()) {
 142   case AArch64::AESErr:
 143   case AArch64::AESDrr:
 144   case AArch64::PMULLv16i8:
 145   case AArch64::PMULLv8i8:
 146   case AArch64::PMULLv1i64:
 147   case AArch64::PMULLv2i64:
 148     return true;
 149   }
 150
 151   return false;
 152 }
 153
 154 /// Literal generation.
 155 static bool isLiteralsPair(const MachineInstr *FirstMI,
 156                            const MachineInstr &SecondMI) {
 157   // Assume the 1st instr to be a wildcard if it is unspecified.
 158
 159   // PC relative address.
 160   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&
 161       SecondMI.getOpcode() == AArch64::ADDXri)
 162     return true;
 163
 164   // 32 bit immediate.
 165   if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&
 166       (SecondMI.getOpcode() == AArch64::MOVKWi &&
 167        SecondMI.getOperand(3).getImm() == 16))
 168     return true;
 169
 170   // Lower half of 64 bit immediate.
 171   if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&
 172      (SecondMI.getOpcode() == AArch64::MOVKXi &&
 173       SecondMI.getOperand(3).getImm() == 16))
 174     return true;
 175
 176   // Upper half of 64 bit immediate.
 177   if ((FirstMI == nullptr ||
 178        (FirstMI->getOpcode() == AArch64::MOVKXi &&
 179         FirstMI->getOperand(3).getImm() == 32)) &&
 180       (SecondMI.getOpcode() == AArch64::MOVKXi &&
 181        SecondMI.getOperand(3).getImm() == 48))
 182     return true;
 183
 184   return false;
 185 }
 186
 187 /// Fuse address generation and loads or stores.
 188 static bool isAddressLdStPair(const MachineInstr *FirstMI,
 189                               const MachineInstr &SecondMI) {
 190   switch (SecondMI.getOpcode()) {
 191   case AArch64::STRBBui:
 192   case AArch64::STRBui:
 193   case AArch64::STRDui:
 194   case AArch64::STRHHui:
 195   case AArch64::STRHui:
 196   case AArch64::STRQui:
 197   case AArch64::STRSui:
 198   case AArch64::STRWui:
 199   case AArch64::STRXui:
 200   case AArch64::LDRBBui:
 201   case AArch64::LDRBui:
 202   case AArch64::LDRDui:
 203   case AArch64::LDRHHui:
 204   case AArch64::LDRHui:
 205   case AArch64::LDRQui:
 206   case AArch64::LDRSui:
 207   case AArch64::LDRWui:
 208   case AArch64::LDRXui:
 209   case AArch64::LDRSBWui:
 210   case AArch64::LDRSBXui:
 211   case AArch64::LDRSHWui:
 212   case AArch64::LDRSHXui:
 213   case AArch64::LDRSWui:
 214     // Assume the 1st instr to be a wildcard if it is unspecified.
 215     if (FirstMI == nullptr)
 216       return true;
 217
 218    switch (FirstMI->getOpcode()) {
 219     case AArch64::ADR:
 220       return SecondMI.getOperand(2).getImm() == 0;
 221     case AArch64::ADRP:
 222       return true;
 223     }
 224   }
 225
 226   return false;
 227 }
 228
 229 /// Compare and conditional select.
 230 static bool isCCSelectPair(const MachineInstr *FirstMI,
 231                            const MachineInstr &SecondMI) {
 232   // 32 bits
 233   if (SecondMI.getOpcode() == AArch64::CSELWr) {
 234     // Assume the 1st instr to be a wildcard if it is unspecified.
 235     if (FirstMI == nullptr)
 236       return true;
 237
 238     if (FirstMI->definesRegister(AArch64::WZR))
 239       switch (FirstMI->getOpcode()) {
 240       case AArch64::SUBSWrs:
 241         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 242       case AArch64::SUBSWrx:
 243         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
 244       case AArch64::SUBSWrr:
 245       case AArch64::SUBSWri:
 246         return true;
 247       }
 248   }
 249
 250   // 64 bits
 251   if (SecondMI.getOpcode() == AArch64::CSELXr) {
 252     // Assume the 1st instr to be a wildcard if it is unspecified.
 253     if (FirstMI == nullptr)
 254       return true;
 255
 256     if (FirstMI->definesRegister(AArch64::XZR))
 257       switch (FirstMI->getOpcode()) {
 258       case AArch64::SUBSXrs:
 259         return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 260       case AArch64::SUBSXrx:
 261       case AArch64::SUBSXrx64:
 262         return !AArch64InstrInfo::hasExtendedReg(*FirstMI);
 263       case AArch64::SUBSXrr:
 264       case AArch64::SUBSXri:
 265         return true;
 266       }
 267   }
 268
 269   return false;
 270 }
 271
 272 // Arithmetic and logic.
 273 static bool isArithmeticLogicPair(const MachineInstr *FirstMI,
 274                                   const MachineInstr &SecondMI) {
 275   if (AArch64InstrInfo::hasShiftedReg(SecondMI))
 276     return false;
 277
 278   switch (SecondMI.getOpcode()) {
 279   // Arithmetic
 280   case AArch64::ADDWrr:
 281   case AArch64::ADDXrr:
 282   case AArch64::SUBWrr:
 283   case AArch64::SUBXrr:
 284   case AArch64::ADDWrs:
 285   case AArch64::ADDXrs:
 286   case AArch64::SUBWrs:
 287   case AArch64::SUBXrs:
 288   // Logic
 289   case AArch64::ANDWrr:
 290   case AArch64::ANDXrr:
 291   case AArch64::BICWrr:
 292   case AArch64::BICXrr:
 293   case AArch64::EONWrr:
 294   case AArch64::EONXrr:
 295   case AArch64::EORWrr:
 296   case AArch64::EORXrr:
 297   case AArch64::ORNWrr:
 298   case AArch64::ORNXrr:
 299   case AArch64::ORRWrr:
 300   case AArch64::ORRXrr:
 301   case AArch64::ANDWrs:
 302   case AArch64::ANDXrs:
 303   case AArch64::BICWrs:
 304   case AArch64::BICXrs:
 305   case AArch64::EONWrs:
 306   case AArch64::EONXrs:
 307   case AArch64::EORWrs:
 308   case AArch64::EORXrs:
 309   case AArch64::ORNWrs:
 310   case AArch64::ORNXrs:
 311   case AArch64::ORRWrs:
 312   case AArch64::ORRXrs:
 313     // Assume the 1st instr to be a wildcard if it is unspecified.
 314     if (FirstMI == nullptr)
 315       return true;
 316
 317     // Arithmetic
 318     switch (FirstMI->getOpcode()) {
 319     case AArch64::ADDWrr:
 320     case AArch64::ADDXrr:
 321     case AArch64::ADDSWrr:
 322     case AArch64::ADDSXrr:
 323     case AArch64::SUBWrr:
 324     case AArch64::SUBXrr:
 325     case AArch64::SUBSWrr:
 326     case AArch64::SUBSXrr:
 327       return true;
 328     case AArch64::ADDWrs:
 329     case AArch64::ADDXrs:
 330     case AArch64::ADDSWrs:
 331     case AArch64::ADDSXrs:
 332     case AArch64::SUBWrs:
 333     case AArch64::SUBXrs:
 334     case AArch64::SUBSWrs:
 335     case AArch64::SUBSXrs:
 336       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 337     }
 338     break;
 339
 340   // Arithmetic, setting flags.
 341   case AArch64::ADDSWrr:
 342   case AArch64::ADDSXrr:
 343   case AArch64::SUBSWrr:
 344   case AArch64::SUBSXrr:
 345   case AArch64::ADDSWrs:
 346   case AArch64::ADDSXrs:
 347   case AArch64::SUBSWrs:
 348   case AArch64::SUBSXrs:
 349     // Assume the 1st instr to be a wildcard if it is unspecified.
 350     if (FirstMI == nullptr)
 351       return true;
 352
 353     // Arithmetic, not setting flags.
 354     switch (FirstMI->getOpcode()) {
 355     case AArch64::ADDWrr:
 356     case AArch64::ADDXrr:
 357     case AArch64::SUBWrr:
 358     case AArch64::SUBXrr:
 359       return true;
 360     case AArch64::ADDWrs:
 361     case AArch64::ADDXrs:
 362     case AArch64::SUBWrs:
 363     case AArch64::SUBXrs:
 364       return !AArch64InstrInfo::hasShiftedReg(*FirstMI);
 365     }
 366     break;
 367   }
 368
 369   return false;
 370 }
 371
 372 /// \brief Check if the instr pair, FirstMI and SecondMI, should be fused
 373 /// together. Given SecondMI, when FirstMI is unspecified, then check if
 374 /// SecondMI may be part of a fused pair at all.
 375 static bool shouldScheduleAdjacent(const TargetInstrInfo &TII,
 376                                    const TargetSubtargetInfo &TSI,
 377                                    const MachineInstr *FirstMI,
 378                                    const MachineInstr &SecondMI) {
 379   const AArch64Subtarget &ST = static_cast<const AArch64Subtarget&>(TSI);
 380
 381   // All checking functions assume that the 1st instr is a wildcard if it is
 382   // unspecified.
 383   if (ST.hasArithmeticBccFusion() && isArithmeticBccPair(FirstMI, SecondMI))
 384     return true;
 385   if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))
 386     return true;
 387   if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))
 388     return true;
 389   if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))
 390     return true;
 391   if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))
 392     return true;
 393   if (ST.hasFuseAddress() && isAddressLdStPair(FirstMI, SecondMI))
 394     return true;
 395   if (ST.hasFuseCCSelect() && isCCSelectPair(FirstMI, SecondMI))
 396     return true;
 397   if (ST.hasFuseArithmeticLogic() && isArithmeticLogicPair(FirstMI, SecondMI))
 398     return true;
 399
 400   return false;
 401 }
 402
 403 } // end namespace
 404
 405
 406 namespace llvm {
 407
 408 std::unique_ptr<ScheduleDAGMutation> createAArch64MacroFusionDAGMutation () {
 409   return createMacroFusionDAGMutation(shouldScheduleAdjacent);
 410 }
 411
 412 } // end namespace llvm