llvm/lib/Target/AArch64/AArch64Combine.td

   1 //=- AArch64Combine.td - Define AArch64 Combine Rules ---------*-tablegen -*-=//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 //
  10 //===----------------------------------------------------------------------===//
  11
  12 include "llvm/Target/GlobalISel/Combine.td"
  13
  14 def fconstant_to_constant : GICombineRule<
  15   (defs root:$root),
  16   (match (G_FCONSTANT $dst, $src):$root,
  17          [{ return matchFConstantToConstant(*${root}, MRI); }]),
  18   (apply [{ applyFConstantToConstant(*${root}); }])>;
  19
  20 def icmp_redundant_trunc_matchdata : GIDefMatchData<"Register">;
  21 def icmp_redundant_trunc : GICombineRule<
  22   (defs root:$root, icmp_redundant_trunc_matchdata:$matchinfo),
  23   (match (G_ICMP $dst, $tst, $src1, $src2):$root,
  24          [{ return matchICmpRedundantTrunc(*${root}, MRI, Helper.getKnownBits(), ${matchinfo}); }]),
  25   (apply [{ applyICmpRedundantTrunc(*${root}, MRI, B, Observer, ${matchinfo}); }])>;
  26
  27 // AArch64-specific offset folding for G_GLOBAL_VALUE.
  28 def fold_global_offset_matchdata : GIDefMatchData<"std::pair<uint64_t, uint64_t>">;
  29 def fold_global_offset : GICombineRule<
  30   (defs root:$root, fold_global_offset_matchdata:$matchinfo),
  31   (match (wip_match_opcode G_GLOBAL_VALUE):$root,
  32           [{ return matchFoldGlobalOffset(*${root}, MRI, ${matchinfo}); }]),
  33   (apply [{ applyFoldGlobalOffset(*${root}, MRI, B, Observer, ${matchinfo});}])
  34 >;
  35
  36 // Boolean: 0 = G_ZEXT, 1 = G_SEXT
  37 def ext_addv_to_udot_addv_matchinfo : GIDefMatchData<"std::tuple<Register, Register, bool>">;
  38 let Predicates = [HasDotProd] in {
  39 def ext_addv_to_udot_addv : GICombineRule<
  40   (defs root:$root, ext_addv_to_udot_addv_matchinfo:$matchinfo),
  41   (match (wip_match_opcode G_VECREDUCE_ADD):$root,
  42          [{ return matchExtAddvToUdotAddv(*${root}, MRI, STI, ${matchinfo}); }]),
  43   (apply [{ applyExtAddvToUdotAddv(*${root}, MRI, B, Observer, STI, ${matchinfo}); }])
  44 >;
  45 }
  46
  47 def ext_uaddv_to_uaddlv_matchinfo : GIDefMatchData<"std::pair<Register, bool>">;
  48 def ext_uaddv_to_uaddlv : GICombineRule<
  49   (defs root:$root, ext_uaddv_to_uaddlv_matchinfo:$matchinfo),
  50   (match (wip_match_opcode G_VECREDUCE_ADD):$root,
  51          [{ return matchExtUaddvToUaddlv(*${root}, MRI, ${matchinfo}); }]),
  52   (apply [{ applyExtUaddvToUaddlv(*${root}, MRI, B, Observer, ${matchinfo}); }])
  53 >;
  54
  55 class push_opcode_through_ext<Instruction opcode, Instruction extOpcode> : GICombineRule <
  56   (defs root:$root),
  57   (match (extOpcode $ext1, $src1):$ExtMI,
  58          (extOpcode $ext2, $src2),
  59          (opcode $dst, $ext1, $ext2):$root,
  60          [{ return matchPushAddSubExt(*${root}, MRI, ${dst}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }]),
  61   (apply [{ applyPushAddSubExt(*${root}, MRI, B, ${ExtMI}->getOpcode() == TargetOpcode::G_SEXT, ${dst}.getReg(), ${src1}.getReg(), ${src2}.getReg()); }])>;
  62
  63 def push_sub_through_zext : push_opcode_through_ext<G_SUB, G_ZEXT>;
  64 def push_add_through_zext : push_opcode_through_ext<G_ADD, G_ZEXT>;
  65 def push_sub_through_sext : push_opcode_through_ext<G_SUB, G_SEXT>;
  66 def push_add_through_sext : push_opcode_through_ext<G_ADD, G_SEXT>;
  67
  68 def AArch64PreLegalizerCombiner: GICombiner<
  69   "AArch64PreLegalizerCombinerImpl", [all_combines,
  70                                       fconstant_to_constant,
  71                                       icmp_redundant_trunc,
  72                                       fold_global_offset,
  73                                       shuffle_to_extract,
  74                                       ext_addv_to_udot_addv,
  75                                       ext_uaddv_to_uaddlv,
  76                                       push_sub_through_zext,
  77                                       push_add_through_zext,
  78                                       push_sub_through_sext,
  79                                       push_add_through_sext]> {
  80   let CombineAllMethodName = "tryCombineAllImpl";
  81 }
  82
  83 def AArch64O0PreLegalizerCombiner: GICombiner<
  84   "AArch64O0PreLegalizerCombinerImpl", [optnone_combines]> {
  85   let CombineAllMethodName = "tryCombineAllImpl";
  86 }
  87
  88 // Matchdata for combines which replace a G_SHUFFLE_VECTOR with a
  89 // target-specific opcode.
  90 def shuffle_matchdata : GIDefMatchData<"ShuffleVectorPseudo">;
  91
  92 def rev : GICombineRule<
  93   (defs root:$root, shuffle_matchdata:$matchinfo),
  94   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
  95          [{ return matchREV(*${root}, MRI, ${matchinfo}); }]),
  96   (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])
  97 >;
  98
  99 def zip : GICombineRule<
 100   (defs root:$root, shuffle_matchdata:$matchinfo),
 101   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 102          [{ return matchZip(*${root}, MRI, ${matchinfo}); }]),
 103   (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])
 104 >;
 105
 106 def uzp : GICombineRule<
 107   (defs root:$root, shuffle_matchdata:$matchinfo),
 108   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 109          [{ return matchUZP(*${root}, MRI, ${matchinfo}); }]),
 110   (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])
 111 >;
 112
 113 def dup: GICombineRule <
 114   (defs root:$root, shuffle_matchdata:$matchinfo),
 115   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 116          [{ return matchDup(*${root}, MRI, ${matchinfo}); }]),
 117   (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])
 118 >;
 119
 120 def trn : GICombineRule<
 121   (defs root:$root, shuffle_matchdata:$matchinfo),
 122   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 123          [{ return matchTRN(*${root}, MRI, ${matchinfo}); }]),
 124   (apply [{ applyShuffleVectorPseudo(*${root}, ${matchinfo}); }])
 125 >;
 126
 127 def ext: GICombineRule <
 128   (defs root:$root, shuffle_matchdata:$matchinfo),
 129   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 130          [{ return matchEXT(*${root}, MRI, ${matchinfo}); }]),
 131   (apply [{ applyEXT(*${root}, ${matchinfo}); }])
 132 >;
 133
 134 def fullrev: GICombineRule <
 135   (defs root:$root, shuffle_matchdata:$matchinfo),
 136   (match (G_IMPLICIT_DEF $src2),
 137          (G_SHUFFLE_VECTOR $src, $src1, $src2, $mask):$root,
 138          [{ return ShuffleVectorInst::isReverseMask(${mask}.getShuffleMask(),
 139                                                     ${mask}.getShuffleMask().size()); }]),
 140   (apply [{ applyFullRev(*${root}, MRI); }])
 141 >;
 142
 143 def insertelt_nonconst: GICombineRule <
 144   (defs root:$root, shuffle_matchdata:$matchinfo),
 145   (match (wip_match_opcode G_INSERT_VECTOR_ELT):$root,
 146          [{ return matchNonConstInsert(*${root}, MRI); }]),
 147   (apply [{ applyNonConstInsert(*${root}, MRI, B); }])
 148 >;
 149
 150 def shuf_to_ins_matchdata : GIDefMatchData<"std::tuple<Register, int, Register, int>">;
 151 def shuf_to_ins: GICombineRule <
 152   (defs root:$root, shuf_to_ins_matchdata:$matchinfo),
 153   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 154          [{ return matchINS(*${root}, MRI, ${matchinfo}); }]),
 155   (apply [{ applyINS(*${root}, MRI, B, ${matchinfo}); }])
 156 >;
 157
 158 def vashr_vlshr_imm_matchdata : GIDefMatchData<"int64_t">;
 159 def vashr_vlshr_imm : GICombineRule<
 160   (defs root:$root, vashr_vlshr_imm_matchdata:$matchinfo),
 161   (match (wip_match_opcode G_ASHR, G_LSHR):$root,
 162           [{ return matchVAshrLshrImm(*${root}, MRI, ${matchinfo}); }]),
 163   (apply [{ applyVAshrLshrImm(*${root}, MRI, ${matchinfo}); }])
 164 >;
 165
 166 def form_duplane_matchdata :
 167   GIDefMatchData<"std::pair<unsigned, int>">;
 168 def form_duplane : GICombineRule <
 169   (defs root:$root, form_duplane_matchdata:$matchinfo),
 170   (match (wip_match_opcode G_SHUFFLE_VECTOR):$root,
 171           [{ return matchDupLane(*${root}, MRI, ${matchinfo}); }]),
 172   (apply [{ applyDupLane(*${root}, MRI, B, ${matchinfo}); }])
 173 >;
 174
 175 def shuffle_vector_lowering : GICombineGroup<[dup, rev, ext, zip, uzp, trn, fullrev,
 176                                               form_duplane, shuf_to_ins]>;
 177
 178 // Turn G_UNMERGE_VALUES -> G_EXTRACT_VECTOR_ELT's
 179 def vector_unmerge_lowering : GICombineRule <
 180   (defs root:$root),
 181   (match (wip_match_opcode G_UNMERGE_VALUES):$root,
 182           [{ return matchScalarizeVectorUnmerge(*${root}, MRI); }]),
 183   (apply [{ applyScalarizeVectorUnmerge(*${root}, MRI, B); }])
 184 >;
 185
 186 def adjust_icmp_imm_matchdata :
 187   GIDefMatchData<"std::pair<uint64_t, CmpInst::Predicate>">;
 188 def adjust_icmp_imm : GICombineRule <
 189   (defs root:$root, adjust_icmp_imm_matchdata:$matchinfo),
 190   (match (G_ICMP $dst, $tst, $src1, $src2):$root,
 191           [{ return matchAdjustICmpImmAndPred(*${root}, MRI, ${matchinfo}); }]),
 192   (apply [{ applyAdjustICmpImmAndPred(*${root}, ${matchinfo}, B, Observer); }])
 193 >;
 194
 195 def swap_icmp_operands : GICombineRule <
 196   (defs root:$root),
 197   (match (G_ICMP $dst, $tst, $src1, $src2):$root,
 198           [{ return trySwapICmpOperands(*${root}, MRI); }]),
 199   (apply [{ applySwapICmpOperands(*${root}, Observer); }])
 200 >;
 201
 202 def icmp_lowering : GICombineGroup<[adjust_icmp_imm, swap_icmp_operands]>;
 203
 204 def extractvecelt_pairwise_add_matchdata : GIDefMatchData<"std::tuple<unsigned, LLT, Register>">;
 205 def extractvecelt_pairwise_add : GICombineRule<
 206   (defs root:$root, extractvecelt_pairwise_add_matchdata:$matchinfo),
 207   (match (wip_match_opcode G_EXTRACT_VECTOR_ELT):$root,
 208           [{ return matchExtractVecEltPairwiseAdd(*${root}, MRI, ${matchinfo}); }]),
 209   (apply [{ applyExtractVecEltPairwiseAdd(*${root}, MRI, B, ${matchinfo}); }])
 210 >;
 211
 212 def mul_const_matchdata : GIDefMatchData<"std::function<void(MachineIRBuilder&, Register)>">;
 213 def mul_const : GICombineRule<
 214   (defs root:$root, mul_const_matchdata:$matchinfo),
 215   (match (wip_match_opcode G_MUL):$root,
 216           [{ return matchAArch64MulConstCombine(*${root}, MRI, ${matchinfo}); }]),
 217   (apply [{ applyAArch64MulConstCombine(*${root}, MRI, B, ${matchinfo}); }])
 218 >;
 219
 220 def lower_mull : GICombineRule<
 221   (defs root:$root),
 222   (match (wip_match_opcode G_MUL):$root,
 223           [{ return matchExtMulToMULL(*${root}, MRI); }]),
 224   (apply [{ applyExtMulToMULL(*${root}, MRI, B, Observer); }])
 225 >;
 226
 227 def build_vector_to_dup : GICombineRule<
 228   (defs root:$root),
 229   (match (wip_match_opcode G_BUILD_VECTOR):$root,
 230           [{ return matchBuildVectorToDup(*${root}, MRI); }]),
 231   (apply [{ applyBuildVectorToDup(*${root}, MRI, B); }])
 232 >;
 233
 234 def build_vector_to_vector_insert : GICombineRule<
 235   (defs root:$root),
 236   (match (G_BUILD_VECTOR $dst, GIVariadic<>:$unused):$root,
 237           [{ return matchLowerBuildToInsertVecElt(*${root}, MRI); }]),
 238   (apply [{ applyLowerBuildToInsertVecElt(*${root}, MRI, B); }])
 239 >;
 240
 241 def build_vector_lowering : GICombineGroup<[build_vector_to_dup,
 242                                             build_vector_to_vector_insert]>;
 243
 244 def lower_vector_fcmp : GICombineRule<
 245   (defs root:$root),
 246   (match (G_FCMP $dst, $tst, $src1, $src2):$root,
 247     [{ return matchLowerVectorFCMP(*${root}, MRI, B); }]),
 248   (apply [{ applyLowerVectorFCMP(*${root}, MRI, B); }])>;
 249
 250 def form_truncstore_matchdata : GIDefMatchData<"Register">;
 251 def form_truncstore : GICombineRule<
 252   (defs root:$root, form_truncstore_matchdata:$matchinfo),
 253   (match (G_STORE $src, $addr):$root,
 254           [{ return matchFormTruncstore(*${root}, MRI, ${matchinfo}); }]),
 255   (apply [{ applyFormTruncstore(*${root}, MRI, B, Observer, ${matchinfo}); }])
 256 >;
 257
 258 def fold_merge_to_zext : GICombineRule<
 259   (defs root:$d),
 260   (match (wip_match_opcode G_MERGE_VALUES):$d,
 261           [{ return matchFoldMergeToZext(*${d}, MRI); }]),
 262   (apply [{ applyFoldMergeToZext(*${d}, MRI, B, Observer); }])
 263 >;
 264
 265 def mutate_anyext_to_zext : GICombineRule<
 266   (defs root:$d),
 267   (match (wip_match_opcode G_ANYEXT):$d,
 268           [{ return matchMutateAnyExtToZExt(*${d}, MRI); }]),
 269   (apply [{ applyMutateAnyExtToZExt(*${d}, MRI, B, Observer); }])
 270 >;
 271
 272 def split_store_zero_128 : GICombineRule<
 273   (defs root:$d),
 274   (match (G_STORE $src, $addr):$d,
 275           [{ return matchSplitStoreZero128(*${d}, MRI); }]),
 276   (apply [{ applySplitStoreZero128(*${d}, MRI, B, Observer); }])
 277 >;
 278
 279 def vector_sext_inreg_to_shift : GICombineRule<
 280   (defs root:$d),
 281   (match (wip_match_opcode G_SEXT_INREG):$d,
 282           [{ return matchVectorSextInReg(*${d}, MRI); }]),
 283   (apply [{ applyVectorSextInReg(*${d}, MRI, B, Observer); }])
 284 >;
 285
 286 def unmerge_ext_to_unmerge_matchdata : GIDefMatchData<"Register">;
 287 def unmerge_ext_to_unmerge : GICombineRule<
 288   (defs root:$d, unmerge_ext_to_unmerge_matchdata:$matchinfo),
 289   (match (wip_match_opcode G_UNMERGE_VALUES):$d,
 290           [{ return matchUnmergeExtToUnmerge(*${d}, MRI, ${matchinfo}); }]),
 291   (apply [{ applyUnmergeExtToUnmerge(*${d}, MRI, B, Observer, ${matchinfo}); }])
 292 >;
 293
 294 def regtriple_matchdata : GIDefMatchData<"std::tuple<Register, Register, Register>">;
 295 def or_to_bsp: GICombineRule <
 296   (defs root:$root, regtriple_matchdata:$matchinfo),
 297   (match (G_OR $dst, $src1, $src2):$root,
 298          [{ return matchOrToBSP(*${root}, MRI, ${matchinfo}); }]),
 299   (apply [{ applyOrToBSP(*${root}, MRI, B, ${matchinfo}); }])
 300 >;
 301
 302 // Combines Mul(And(Srl(X, 15), 0x10001), 0xffff) into CMLTz
 303 def combine_mul_cmlt : GICombineRule<
 304   (defs root:$root, register_matchinfo:$matchinfo),
 305   (match (wip_match_opcode G_MUL):$root,
 306         [{ return matchCombineMulCMLT(*${root}, MRI, ${matchinfo}); }]),
 307   (apply [{ applyCombineMulCMLT(*${root}, MRI, B, ${matchinfo}); }])
 308 >;
 309
 310 // Post-legalization combines which should happen at all optimization levels.
 311 // (E.g. ones that facilitate matching for the selector) For example, matching
 312 // pseudos.
 313 def AArch64PostLegalizerLowering
 314     : GICombiner<"AArch64PostLegalizerLoweringImpl",
 315                        [shuffle_vector_lowering, vashr_vlshr_imm,
 316                         icmp_lowering, build_vector_lowering,
 317                         lower_vector_fcmp, form_truncstore,
 318                         vector_sext_inreg_to_shift,
 319                         unmerge_ext_to_unmerge, lower_mull,
 320                         vector_unmerge_lowering, insertelt_nonconst]> {
 321 }
 322
 323 // Post-legalization combines which are primarily optimizations.
 324 def AArch64PostLegalizerCombiner
 325     : GICombiner<"AArch64PostLegalizerCombinerImpl",
 326                        [copy_prop, cast_of_cast_combines, buildvector_of_truncate,
 327                         integer_of_truncate, mutate_anyext_to_zext,
 328                         combines_for_extload, combine_indexed_load_store, sext_trunc_sextload,
 329                         hoist_logic_op_with_same_opcode_hands,
 330                         redundant_and, xor_of_and_with_same_reg,
 331                         extractvecelt_pairwise_add, redundant_or,
 332                         mul_const, redundant_sext_inreg,
 333                         form_bitfield_extract, rotate_out_of_range,
 334                         icmp_to_true_false_known_bits, overflow_combines,
 335                         select_combines, fold_merge_to_zext, merge_combines,
 336                         constant_fold_binops, identity_combines,
 337                         ptr_add_immed_chain, overlapping_and,
 338                         split_store_zero_128, undef_combines,
 339                         select_to_minmax, or_to_bsp, combine_concat_vector,
 340                         commute_constant_to_rhs,
 341                         push_freeze_to_prevent_poison_from_propagating,
 342                         combine_mul_cmlt, combine_use_vector_truncate]> {
 343 }