llvm/lib/Transforms/Vectorize/VPlanTransforms.h

   1 //===- VPlanTransforms.h - Utility VPlan to VPlan transforms --------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file provides utility VPlan to VPlan transformations.
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
  14 #define LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H
  15
  16 #include "VPlan.h"
  17 #include "llvm/ADT/STLFunctionalExtras.h"
  18
  19 namespace llvm {
  20
  21 class InductionDescriptor;
  22 class Instruction;
  23 class PHINode;
  24 class ScalarEvolution;
  25 class PredicatedScalarEvolution;
  26 class TargetLibraryInfo;
  27 class VPBuilder;
  28 class VPRecipeBuilder;
  29
  30 struct VPlanTransforms {
  31   /// Replaces the VPInstructions in \p Plan with corresponding
  32   /// widen recipes.
  33   static void
  34   VPInstructionsToVPRecipes(VPlanPtr &Plan,
  35                             function_ref<const InductionDescriptor *(PHINode *)>
  36                                 GetIntOrFpInductionDescriptor,
  37                             ScalarEvolution &SE, const TargetLibraryInfo &TLI);
  38
  39   /// Try to have all users of fixed-order recurrences appear after the recipe
  40   /// defining their previous value, by either sinking users or hoisting recipes
  41   /// defining their previous value (and its operands). Then introduce
  42   /// FirstOrderRecurrenceSplice VPInstructions to combine the value from the
  43   /// recurrence phis and previous values.
  44   /// \returns true if all users of fixed-order recurrences could be re-arranged
  45   /// as needed or false if it is not possible. In the latter case, \p Plan is
  46   /// not valid.
  47   static bool adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder);
  48
  49   /// Clear NSW/NUW flags from reduction instructions if necessary.
  50   static void clearReductionWrapFlags(VPlan &Plan);
  51
  52   /// Explicitly unroll \p Plan by \p UF.
  53   static void unrollByUF(VPlan &Plan, unsigned UF, LLVMContext &Ctx);
  54
  55   /// Optimize \p Plan based on \p BestVF and \p BestUF. This may restrict the
  56   /// resulting plan to \p BestVF and \p BestUF.
  57   static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF,
  58                                  unsigned BestUF,
  59                                  PredicatedScalarEvolution &PSE);
  60
  61   /// Apply VPlan-to-VPlan optimizations to \p Plan, including induction recipe
  62   /// optimizations, dead recipe removal, replicate region optimizations and
  63   /// block merging.
  64   static void optimize(VPlan &Plan);
  65
  66   /// Wrap predicated VPReplicateRecipes with a mask operand in an if-then
  67   /// region block and remove the mask operand. Optimize the created regions by
  68   /// iteratively sinking scalar operands into the region, followed by merging
  69   /// regions until no improvements are remaining.
  70   static void createAndOptimizeReplicateRegions(VPlan &Plan);
  71
  72   /// Replace (ICMP_ULE, wide canonical IV, backedge-taken-count) checks with an
  73   /// (active-lane-mask recipe, wide canonical IV, trip-count). If \p
  74   /// UseActiveLaneMaskForControlFlow is true, introduce an
  75   /// VPActiveLaneMaskPHIRecipe. If \p DataAndControlFlowWithoutRuntimeCheck is
  76   /// true, no minimum-iteration runtime check will be created (during skeleton
  77   /// creation) and instead it is handled using active-lane-mask. \p
  78   /// DataAndControlFlowWithoutRuntimeCheck implies \p
  79   /// UseActiveLaneMaskForControlFlow.
  80   static void addActiveLaneMask(VPlan &Plan,
  81                                 bool UseActiveLaneMaskForControlFlow,
  82                                 bool DataAndControlFlowWithoutRuntimeCheck);
  83
  84   /// Insert truncates and extends for any truncated recipe. Redundant casts
  85   /// will be folded later.
  86   static void
  87   truncateToMinimalBitwidths(VPlan &Plan,
  88                              const MapVector<Instruction *, uint64_t> &MinBWs);
  89
  90   /// Drop poison flags from recipes that may generate a poison value that is
  91   /// used after vectorization, even when their operands are not poison. Those
  92   /// recipes meet the following conditions:
  93   ///  * Contribute to the address computation of a recipe generating a widen
  94   ///    memory load/store (VPWidenMemoryInstructionRecipe or
  95   ///    VPInterleaveRecipe).
  96   ///  * Such a widen memory load/store has at least one underlying Instruction
  97   ///    that is in a basic block that needs predication and after vectorization
  98   ///    the generated instruction won't be predicated.
  99   /// Uses \p BlockNeedsPredication to check if a block needs predicating.
 100   /// TODO: Replace BlockNeedsPredication callback with retrieving info from
 101   ///       VPlan directly.
 102   static void dropPoisonGeneratingRecipes(
 103       VPlan &Plan, function_ref<bool(BasicBlock *)> BlockNeedsPredication);
 104
 105   /// Add a VPEVLBasedIVPHIRecipe and related recipes to \p Plan and
 106   /// replaces all uses except the canonical IV increment of
 107   /// VPCanonicalIVPHIRecipe with a VPEVLBasedIVPHIRecipe.
 108   /// VPCanonicalIVPHIRecipe is only used to control the loop after
 109   /// this transformation.
 110   /// \returns true if the transformation succeeds, or false if it doesn't.
 111   static bool
 112   tryAddExplicitVectorLength(VPlan &Plan,
 113                              const std::optional<unsigned> &MaxEVLSafeElements);
 114
 115   // For each Interleave Group in \p InterleaveGroups replace the Recipes
 116   // widening its memory instructions with a single VPInterleaveRecipe at its
 117   // insertion point.
 118   static void createInterleaveGroups(
 119       VPlan &Plan,
 120       const SmallPtrSetImpl<const InterleaveGroup<Instruction> *>
 121           &InterleaveGroups,
 122       VPRecipeBuilder &RecipeBuilder, bool ScalarEpilogueAllowed);
 123
 124   /// Remove dead recipes from \p Plan.
 125   static void removeDeadRecipes(VPlan &Plan);
 126
 127   /// Update \p Plan to account for the uncountable early exit block in \p
 128   /// UncountableExitingBlock by
 129   ///  * updating the condition exiting the vector loop to include the early
 130   ///    exit conditions
 131   ///  * splitting the original middle block to branch to the early exit block
 132   ///    if taken.
 133   static void handleUncountableEarlyExit(VPlan &Plan, ScalarEvolution &SE,
 134                                          Loop *OrigLoop,
 135                                          BasicBlock *UncountableExitingBlock,
 136                                          VPRecipeBuilder &RecipeBuilder);
 137
 138   /// Lower abstract recipes to concrete ones, that can be codegen'd.
 139   static void convertToConcreteRecipes(VPlan &Plan);
 140 };
 141
 142 } // namespace llvm
 143
 144 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLANTRANSFORMS_H