gcc/gimple-isel.cc

   1 /* Schedule GIMPLE vector statements.
   2    Copyright (C) 2020-2025 Free Software Foundation, Inc.
   3
   4 This file is part of GCC.
   5
   6 GCC is free software; you can redistribute it and/or modify it
   7 under the terms of the GNU General Public License as published by the
   8 Free Software Foundation; either version 3, or (at your option) any
   9 later version.
  10
  11 GCC is distributed in the hope that it will be useful, but WITHOUT
  12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14 for more details.
  15
  16 You should have received a copy of the GNU General Public License
  17 along with GCC; see the file COPYING3.  If not see
  18 <http://www.gnu.org/licenses/>.  */
  19
  20 #include "config.h"
  21 #include "system.h"
  22 #include "coretypes.h"
  23 #include "backend.h"
  24 #include "rtl.h"
  25 #include "tree.h"
  26 #include "gimple.h"
  27 #include "tree-pass.h"
  28 #include "ssa.h"
  29 #include "expmed.h"
  30 #include "optabs-tree.h"
  31 #include "tree-eh.h"
  32 #include "gimple-iterator.h"
  33 #include "gimplify-me.h"
  34 #include "gimplify.h"
  35 #include "tree-cfg.h"
  36 #include "bitmap.h"
  37 #include "tree-ssa-dce.h"
  38 #include "memmodel.h"
  39 #include "optabs.h"
  40 #include "gimple-fold.h"
  41 #include "internal-fn.h"
  42
  43 /* Expand all ARRAY_REF(VIEW_CONVERT_EXPR) gimple assignments into calls to
  44    internal function based on vector type of selected expansion.
  45
  46    For vec_set:
  47
  48      VIEW_CONVERT_EXPR<int[4]>(u)[_1] = i_4(D);
  49    =>
  50      _7 = u;
  51      _8 = .VEC_SET (_7, i_4(D), _1);
  52      u = _8;
  53
  54    For vec_extract:
  55
  56       _3 = VIEW_CONVERT_EXPR<intD.1[4]>(vD.2208)[idx_2(D)];
  57    =>
  58       _4 = vD.2208;
  59       _3 = .VEC_EXTRACT (_4, idx_2(D));  */
  60
  61 static bool
  62 gimple_expand_vec_set_extract_expr (struct function *fun,
  63                                     gimple_stmt_iterator *gsi)
  64 {
  65   gcall *new_stmt = NULL;
  66   gassign *ass_stmt = NULL;
  67   bool cfg_changed = false;
  68
  69   /* Only consider code == GIMPLE_ASSIGN.  */
  70   gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
  71   if (!stmt)
  72     return false;
  73
  74   bool is_extract = false;
  75
  76   tree lhs = gimple_assign_lhs (stmt);
  77   tree rhs = gimple_assign_rhs1 (stmt);
  78   tree val, ref;
  79   if (TREE_CODE (lhs) == ARRAY_REF)
  80     {
  81       /* Assume it is a vec_set.  */
  82       val = rhs;
  83       ref = lhs;
  84     }
  85   else if (TREE_CODE (rhs) == ARRAY_REF)
  86     {
  87       /* vec_extract.  */
  88       is_extract = true;
  89       val = lhs;
  90       ref = rhs;
  91     }
  92   else
  93     return false;
  94
  95   tree op0 = TREE_OPERAND (ref, 0);
  96   if (TREE_CODE (op0) == VIEW_CONVERT_EXPR && DECL_P (TREE_OPERAND (op0, 0))
  97       && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (op0, 0)))
  98       && TYPE_MODE (TREE_TYPE (ref))
  99            == TYPE_MODE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (op0, 0)))))
 100     {
 101       tree pos = TREE_OPERAND (ref, 1);
 102
 103       tree view_op0 = TREE_OPERAND (op0, 0);
 104       machine_mode outermode = TYPE_MODE (TREE_TYPE (view_op0));
 105       machine_mode extract_mode = TYPE_MODE (TREE_TYPE (ref));
 106
 107       if ((auto_var_in_fn_p (view_op0, fun->decl)
 108            || (VAR_P (view_op0) && DECL_HARD_REGISTER (view_op0)))
 109           && !TREE_ADDRESSABLE (view_op0)
 110           && ((!is_extract && can_vec_set_var_idx_p (outermode))
 111               || (is_extract
 112                   && can_vec_extract_var_idx_p (outermode, extract_mode))))
 113         {
 114           location_t loc = gimple_location (stmt);
 115           tree var_src = make_ssa_name (TREE_TYPE (view_op0));
 116
 117           ass_stmt = gimple_build_assign (var_src, view_op0);
 118           gimple_set_vuse (ass_stmt, gimple_vuse (stmt));
 119           gimple_set_location (ass_stmt, loc);
 120           gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
 121
 122           if (!is_extract)
 123             {
 124               tree var_dst = make_ssa_name (TREE_TYPE (view_op0));
 125
 126               new_stmt = gimple_build_call_internal (IFN_VEC_SET, 3, var_src,
 127                                                      val, pos);
 128
 129               gimple_call_set_lhs (new_stmt, var_dst);
 130               gimple_set_location (new_stmt, loc);
 131               gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
 132
 133               ass_stmt = gimple_build_assign (view_op0, var_dst);
 134               gimple_set_location (ass_stmt, loc);
 135               gimple_move_vops (ass_stmt, stmt);
 136               gsi_insert_before (gsi, ass_stmt, GSI_SAME_STMT);
 137
 138               basic_block bb = gimple_bb (stmt);
 139               if (gsi_remove (gsi, true)
 140                   && gimple_purge_dead_eh_edges (bb))
 141                 cfg_changed = true;
 142               *gsi = gsi_for_stmt (ass_stmt);
 143             }
 144           else
 145             {
 146               new_stmt
 147                 = gimple_build_call_internal (IFN_VEC_EXTRACT, 2, var_src, pos);
 148               gimple_call_set_lhs (new_stmt, lhs);
 149
 150               gsi_replace (gsi, new_stmt, true);
 151               cfg_changed = true;
 152             }
 153         }
 154     }
 155
 156   return cfg_changed;
 157 }
 158
 159 /* Expand all VEC_COND_EXPR gimple assignments into calls to internal
 160    function based on type of selected expansion.  */
 161
 162 static gimple *
 163 gimple_expand_vec_cond_expr (gimple_stmt_iterator *gsi)
 164 {
 165   tree lhs, op0a = NULL_TREE;
 166   enum tree_code code;
 167   enum tree_code tcode;
 168
 169   /* Only consider code == GIMPLE_ASSIGN.  */
 170   gassign *stmt = dyn_cast<gassign *> (gsi_stmt (*gsi));
 171   if (!stmt)
 172     return NULL;
 173
 174   code = gimple_assign_rhs_code (stmt);
 175   if (code != VEC_COND_EXPR)
 176     return NULL;
 177
 178   tree op0 = gimple_assign_rhs1 (stmt);
 179   tree op1 = gimple_assign_rhs2 (stmt);
 180   tree op2 = gimple_assign_rhs3 (stmt);
 181   lhs = gimple_assign_lhs (stmt);
 182   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
 183
 184   /* Lower mask typed, non-vector mode VEC_COND_EXPRs to bitwise operations.
 185      Those can end up generated by folding and at least for integer mode masks
 186      we cannot expect vcond expanders to exist.  We lower a ? b : c
 187      to (b & a) | (c & ~a).  */
 188   if (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (lhs))
 189       && !VECTOR_MODE_P (mode))
 190     {
 191       gcc_assert (types_compatible_p (TREE_TYPE (op0), TREE_TYPE (op1)));
 192       gimple_seq stmts = NULL;
 193       tree type = TREE_TYPE (lhs);
 194       location_t loc = gimple_location (stmt);
 195       tree tem0 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op1, op0);
 196       tree tem1 = gimple_build (&stmts, loc, BIT_NOT_EXPR, type, op0);
 197       tree tem2 = gimple_build (&stmts, loc, BIT_AND_EXPR, type, op2, tem1);
 198       tree tem3 = gimple_build (&stmts, loc, BIT_IOR_EXPR, type, tem0, tem2);
 199       gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
 200       return gimple_build_assign (lhs, tem3);
 201     }
 202
 203   bool can_compute_op0 = true;
 204   gcc_assert (!COMPARISON_CLASS_P (op0));
 205   if (TREE_CODE (op0) == SSA_NAME)
 206     {
 207       gassign *def_stmt = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (op0));
 208       if (def_stmt)
 209         {
 210           tcode = gimple_assign_rhs_code (def_stmt);
 211           op0a = gimple_assign_rhs1 (def_stmt);
 212
 213           tree op0_type = TREE_TYPE (op0);
 214           tree op0a_type = TREE_TYPE (op0a);
 215           if (TREE_CODE_CLASS (tcode) == tcc_comparison)
 216             can_compute_op0 = expand_vec_cmp_expr_p (op0a_type, op0_type,
 217                                                      tcode);
 218           gcc_assert (can_compute_op0);
 219
 220           if (can_compute_op0
 221               && TYPE_MODE (TREE_TYPE (lhs)) == TYPE_MODE (TREE_TYPE (op0)))
 222             {
 223               /* Assuming c = x CMP y.  */
 224               bool op1_minus_onep = integer_minus_onep (op1);
 225               bool op2_zerop = integer_zerop (op2);
 226               tree vtype = TREE_TYPE (lhs);
 227               machine_mode vmode = TYPE_MODE (vtype);
 228               /* Try to fold r = c ? -1 : 0 to r = c.  */
 229               if (op1_minus_onep && op2_zerop)
 230                 {
 231                   tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
 232                   return gimple_build_assign (lhs, conv_op);
 233                 }
 234               /* Try to fold r = c ? -1 : z to r = c | z, or
 235                  r = c ? c : z.  */
 236               if (op1_minus_onep)
 237                 {
 238                   tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
 239                   tree new_op1 = make_ssa_name (vtype);
 240                   gassign *new_stmt = gimple_build_assign (new_op1, conv_op);
 241                   gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
 242                   if (optab_handler (ior_optab, vmode) != CODE_FOR_nothing)
 243                     /* r = c | z */
 244                     return gimple_build_assign (lhs, BIT_IOR_EXPR, new_op1,
 245                                                 op2);
 246                   /* r = c ? c : z */
 247                   op1 = new_op1;
 248                 }
 249               /* Try to fold r = c ? z : 0 to r = c & z, or
 250                  r = c ? z : c.  */
 251               else if (op2_zerop)
 252                 {
 253                   tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
 254                   tree new_op2 = make_ssa_name (vtype);
 255                   gassign *new_stmt = gimple_build_assign (new_op2, conv_op);
 256                   gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
 257                   if (optab_handler (and_optab, vmode) != CODE_FOR_nothing)
 258                     /* r = c | z */
 259                     return gimple_build_assign (lhs, BIT_AND_EXPR, new_op2,
 260                                                 op1);
 261                   /* r = c ? z : c */
 262                   op2 = new_op2;
 263                 }
 264               bool op1_zerop = integer_zerop (op1);
 265               bool op2_minus_onep = integer_minus_onep (op2);
 266               /* Try to fold r = c ? 0 : z to r = .BIT_ANDN (z, c).  */
 267               if (op1_zerop
 268                   && (direct_internal_fn_supported_p (IFN_BIT_ANDN, vtype,
 269                                                       OPTIMIZE_FOR_BOTH)))
 270                 {
 271                   tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
 272                   tree new_op = make_ssa_name (vtype);
 273                   gassign *new_stmt = gimple_build_assign (new_op, conv_op);
 274                   gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
 275                   return gimple_build_call_internal (IFN_BIT_ANDN, 2, op2,
 276                                                      new_op);
 277                 }
 278               /* Try to fold r = c ? z : -1 to r = .BIT_IORN (z, c).  */
 279               else if (op2_minus_onep
 280                        && (direct_internal_fn_supported_p (IFN_BIT_IORN, vtype,
 281                                                            OPTIMIZE_FOR_BOTH)))
 282                 {
 283                   tree conv_op = build1 (VIEW_CONVERT_EXPR, vtype, op0);
 284                   tree new_op = make_ssa_name (vtype);
 285                   gassign *new_stmt = gimple_build_assign (new_op, conv_op);
 286                   gsi_insert_seq_before (gsi, new_stmt, GSI_SAME_STMT);
 287                   return gimple_build_call_internal (IFN_BIT_IORN, 2, op1,
 288                                                      new_op);
 289                 }
 290             }
 291         }
 292     }
 293
 294   gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
 295   gcc_assert (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
 296               != CODE_FOR_nothing);
 297   return gimple_build_call_internal (IFN_VCOND_MASK, 3, op0, op1, op2);
 298 }
 299
 300 /* Duplicate COND_EXPR condition defs of STMT located in BB when they are
 301    comparisons so RTL expansion with the help of TER
 302    can perform better if conversion.  */
 303 static void
 304 maybe_duplicate_comparison (gassign *stmt, basic_block bb)
 305 {
 306   imm_use_iterator imm_iter;
 307   use_operand_p use_p;
 308   auto_vec<gassign *, 4> cond_exprs;
 309   tree lhs = gimple_assign_lhs (stmt);
 310   unsigned cnt = 0;
 311
 312   /* This is should not be used for -O0 nor it is not useful
 313      when ter is turned off. */
 314   if (!optimize || !flag_tree_ter)
 315     return;
 316
 317   FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
 318     {
 319       if (is_gimple_debug (USE_STMT (use_p)))
 320         continue;
 321       cnt++;
 322       /* Add the use statement if it was a cond_expr.  */
 323       if (gimple_bb (USE_STMT (use_p)) == bb
 324           && is_gimple_assign (USE_STMT (use_p))
 325           && gimple_assign_rhs_code (USE_STMT (use_p)) == COND_EXPR
 326           && gimple_assign_rhs1_ptr (USE_STMT (use_p)) == use_p->use)
 327         cond_exprs.safe_push (as_a <gassign *> (USE_STMT (use_p)));
 328     }
 329
 330   /* If the comparison has 0 or 1 uses, no reason to do anything. */
 331   if (cnt <= 1)
 332     return;
 333
 334   /* If we only use the expression inside cond_exprs in that BB, we don't
 335      need to duplicate for one of them so pop the top. */
 336   if (cond_exprs.length () == cnt)
 337     cond_exprs.pop();
 338
 339   while (!cond_exprs.is_empty())
 340     {
 341       auto old_top = cond_exprs.pop();
 342       gassign *copy = as_a <gassign *> (gimple_copy (stmt));
 343       tree new_def = duplicate_ssa_name (lhs, copy);
 344       gimple_assign_set_lhs (copy, new_def);
 345       auto gsi2 = gsi_for_stmt (old_top);
 346       gsi_insert_before (&gsi2, copy, GSI_SAME_STMT);
 347       gimple_assign_set_rhs1 (old_top, new_def);
 348       update_stmt (old_top);
 349     }
 350 }
 351
 352
 353 namespace {
 354
 355 const pass_data pass_data_gimple_isel =
 356 {
 357   GIMPLE_PASS, /* type */
 358   "isel", /* name */
 359   OPTGROUP_VEC, /* optinfo_flags */
 360   TV_NONE, /* tv_id */
 361   PROP_cfg, /* properties_required */
 362   0, /* properties_provided */
 363   0, /* properties_destroyed */
 364   0, /* todo_flags_start */
 365   TODO_update_ssa, /* todo_flags_finish */
 366 };
 367
 368 class pass_gimple_isel : public gimple_opt_pass
 369 {
 370 public:
 371   pass_gimple_isel (gcc::context *ctxt)
 372     : gimple_opt_pass (pass_data_gimple_isel, ctxt)
 373   {}
 374
 375   /* opt_pass methods: */
 376   bool gate (function *) final override
 377     {
 378       return true;
 379     }
 380
 381   unsigned int execute (function *fun) final override;
 382 }; // class pass_gimple_isel
 383
 384
 385 /* Iterate all gimple statements and perform pre RTL expansion
 386    GIMPLE massaging to improve instruction selection.  */
 387
 388 unsigned int
 389 pass_gimple_isel::execute (struct function *fun)
 390 {
 391   gimple_stmt_iterator gsi;
 392   basic_block bb;
 393   bool cfg_changed = false;
 394
 395   FOR_EACH_BB_FN (bb, fun)
 396     {
 397       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
 398         {
 399           /* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
 400              calls mapping to supported optabs.  */
 401           gimple *g = gimple_expand_vec_cond_expr (&gsi);
 402           if (g != NULL)
 403             {
 404               tree lhs = gimple_assign_lhs (gsi_stmt (gsi));
 405               gimple_set_lhs (g, lhs);
 406               gsi_replace (&gsi, g, false);
 407             }
 408
 409           /* Recognize .VEC_SET and .VEC_EXTRACT patterns.  */
 410           cfg_changed |= gimple_expand_vec_set_extract_expr (fun, &gsi);
 411           if (gsi_end_p (gsi))
 412             break;
 413
 414           gassign *stmt = dyn_cast <gassign *> (*gsi);
 415           if (!stmt)
 416             continue;
 417
 418           tree_code code = gimple_assign_rhs_code (stmt);
 419           if (TREE_CODE_CLASS (code) == tcc_comparison)
 420             maybe_duplicate_comparison (stmt, bb);
 421         }
 422     }
 423
 424   return cfg_changed ? TODO_cleanup_cfg : 0;
 425 }
 426
 427 } // anon namespace
 428
 429 gimple_opt_pass *
 430 make_pass_gimple_isel (gcc::context *ctxt)
 431 {
 432   return new pass_gimple_isel (ctxt);
 433 }
 434