1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2024 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
21 #define INCLUDE_MEMORY
23 #include "coretypes.h"
28 #include "tree-pass.h"
31 #include "optabs-tree.h"
32 #include "diagnostic.h"
33 #include "fold-const.h"
34 #include "stor-layout.h"
35 #include "langhooks.h"
37 #include "gimple-iterator.h"
38 #include "gimplify-me.h"
41 #include "tree-vector-builder.h"
42 #include "vec-perm-indices.h"
43 #include "insn-config.h"
44 #include "tree-ssa-dce.h"
45 #include "gimple-fold.h"
46 #include "gimple-match.h"
47 #include "recog.h" /* FIXME: for insn_data */
48 #include "optabs-libfuncs.h"
50 #include "tree-vectorizer.h"
53 /* Build a ternary operation and gimplify it. Emit code before GSI.
54 Return the gimple_val holding the result. */
57 gimplify_build3 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
58 tree type
, tree a
, tree b
, tree c
)
60 location_t loc
= gimple_location (gsi_stmt (*gsi
));
61 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
, b
, c
);
64 /* Build a binary operation and gimplify it. Emit code before GSI.
65 Return the gimple_val holding the result. */
68 gimplify_build2 (gimple_stmt_iterator
*gsi
, enum tree_code code
,
69 tree type
, tree a
, tree b
)
71 location_t loc
= gimple_location (gsi_stmt (*gsi
));
72 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
, b
);
75 /* Build a unary operation and gimplify it. Emit code before GSI.
76 Return the gimple_val holding the result. */
79 gimplify_build1 (gimple_stmt_iterator
*gsi
, enum tree_code code
, tree type
,
82 location_t loc
= gimple_location (gsi_stmt (*gsi
));
83 return gimple_build (gsi
, true, GSI_SAME_STMT
, loc
, code
, type
, a
);
87 static void expand_vector_operations_1 (gimple_stmt_iterator
*, bitmap
);
89 /* Return the number of elements in a vector type TYPE that we have
90 already decided needs to be expanded piecewise. We don't support
91 this kind of expansion for variable-length vectors, since we should
92 always check for target support before introducing uses of those. */
94 nunits_for_known_piecewise_op (const_tree type
)
96 return TYPE_VECTOR_SUBPARTS (type
).to_constant ();
99 /* Return true if TYPE1 has more elements than TYPE2, where either
100 type may be a vector or a scalar. */
103 subparts_gt (tree type1
, tree type2
)
105 poly_uint64 n1
= VECTOR_TYPE_P (type1
) ? TYPE_VECTOR_SUBPARTS (type1
) : 1;
106 poly_uint64 n2
= VECTOR_TYPE_P (type2
) ? TYPE_VECTOR_SUBPARTS (type2
) : 1;
107 return known_gt (n1
, n2
);
110 static GTY(()) tree vector_inner_type
;
111 static GTY(()) tree vector_last_type
;
112 static GTY(()) int vector_last_nunits
;
114 /* Return a suitable vector types made of SUBPARTS units each of mode
115 "word_mode" (the global variable). */
117 build_word_mode_vector_type (int nunits
)
119 if (!vector_inner_type
)
120 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
121 else if (vector_last_nunits
== nunits
)
123 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
124 return vector_last_type
;
127 vector_last_nunits
= nunits
;
128 vector_last_type
= build_vector_type (vector_inner_type
, nunits
);
129 return vector_last_type
;
132 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
133 tree
, tree
, tree
, tree
, tree
, enum tree_code
,
136 /* Extract the vector element of type TYPE at BITPOS with BITSIZE from T
140 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
141 tree t
, tree bitsize
, tree bitpos
)
143 /* We're using the resimplify API and maybe_push_res_to_seq to
144 simplify the BIT_FIELD_REF but restrict the simplification to
145 a single stmt while at the same time following SSA edges for
146 simplification with already emitted CTORs. */
148 opr
.set_op (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
149 opr
.resimplify (NULL
, follow_all_ssa_edges
);
150 gimple_seq stmts
= NULL
;
151 tree res
= maybe_push_res_to_seq (&opr
, &stmts
);
154 /* This can happen if SSA_NAME_OCCURS_IN_ABNORMAL_PHI are
155 used. Build BIT_FIELD_REF manually otherwise. */
156 t
= build3 (BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
157 res
= make_ssa_name (type
);
158 gimple
*g
= gimple_build_assign (res
, t
);
159 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
162 gsi_insert_seq_before (gsi
, stmts
, GSI_SAME_STMT
);
167 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
168 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
169 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
171 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
172 return gimplify_build1 (gsi
, code
, inner_type
, a
);
176 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
177 tree bitpos
, tree bitsize
, enum tree_code code
,
178 tree type ATTRIBUTE_UNUSED
)
180 if (VECTOR_TYPE_P (TREE_TYPE (a
)))
181 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
182 if (VECTOR_TYPE_P (TREE_TYPE (b
)))
183 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
184 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
187 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
189 INNER_TYPE is the type of A and B elements
191 returned expression is of signed integer type with the
192 size equal to the size of INNER_TYPE. */
194 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
195 tree bitpos
, tree bitsize
, enum tree_code code
, tree type
)
197 tree stype
= TREE_TYPE (type
);
198 tree cst_false
= build_zero_cst (stype
);
199 tree cst_true
= build_all_ones_cst (stype
);
202 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
203 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
205 cmp
= build2 (code
, boolean_type_node
, a
, b
);
206 return gimplify_build3 (gsi
, COND_EXPR
, stype
, cmp
, cst_true
, cst_false
);
209 /* Expand vector addition to scalars. This does bit twiddling
210 in order to increase parallelism:
212 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
215 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
216 (a ^ ~b) & 0x80808080
218 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
220 This optimization should be done only if 4 vector items or more
223 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
224 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
225 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
227 unsigned int width
= vector_element_bits (TREE_TYPE (a
));
228 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
229 unsigned HOST_WIDE_INT max
;
230 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
232 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
233 low_bits
= build_replicated_int_cst (word_type
, width
, max
>> 1);
234 high_bits
= build_replicated_int_cst (word_type
, width
, max
& ~(max
>> 1));
236 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
237 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
239 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
240 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
241 if (code
== PLUS_EXPR
)
242 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
245 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
246 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
249 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
250 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
251 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
255 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
256 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
257 tree bitsize ATTRIBUTE_UNUSED
,
258 enum tree_code code ATTRIBUTE_UNUSED
,
259 tree type ATTRIBUTE_UNUSED
)
261 unsigned int width
= vector_element_bits (TREE_TYPE (b
));
262 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
264 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
266 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
267 low_bits
= build_replicated_int_cst (word_type
, width
, max
>> 1);
268 high_bits
= build_replicated_int_cst (word_type
, width
, max
& ~(max
>> 1));
270 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
272 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
273 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
274 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
275 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
276 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
279 /* Expand a vector operation to scalars, by using many operations
280 whose type is the vector type's inner type. */
282 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
283 tree type
, tree inner_type
,
284 tree a
, tree b
, enum tree_code code
,
285 bool parallel_p
, tree ret_type
= NULL_TREE
)
287 vec
<constructor_elt
, va_gc
> *v
;
288 tree part_width
= TYPE_SIZE (inner_type
);
289 tree index
= bitsize_int (0);
290 int nunits
= nunits_for_known_piecewise_op (type
);
291 int delta
= tree_to_uhwi (part_width
) / vector_element_bits (type
);
293 location_t loc
= gimple_location (gsi_stmt (*gsi
));
296 || warning_suppressed_p (gsi_stmt (*gsi
),
297 OPT_Wvector_operation_performance
))
298 /* Do not diagnose decomposing single element vectors or when
299 decomposing vectorizer produced operations. */
301 else if (ret_type
|| !parallel_p
)
302 warning_at (loc
, OPT_Wvector_operation_performance
,
303 "vector operation will be expanded piecewise");
305 warning_at (loc
, OPT_Wvector_operation_performance
,
306 "vector operation will be expanded in parallel");
310 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
311 bool constant_p
= true;
312 for (i
= 0; i
< nunits
;
313 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
315 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
,
317 if (!CONSTANT_CLASS_P (result
))
319 constructor_elt ce
= {NULL_TREE
, result
};
324 return build_vector_from_ctor (ret_type
, v
);
326 return build_constructor (ret_type
, v
);
329 /* Expand a vector operation to scalars with the freedom to use
330 a scalar integer type, or to use a different size for the items
331 in the vector type. */
333 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
334 tree a
, tree b
, enum tree_code code
)
336 tree result
, compute_type
;
337 int n_words
= tree_to_uhwi (TYPE_SIZE_UNIT (type
)) / UNITS_PER_WORD
;
338 location_t loc
= gimple_location (gsi_stmt (*gsi
));
340 /* We have three strategies. If the type is already correct, just do
341 the operation an element at a time. Else, if the vector is wider than
342 one word, do it a word at a time; finally, if the vector is smaller
343 than one word, do it as a scalar. */
344 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
345 return expand_vector_piecewise (gsi
, f
,
346 type
, TREE_TYPE (type
),
348 else if (n_words
> 1)
350 tree word_type
= build_word_mode_vector_type (n_words
);
351 result
= expand_vector_piecewise (gsi
, f
,
352 word_type
, TREE_TYPE (word_type
),
354 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
359 /* Use a single scalar operation with a mode no wider than word_mode. */
360 if (!warning_suppressed_p (gsi_stmt (*gsi
),
361 OPT_Wvector_operation_performance
))
362 warning_at (loc
, OPT_Wvector_operation_performance
,
363 "vector operation will be expanded with a "
364 "single scalar operation");
366 = int_mode_for_size (tree_to_uhwi (TYPE_SIZE (type
)), 0).require ();
367 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
368 result
= f (gsi
, compute_type
, a
, b
, bitsize_zero_node
,
369 TYPE_SIZE (compute_type
), code
, type
);
375 /* Expand a vector operation to scalars; for integer types we can use
376 special bit twiddling tricks to do the sums a word at a time, using
377 function F_PARALLEL instead of F. These tricks are done only if
378 they can process at least four items, that is, only if the vector
379 holds at least four items and if a word can hold four items. */
381 expand_vector_addition (gimple_stmt_iterator
*gsi
,
382 elem_op_func f
, elem_op_func f_parallel
,
383 tree type
, tree a
, tree b
, enum tree_code code
)
385 int parts_per_word
= BITS_PER_WORD
/ vector_element_bits (type
);
387 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
388 && parts_per_word
>= 4
389 && nunits_for_known_piecewise_op (type
) >= 4)
390 return expand_vector_parallel (gsi
, f_parallel
,
393 return expand_vector_piecewise (gsi
, f
,
394 type
, TREE_TYPE (type
),
399 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
);
401 /* Try to expand vector comparison expression OP0 CODE OP1 by
402 querying optab if the following expression:
403 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
406 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
407 tree op1
, enum tree_code code
,
408 bitmap dce_ssa_names
)
410 tree lhs
= gimple_assign_lhs (gsi_stmt (*gsi
));
412 imm_use_iterator iterator
;
413 bool vec_cond_expr_only
= true;
415 /* As seen in PR95830, we should not expand comparisons that are only
416 feeding a VEC_COND_EXPR statement. */
417 auto_vec
<gimple
*> uses
;
418 FOR_EACH_IMM_USE_FAST (use_p
, iterator
, lhs
)
420 gimple
*use
= USE_STMT (use_p
);
421 if (is_gimple_debug (use
))
423 if (is_gimple_assign (use
)
424 && gimple_assign_rhs_code (use
) == VEC_COND_EXPR
425 && gimple_assign_rhs1 (use
) == lhs
426 && gimple_assign_rhs2 (use
) != lhs
427 && gimple_assign_rhs3 (use
) != lhs
)
428 uses
.safe_push (use
);
430 vec_cond_expr_only
= false;
433 if (vec_cond_expr_only
)
434 for (gimple
*use
: uses
)
436 gimple_stmt_iterator it
= gsi_for_stmt (use
);
437 if (!expand_vector_condition (&it
, dce_ssa_names
))
439 vec_cond_expr_only
= false;
444 if (!uses
.is_empty () && vec_cond_expr_only
)
448 if (!expand_vec_cmp_expr_p (TREE_TYPE (op0
), type
, code
))
450 if (VECTOR_BOOLEAN_TYPE_P (type
)
451 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
452 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
453 TYPE_VECTOR_SUBPARTS (type
)
454 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
455 (TREE_TYPE (type
)))))
457 tree inner_type
= TREE_TYPE (TREE_TYPE (op0
));
458 tree part_width
= vector_element_bits_tree (TREE_TYPE (op0
));
459 tree index
= bitsize_int (0);
460 int nunits
= nunits_for_known_piecewise_op (TREE_TYPE (op0
));
461 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (type
));
462 tree ret_type
= build_nonstandard_integer_type (prec
, 1);
463 tree ret_inner_type
= boolean_type_node
;
465 location_t loc
= gimple_location (gsi_stmt (*gsi
));
466 t
= build_zero_cst (ret_type
);
468 if (TYPE_PRECISION (ret_inner_type
) != 1)
469 ret_inner_type
= build_nonstandard_integer_type (1, 1);
470 if (!warning_suppressed_p (gsi_stmt (*gsi
),
471 OPT_Wvector_operation_performance
))
472 warning_at (loc
, OPT_Wvector_operation_performance
,
473 "vector operation will be expanded piecewise");
474 for (i
= 0; i
< nunits
;
475 i
++, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
477 tree a
= tree_vec_extract (gsi
, inner_type
, op0
, part_width
,
479 tree b
= tree_vec_extract (gsi
, inner_type
, op1
, part_width
,
481 tree result
= gimplify_build2 (gsi
, code
, ret_inner_type
, a
, b
);
482 t
= gimplify_build3 (gsi
, BIT_INSERT_EXPR
, ret_type
, t
, result
,
485 t
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
488 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
489 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
,
498 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
499 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
500 the result if successful, otherwise return NULL_TREE. */
502 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
505 unsigned int i
, nunits
= nunits_for_known_piecewise_op (type
);
506 bool scalar_shift
= true;
508 for (i
= 1; i
< nunits
; i
++)
510 if (shiftcnts
[i
] != shiftcnts
[0])
511 scalar_shift
= false;
514 if (scalar_shift
&& shiftcnts
[0] == 0)
519 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
520 if (op
!= unknown_optab
521 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
522 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
523 build_int_cst (NULL_TREE
, shiftcnts
[0]));
526 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
527 if (op
!= unknown_optab
528 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
530 tree_vector_builder
vec (type
, nunits
, 1);
531 for (i
= 0; i
< nunits
; i
++)
532 vec
.quick_push (build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]));
533 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
, vec
.build ());
539 /* Try to expand integer vector division by constant using
540 widening multiply, shifts and additions. */
542 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
543 tree op1
, enum tree_code code
)
545 bool use_pow2
= true;
546 bool has_vector_shift
= true;
547 bool use_abs_op1
= false;
548 int mode
= -1, this_mode
;
549 int pre_shift
= -1, post_shift
;
550 unsigned int nunits
= nunits_for_known_piecewise_op (type
);
551 int *shifts
= XALLOCAVEC (int, nunits
* 4);
552 int *pre_shifts
= shifts
+ nunits
;
553 int *post_shifts
= pre_shifts
+ nunits
;
554 int *shift_temps
= post_shifts
+ nunits
;
555 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
556 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
558 signop sign_p
= TYPE_SIGN (TREE_TYPE (type
));
559 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
560 tree cur_op
, mulcst
, tem
;
563 if (prec
> HOST_BITS_PER_WIDE_INT
)
566 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
567 if (op
== unknown_optab
568 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
569 has_vector_shift
= false;
571 /* Analysis phase. Determine if all op1 elements are either power
572 of two and it is possible to expand it using shifts (or for remainder
573 using masking). Additionally compute the multiplicative constants
574 and pre and post shifts if the division is to be expanded using
575 widening or high part multiplication plus shifts. */
576 for (i
= 0; i
< nunits
; i
++)
578 tree cst
= VECTOR_CST_ELT (op1
, i
);
579 unsigned HOST_WIDE_INT ml
;
581 if (TREE_CODE (cst
) != INTEGER_CST
|| integer_zerop (cst
))
587 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
591 shifts
[i
] = tree_log2 (cst
);
592 if (shifts
[i
] != shifts
[0]
593 && code
== TRUNC_DIV_EXPR
594 && !has_vector_shift
)
599 if (sign_p
== UNSIGNED
)
601 unsigned HOST_WIDE_INT mh
;
602 unsigned HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
) & mask
;
604 if (d
>= (HOST_WIDE_INT_1U
<< (prec
- 1)))
605 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
614 /* Find a suitable multiplier and right shift count instead of
615 directly dividing by D. */
616 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
);
618 /* If the suggested multiplier is more than PREC bits, we can
619 do better for even divisors, using an initial right shift. */
620 if ((mh
!= 0 && (d
& 1) == 0)
621 || (!has_vector_shift
&& pre_shift
!= -1))
623 if (has_vector_shift
)
624 pre_shift
= ctz_or_zero (d
);
625 else if (pre_shift
== -1)
628 for (j
= 0; j
< nunits
; j
++)
630 tree cst2
= VECTOR_CST_ELT (op1
, j
);
631 unsigned HOST_WIDE_INT d2
;
634 if (!tree_fits_uhwi_p (cst2
))
636 d2
= tree_to_uhwi (cst2
) & mask
;
639 this_pre_shift
= floor_log2 (d2
& -d2
);
640 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
641 pre_shift
= this_pre_shift
;
643 if (i
!= 0 && pre_shift
!= 0)
653 if ((d
>> pre_shift
) <= 1)
658 mh
= choose_multiplier (d
>> pre_shift
, prec
,
662 pre_shifts
[i
] = pre_shift
;
672 HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
);
673 unsigned HOST_WIDE_INT abs_d
;
678 /* Since d might be INT_MIN, we have to cast to
679 unsigned HOST_WIDE_INT before negating to avoid
680 undefined signed overflow. */
682 ? (unsigned HOST_WIDE_INT
) d
683 : - (unsigned HOST_WIDE_INT
) d
);
685 /* n rem d = n rem -d */
686 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
691 if (abs_d
== HOST_WIDE_INT_1U
<< (prec
- 1))
693 /* This case is not handled correctly below. */
703 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
705 if (ml
>= HOST_WIDE_INT_1U
<< (prec
- 1))
707 this_mode
= 4 + (d
< 0);
708 ml
|= HOST_WIDE_INT_M1U
<< (prec
- 1);
711 this_mode
= 2 + (d
< 0);
714 post_shifts
[i
] = post_shift
;
715 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
716 || post_shift
>= prec
717 || pre_shifts
[i
] >= prec
)
722 else if (mode
!= this_mode
)
728 tree addend
= NULL_TREE
;
729 if (sign_p
== SIGNED
)
733 /* Both division and remainder sequences need
734 op0 < 0 ? mask : 0 computed. It can be either computed as
735 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
736 if none of the shifts is 0, or as the conditional. */
737 for (i
= 0; i
< nunits
; i
++)
741 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
743 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
745 for (i
= 0; i
< nunits
; i
++)
746 shift_temps
[i
] = prec
- 1;
747 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
748 if (cur_op
!= NULL_TREE
)
750 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
752 for (i
= 0; i
< nunits
; i
++)
753 shift_temps
[i
] = prec
- shifts
[i
];
754 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
755 if (cur_op
!= NULL_TREE
)
756 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
760 if (addend
== NULL_TREE
761 && expand_vec_cond_expr_p (type
, type
, LT_EXPR
))
763 tree zero
, cst
, mask_type
, mask
;
766 mask_type
= truth_type_for (type
);
767 zero
= build_zero_cst (type
);
768 mask
= make_ssa_name (mask_type
);
769 cond
= gimple_build_assign (mask
, LT_EXPR
, op0
, zero
);
770 gsi_insert_before (gsi
, cond
, GSI_SAME_STMT
);
771 tree_vector_builder
vec (type
, nunits
, 1);
772 for (i
= 0; i
< nunits
; i
++)
773 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
777 addend
= make_ssa_name (type
);
779 = gimple_build_assign (addend
, VEC_COND_EXPR
, mask
, cst
, zero
);
780 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
783 if (code
== TRUNC_DIV_EXPR
)
785 if (sign_p
== UNSIGNED
)
787 /* q = op0 >> shift; */
788 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
789 if (cur_op
!= NULL_TREE
)
792 else if (addend
!= NULL_TREE
)
794 /* t1 = op0 + addend;
796 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
797 if (op
!= unknown_optab
798 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
800 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
801 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
802 if (cur_op
!= NULL_TREE
)
810 tree_vector_builder
vec (type
, nunits
, 1);
811 for (i
= 0; i
< nunits
; i
++)
812 vec
.quick_push (build_int_cst (TREE_TYPE (type
),
816 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
817 if (op
!= unknown_optab
818 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
820 if (sign_p
== UNSIGNED
)
821 /* r = op0 & mask; */
822 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
823 else if (addend
!= NULL_TREE
)
825 /* t1 = op0 + addend;
828 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
829 if (op
!= unknown_optab
830 && optab_handler (op
, TYPE_MODE (type
))
833 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
835 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
837 op
= optab_for_tree_code (MINUS_EXPR
, type
,
839 if (op
!= unknown_optab
840 && optab_handler (op
, TYPE_MODE (type
))
842 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
850 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
853 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
861 gcc_assert (sign_p
== UNSIGNED
);
862 /* t1 = oprnd0 >> pre_shift;
864 q = t2 >> post_shift; */
865 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
866 if (cur_op
== NULL_TREE
)
870 gcc_assert (sign_p
== UNSIGNED
);
871 for (i
= 0; i
< nunits
; i
++)
881 gcc_assert (sign_p
== SIGNED
);
882 for (i
= 0; i
< nunits
; i
++)
883 shift_temps
[i
] = prec
- 1;
889 tree_vector_builder
vec (type
, nunits
, 1);
890 for (i
= 0; i
< nunits
; i
++)
891 vec
.quick_push (build_int_cst (TREE_TYPE (type
), mulc
[i
]));
892 mulcst
= vec
.build ();
894 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
899 /* t1 = oprnd0 >> pre_shift;
901 q = t2 >> post_shift; */
902 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
905 /* t1 = oprnd0 h* ml;
909 q = t4 >> (post_shift - 1); */
910 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
911 if (op
== unknown_optab
912 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
914 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
915 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
916 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
917 if (op
== unknown_optab
918 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
920 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
921 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
922 if (cur_op
== NULL_TREE
)
929 /* t1 = oprnd0 h* ml;
930 t2 = t1; [ iff (mode & 2) != 0 ]
931 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
932 t3 = t2 >> post_shift;
933 t4 = oprnd0 >> (prec - 1);
934 q = t3 - t4; [ iff (mode & 1) == 0 ]
935 q = t4 - t3; [ iff (mode & 1) != 0 ] */
938 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
939 if (op
== unknown_optab
940 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
942 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
944 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
945 if (cur_op
== NULL_TREE
)
947 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
948 if (tem
== NULL_TREE
)
950 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
951 if (op
== unknown_optab
952 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
955 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
957 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
963 if (code
== TRUNC_DIV_EXPR
)
966 /* We divided. Now finish by:
969 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
970 if (op
== unknown_optab
971 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
975 tree_vector_builder elts
;
976 if (!elts
.new_unary_operation (type
, op1
, false))
978 unsigned int count
= elts
.encoded_nelts ();
979 for (unsigned int i
= 0; i
< count
; ++i
)
981 tree elem1
= VECTOR_CST_ELT (op1
, i
);
983 tree elt
= const_unop (ABS_EXPR
, TREE_TYPE (elem1
), elem1
);
984 if (elt
== NULL_TREE
)
986 elts
.quick_push (elt
);
990 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
991 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
992 if (op
== unknown_optab
993 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
995 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
998 /* Expand a vector condition to scalars, by using many conditions
999 on the vector's elements. */
1002 expand_vector_condition (gimple_stmt_iterator
*gsi
, bitmap dce_ssa_names
)
1004 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1005 tree type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1006 tree a
= gimple_assign_rhs1 (stmt
);
1008 tree a2
= NULL_TREE
;
1009 bool a_is_comparison
= false;
1010 bool a_is_scalar_bitmask
= false;
1011 tree b
= gimple_assign_rhs2 (stmt
);
1012 tree c
= gimple_assign_rhs3 (stmt
);
1013 vec
<constructor_elt
, va_gc
> *v
;
1015 tree inner_type
= TREE_TYPE (type
);
1016 tree width
= vector_element_bits_tree (type
);
1017 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
1018 tree comp_inner_type
= cond_type
;
1019 tree index
= bitsize_int (0);
1020 tree comp_width
= width
;
1021 tree comp_index
= index
;
1022 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1023 tree_code code
= TREE_CODE (a
);
1024 gassign
*assign
= NULL
;
1026 if (code
== SSA_NAME
)
1028 assign
= dyn_cast
<gassign
*> (SSA_NAME_DEF_STMT (a
));
1030 && TREE_CODE_CLASS (gimple_assign_rhs_code (assign
)) == tcc_comparison
)
1032 a_is_comparison
= true;
1033 a1
= gimple_assign_rhs1 (assign
);
1034 a2
= gimple_assign_rhs2 (assign
);
1035 code
= gimple_assign_rhs_code (assign
);
1036 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
1037 comp_width
= vector_element_bits_tree (TREE_TYPE (a1
));
1041 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
), code
)
1042 || (integer_all_onesp (b
) && integer_zerop (c
)
1043 && expand_vec_cmp_expr_p (type
, TREE_TYPE (a1
), code
)))
1045 gcc_assert (TREE_CODE (a
) == SSA_NAME
|| TREE_CODE (a
) == VECTOR_CST
);
1049 /* If a has vector boolean type and is a comparison, above
1050 expand_vec_cond_expr_p might fail, even if both the comparison and
1051 VEC_COND_EXPR could be supported individually. See PR109176. */
1053 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1054 && expand_vec_cond_expr_p (type
, TREE_TYPE (a
), SSA_NAME
)
1055 && expand_vec_cmp_expr_p (TREE_TYPE (a1
), TREE_TYPE (a
), code
))
1058 /* Handle vector boolean types with bitmasks. If there is a comparison
1059 and we can expand the comparison into the vector boolean bitmask,
1060 or otherwise if it is compatible with type, we can transform
1061 vbfld_1 = x_2 < y_3 ? vbfld_4 : vbfld_5;
1064 tmp_7 = tmp_6 & vbfld_4;
1066 tmp_9 = tmp_8 & vbfld_5;
1067 vbfld_1 = tmp_7 | tmp_9;
1068 Similarly for vbfld_10 instead of x_2 < y_3. */
1069 if (VECTOR_BOOLEAN_TYPE_P (type
)
1070 && SCALAR_INT_MODE_P (TYPE_MODE (type
))
1071 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (type
)),
1072 TYPE_VECTOR_SUBPARTS (type
)
1073 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (type
))))
1075 ? useless_type_conversion_p (type
, TREE_TYPE (a
))
1076 : expand_vec_cmp_expr_p (TREE_TYPE (a1
), type
, TREE_CODE (a
))))
1078 if (a_is_comparison
)
1079 a
= gimplify_build2 (gsi
, code
, type
, a1
, a2
);
1080 a1
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a
, b
);
1081 a2
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, type
, a
);
1082 a2
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, a2
, c
);
1083 a
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, type
, a1
, a2
);
1084 gimple_assign_set_rhs_from_tree (gsi
, a
);
1085 update_stmt (gsi_stmt (*gsi
));
1089 /* TODO: try and find a smaller vector type. */
1091 if (!warning_suppressed_p (stmt
, OPT_Wvector_operation_performance
))
1092 warning_at (loc
, OPT_Wvector_operation_performance
,
1093 "vector condition will be expanded piecewise");
1095 if (!a_is_comparison
1096 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
))
1097 && SCALAR_INT_MODE_P (TYPE_MODE (TREE_TYPE (a
)))
1098 && known_lt (GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (a
))),
1099 TYPE_VECTOR_SUBPARTS (TREE_TYPE (a
))
1100 * GET_MODE_BITSIZE (SCALAR_TYPE_MODE
1101 (TREE_TYPE (TREE_TYPE (a
))))))
1103 a_is_scalar_bitmask
= true;
1104 int prec
= GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (a
)));
1105 tree atype
= build_nonstandard_integer_type (prec
, 1);
1106 a
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, atype
, a
);
1108 else if (!a_is_comparison
1109 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (a
)))
1110 comp_width
= vector_element_bits_tree (TREE_TYPE (a
));
1112 int nunits
= nunits_for_known_piecewise_op (type
);
1113 vec_alloc (v
, nunits
);
1114 bool constant_p
= true;
1115 for (int i
= 0; i
< nunits
; i
++)
1118 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
1119 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
1120 if (a_is_comparison
)
1122 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
,
1123 comp_width
, comp_index
);
1124 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
,
1125 comp_width
, comp_index
);
1126 aa
= gimplify_build2 (gsi
, code
, boolean_type_node
, aa1
, aa2
);
1128 else if (a_is_scalar_bitmask
)
1130 wide_int w
= wi::set_bit_in_zero (i
, TYPE_PRECISION (TREE_TYPE (a
)));
1131 result
= gimplify_build2 (gsi
, BIT_AND_EXPR
, TREE_TYPE (a
),
1132 a
, wide_int_to_tree (TREE_TYPE (a
), w
));
1133 aa
= gimplify_build2 (gsi
, NE_EXPR
, boolean_type_node
, result
,
1134 build_zero_cst (TREE_TYPE (a
)));
1138 result
= tree_vec_extract (gsi
, cond_type
, a
, comp_width
, comp_index
);
1139 aa
= gimplify_build2 (gsi
, NE_EXPR
, boolean_type_node
, result
,
1140 build_zero_cst (cond_type
));
1142 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
1143 if (!CONSTANT_CLASS_P (result
))
1145 constructor_elt ce
= {NULL_TREE
, result
};
1147 index
= int_const_binop (PLUS_EXPR
, index
, width
);
1148 if (width
== comp_width
)
1151 comp_index
= int_const_binop (PLUS_EXPR
, comp_index
, comp_width
);
1155 constr
= build_vector_from_ctor (type
, v
);
1157 constr
= build_constructor (type
, v
);
1158 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1159 update_stmt (gsi_stmt (*gsi
));
1161 if (a_is_comparison
)
1162 bitmap_set_bit (dce_ssa_names
,
1163 SSA_NAME_VERSION (gimple_assign_lhs (assign
)));
1169 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
1170 gassign
*assign
, enum tree_code code
,
1171 bitmap dce_ssa_names
)
1173 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1175 /* If the compute mode is not a vector mode (hence we are not decomposing
1176 a BLKmode vector to smaller, hardware-supported vectors), we may want
1177 to expand the operations in parallel. */
1178 if (!VECTOR_MODE_P (compute_mode
))
1183 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1184 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
1185 gimple_assign_rhs1 (assign
),
1186 gimple_assign_rhs2 (assign
), code
);
1190 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
1191 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
1192 gimple_assign_rhs1 (assign
),
1199 return expand_vector_parallel (gsi
, do_binop
, type
,
1200 gimple_assign_rhs1 (assign
),
1201 gimple_assign_rhs2 (assign
), code
);
1204 return expand_vector_parallel (gsi
, do_unop
, type
,
1205 gimple_assign_rhs1 (assign
),
1220 case UNORDERED_EXPR
:
1222 tree rhs1
= gimple_assign_rhs1 (assign
);
1223 tree rhs2
= gimple_assign_rhs2 (assign
);
1225 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
,
1229 case TRUNC_DIV_EXPR
:
1230 case TRUNC_MOD_EXPR
:
1232 tree rhs1
= gimple_assign_rhs1 (assign
);
1233 tree rhs2
= gimple_assign_rhs2 (assign
);
1237 || !VECTOR_INTEGER_TYPE_P (type
)
1238 || TREE_CODE (rhs2
) != VECTOR_CST
1239 || !VECTOR_MODE_P (TYPE_MODE (type
)))
1242 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
1243 if (ret
!= NULL_TREE
)
1252 if (TREE_CODE_CLASS (code
) == tcc_unary
)
1253 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
1254 gimple_assign_rhs1 (assign
),
1255 NULL_TREE
, code
, false);
1257 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
1258 gimple_assign_rhs1 (assign
),
1259 gimple_assign_rhs2 (assign
), code
, false);
1263 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1265 _9 = { b_7, b_7, b_7, b_7 };
1266 a_5 = _9 + { 0, 3, 6, 9 };
1267 because vector splat operation is usually more efficient
1268 than piecewise initialization of the vector. */
1271 optimize_vector_constructor (gimple_stmt_iterator
*gsi
)
1273 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1274 tree lhs
= gimple_assign_lhs (stmt
);
1275 tree rhs
= gimple_assign_rhs1 (stmt
);
1276 tree type
= TREE_TYPE (rhs
);
1278 unsigned HOST_WIDE_INT nelts
;
1279 bool all_same
= true;
1280 constructor_elt
*elt
;
1282 tree base
= NULL_TREE
;
1285 if (!TYPE_VECTOR_SUBPARTS (type
).is_constant (&nelts
)
1287 || CONSTRUCTOR_NELTS (rhs
) != nelts
)
1289 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
1290 if (op
== unknown_optab
1291 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1293 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs
), i
, elt
)
1294 if (TREE_CODE (elt
->value
) != SSA_NAME
1295 || TREE_CODE (TREE_TYPE (elt
->value
)) == VECTOR_TYPE
)
1299 tree this_base
= elt
->value
;
1300 if (this_base
!= CONSTRUCTOR_ELT (rhs
, 0)->value
)
1302 for (j
= 0; j
< nelts
+ 1; j
++)
1304 g
= SSA_NAME_DEF_STMT (this_base
);
1305 if (is_gimple_assign (g
)
1306 && gimple_assign_rhs_code (g
) == PLUS_EXPR
1307 && TREE_CODE (gimple_assign_rhs2 (g
)) == INTEGER_CST
1308 && TREE_CODE (gimple_assign_rhs1 (g
)) == SSA_NAME
1309 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g
)))
1310 this_base
= gimple_assign_rhs1 (g
);
1316 else if (this_base
!= base
)
1321 tree_vector_builder
cst (type
, nelts
, 1);
1322 for (i
= 0; i
< nelts
; i
++)
1324 tree this_base
= CONSTRUCTOR_ELT (rhs
, i
)->value
;
1325 tree elt
= build_zero_cst (TREE_TYPE (base
));
1326 while (this_base
!= base
)
1328 g
= SSA_NAME_DEF_STMT (this_base
);
1329 elt
= fold_binary (PLUS_EXPR
, TREE_TYPE (base
),
1330 elt
, gimple_assign_rhs2 (g
));
1331 if (elt
== NULL_TREE
1332 || TREE_CODE (elt
) != INTEGER_CST
1333 || TREE_OVERFLOW (elt
))
1335 this_base
= gimple_assign_rhs1 (g
);
1337 cst
.quick_push (elt
);
1339 for (i
= 0; i
< nelts
; i
++)
1340 CONSTRUCTOR_ELT (rhs
, i
)->value
= base
;
1341 g
= gimple_build_assign (make_ssa_name (type
), rhs
);
1342 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1343 g
= gimple_build_assign (lhs
, PLUS_EXPR
, gimple_assign_lhs (g
),
1345 gsi_replace (gsi
, g
, false);
1348 /* Return a type for the widest vector mode with the same element type as
1349 type ORIGINAL_VECTOR_TYPE, with at most the same number of elements as type
1350 ORIGINAL_VECTOR_TYPE and that is supported by the target for an operation
1351 with optab OP, or return NULL_TREE if none is found. */
1354 type_for_widest_vector_mode (tree original_vector_type
, optab op
)
1356 gcc_assert (VECTOR_TYPE_P (original_vector_type
));
1357 tree type
= TREE_TYPE (original_vector_type
);
1358 machine_mode inner_mode
= TYPE_MODE (type
);
1359 machine_mode best_mode
= VOIDmode
, mode
;
1360 poly_int64 best_nunits
= 0;
1362 if (SCALAR_FLOAT_MODE_P (inner_mode
))
1363 mode
= MIN_MODE_VECTOR_FLOAT
;
1364 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1365 mode
= MIN_MODE_VECTOR_FRACT
;
1366 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1367 mode
= MIN_MODE_VECTOR_UFRACT
;
1368 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1369 mode
= MIN_MODE_VECTOR_ACCUM
;
1370 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1371 mode
= MIN_MODE_VECTOR_UACCUM
;
1372 else if (inner_mode
== BImode
)
1373 mode
= MIN_MODE_VECTOR_BOOL
;
1375 mode
= MIN_MODE_VECTOR_INT
;
1377 FOR_EACH_MODE_FROM (mode
, mode
)
1378 if (GET_MODE_INNER (mode
) == inner_mode
1379 && maybe_gt (GET_MODE_NUNITS (mode
), best_nunits
)
1380 && optab_handler (op
, mode
) != CODE_FOR_nothing
1381 && known_le (GET_MODE_NUNITS (mode
),
1382 TYPE_VECTOR_SUBPARTS (original_vector_type
)))
1383 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1385 if (best_mode
== VOIDmode
)
1388 return build_vector_type_for_mode (type
, best_mode
);
1392 /* Build a reference to the element of the vector VECT. Function
1393 returns either the element itself, either BIT_FIELD_REF, or an
1394 ARRAY_REF expression.
1396 GSI is required to insert temporary variables while building a
1397 refernece to the element of the vector VECT.
1399 PTMPVEC is a pointer to the temporary variable for caching
1400 purposes. In case when PTMPVEC is NULL new temporary variable
1403 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1405 tree vect_type
, vect_elt_type
;
1409 bool need_asgn
= true;
1410 unsigned int elements
;
1412 vect_type
= TREE_TYPE (vect
);
1413 vect_elt_type
= TREE_TYPE (vect_type
);
1414 elements
= nunits_for_known_piecewise_op (vect_type
);
1416 if (TREE_CODE (idx
) == INTEGER_CST
)
1418 unsigned HOST_WIDE_INT index
;
1420 /* Given that we're about to compute a binary modulus,
1421 we don't care about the high bits of the value. */
1422 index
= TREE_INT_CST_LOW (idx
);
1423 if (!tree_fits_uhwi_p (idx
) || index
>= elements
)
1425 index
&= elements
- 1;
1426 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1429 /* When lowering a vector statement sequence do some easy
1430 simplification by looking through intermediate vector results. */
1431 if (TREE_CODE (vect
) == SSA_NAME
)
1433 gimple
*def_stmt
= SSA_NAME_DEF_STMT (vect
);
1434 if (is_gimple_assign (def_stmt
)
1435 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1436 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1437 vect
= gimple_assign_rhs1 (def_stmt
);
1440 if (TREE_CODE (vect
) == VECTOR_CST
)
1441 return VECTOR_CST_ELT (vect
, index
);
1442 else if (TREE_CODE (vect
) == CONSTRUCTOR
1443 && (CONSTRUCTOR_NELTS (vect
) == 0
1444 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1447 if (index
< CONSTRUCTOR_NELTS (vect
))
1448 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1449 return build_zero_cst (vect_elt_type
);
1453 tree size
= vector_element_bits_tree (vect_type
);
1454 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1456 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1461 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1463 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1472 TREE_ADDRESSABLE (tmpvec
) = 1;
1473 asgn
= gimple_build_assign (tmpvec
, vect
);
1474 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1477 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1478 return build4 (ARRAY_REF
, vect_elt_type
,
1479 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1480 idx
, NULL_TREE
, NULL_TREE
);
1483 /* Check if VEC_PERM_EXPR within the given setting is supported
1484 by hardware, or lower it piecewise.
1486 When VEC_PERM_EXPR has the same first and second operands:
1487 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1488 {v0[mask[0]], v0[mask[1]], ...}
1489 MASK and V0 must have the same number of elements.
1491 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1492 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1493 V0 and V1 must have the same type. MASK, V0, V1 must have the
1494 same number of arguments. */
1497 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1499 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1500 tree mask
= gimple_assign_rhs3 (stmt
);
1501 tree vec0
= gimple_assign_rhs1 (stmt
);
1502 tree vec1
= gimple_assign_rhs2 (stmt
);
1503 tree vect_type
= TREE_TYPE (vec0
);
1504 tree mask_type
= TREE_TYPE (mask
);
1505 tree vect_elt_type
= TREE_TYPE (vect_type
);
1506 tree mask_elt_type
= TREE_TYPE (mask_type
);
1507 unsigned HOST_WIDE_INT elements
;
1508 vec
<constructor_elt
, va_gc
> *v
;
1509 tree constr
, t
, si
, i_val
;
1510 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1511 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1512 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1515 if (!TYPE_VECTOR_SUBPARTS (vect_type
).is_constant (&elements
))
1518 if (TREE_CODE (mask
) == SSA_NAME
)
1520 gimple
*def_stmt
= SSA_NAME_DEF_STMT (mask
);
1521 if (is_gimple_assign (def_stmt
)
1522 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1523 mask
= gimple_assign_rhs1 (def_stmt
);
1526 vec_perm_builder sel_int
;
1528 if (TREE_CODE (mask
) == VECTOR_CST
1529 && tree_to_vec_perm_builder (&sel_int
, mask
))
1531 vec_perm_indices
indices (sel_int
, 2, elements
);
1532 machine_mode vmode
= TYPE_MODE (vect_type
);
1533 tree lhs_type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1534 machine_mode lhs_mode
= TYPE_MODE (lhs_type
);
1535 if (can_vec_perm_const_p (lhs_mode
, vmode
, indices
))
1537 gimple_assign_set_rhs3 (stmt
, mask
);
1541 /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero
1542 vector as VEC1 and a right element shift MASK. */
1543 if (optab_handler (vec_shr_optab
, TYPE_MODE (vect_type
))
1545 && TREE_CODE (vec1
) == VECTOR_CST
1546 && initializer_zerop (vec1
)
1547 && maybe_ne (indices
[0], 0)
1548 && known_lt (poly_uint64 (indices
[0]), elements
))
1550 bool ok_p
= indices
.series_p (0, 1, indices
[0], 1);
1553 for (i
= 1; i
< elements
; ++i
)
1555 poly_uint64 actual
= indices
[i
];
1556 poly_uint64 expected
= i
+ indices
[0];
1557 /* Indices into the second vector are all equivalent. */
1558 if (maybe_lt (actual
, elements
)
1559 ? maybe_ne (actual
, expected
)
1560 : maybe_lt (expected
, elements
))
1563 ok_p
= i
== elements
;
1567 gimple_assign_set_rhs3 (stmt
, mask
);
1572 /* And similarly vec_shl pattern. */
1573 if (optab_handler (vec_shl_optab
, TYPE_MODE (vect_type
))
1575 && TREE_CODE (vec0
) == VECTOR_CST
1576 && initializer_zerop (vec0
))
1578 unsigned int first
= 0;
1579 for (i
= 0; i
< elements
; ++i
)
1580 if (known_eq (poly_uint64 (indices
[i
]), elements
))
1582 if (i
== 0 || first
)
1587 ? maybe_ne (poly_uint64 (indices
[i
]),
1588 elements
+ i
- first
)
1589 : maybe_ge (poly_uint64 (indices
[i
]), elements
))
1591 if (first
&& i
== elements
)
1593 gimple_assign_set_rhs3 (stmt
, mask
);
1599 else if (can_vec_perm_var_p (TYPE_MODE (vect_type
)))
1602 if (!warning_suppressed_p (stmt
, OPT_Wvector_operation_performance
))
1603 warning_at (loc
, OPT_Wvector_operation_performance
,
1604 "vector shuffling operation will be expanded piecewise");
1606 vec_alloc (v
, elements
);
1607 bool constant_p
= true;
1608 for (i
= 0; i
< elements
; i
++)
1611 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1613 if (TREE_CODE (i_val
) == INTEGER_CST
)
1615 unsigned HOST_WIDE_INT index
;
1617 index
= TREE_INT_CST_LOW (i_val
);
1618 if (!tree_fits_uhwi_p (i_val
) || index
>= elements
)
1619 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1621 if (two_operand_p
&& (index
& elements
) != 0)
1622 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1624 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1626 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1627 true, GSI_SAME_STMT
);
1631 tree cond
= NULL_TREE
, v0_val
;
1635 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1636 build_int_cst (mask_elt_type
, elements
));
1637 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1638 true, GSI_SAME_STMT
);
1641 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1642 build_int_cst (mask_elt_type
, elements
- 1));
1643 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1644 true, GSI_SAME_STMT
);
1646 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1647 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1648 true, GSI_SAME_STMT
);
1654 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1655 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1656 true, GSI_SAME_STMT
);
1658 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1659 cond
, build_zero_cst (mask_elt_type
));
1660 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1661 cond
, v0_val
, v1_val
);
1662 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1663 true, GSI_SAME_STMT
);
1669 if (!CONSTANT_CLASS_P (t
))
1671 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1675 constr
= build_vector_from_ctor (vect_type
, v
);
1677 constr
= build_constructor (vect_type
, v
);
1678 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1679 update_stmt (gsi_stmt (*gsi
));
1682 /* If OP is a uniform vector return the element it is a splat from. */
1685 ssa_uniform_vector_p (tree op
)
1687 if (TREE_CODE (op
) == VECTOR_CST
1688 || TREE_CODE (op
) == VEC_DUPLICATE_EXPR
1689 || TREE_CODE (op
) == CONSTRUCTOR
)
1690 return uniform_vector_p (op
);
1691 if (TREE_CODE (op
) == SSA_NAME
)
1693 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
1694 if (gimple_assign_single_p (def_stmt
))
1695 return uniform_vector_p (gimple_assign_rhs1 (def_stmt
));
1700 /* Return type in which CODE operation with optab OP can be
1704 get_compute_type (enum tree_code code
, optab op
, tree type
)
1706 /* For very wide vectors, try using a smaller vector mode. */
1707 tree compute_type
= type
;
1709 && (!VECTOR_MODE_P (TYPE_MODE (type
))
1710 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
))
1712 tree vector_compute_type
1713 = type_for_widest_vector_mode (type
, op
);
1714 if (vector_compute_type
!= NULL_TREE
1715 && maybe_ne (TYPE_VECTOR_SUBPARTS (vector_compute_type
), 1U)
1716 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1717 != CODE_FOR_nothing
))
1718 compute_type
= vector_compute_type
;
1721 /* If we are breaking a BLKmode vector into smaller pieces,
1722 type_for_widest_vector_mode has already looked into the optab,
1723 so skip these checks. */
1724 if (compute_type
== type
)
1726 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1727 if (VECTOR_MODE_P (compute_mode
))
1730 && (optab_handler (op
, compute_mode
) != CODE_FOR_nothing
1731 || optab_libfunc (op
, compute_mode
)))
1732 return compute_type
;
1733 if (code
== MULT_HIGHPART_EXPR
1734 && can_mult_highpart_p (compute_mode
,
1735 TYPE_UNSIGNED (compute_type
)))
1736 return compute_type
;
1738 /* There is no operation in hardware, so fall back to scalars. */
1739 compute_type
= TREE_TYPE (type
);
1742 return compute_type
;
1746 do_cond (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
1747 tree bitpos
, tree bitsize
, enum tree_code code
,
1748 tree type ATTRIBUTE_UNUSED
)
1750 if (VECTOR_TYPE_P (TREE_TYPE (a
)))
1751 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1752 if (VECTOR_TYPE_P (TREE_TYPE (b
)))
1753 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
1754 tree cond
= gimple_assign_rhs1 (gsi_stmt (*gsi
));
1755 return gimplify_build3 (gsi
, code
, inner_type
, unshare_expr (cond
), a
, b
);
1758 /* Expand a vector COND_EXPR to scalars, piecewise. */
1760 expand_vector_scalar_condition (gimple_stmt_iterator
*gsi
)
1762 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1763 tree lhs
= gimple_assign_lhs (stmt
);
1764 tree type
= TREE_TYPE (lhs
);
1765 tree compute_type
= get_compute_type (COND_EXPR
, mov_optab
, type
);
1766 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1767 gcc_assert (compute_mode
!= BLKmode
);
1768 tree rhs2
= gimple_assign_rhs2 (stmt
);
1769 tree rhs3
= gimple_assign_rhs3 (stmt
);
1772 /* If the compute mode is not a vector mode (hence we are not decomposing
1773 a BLKmode vector to smaller, hardware-supported vectors), we may want
1774 to expand the operations in parallel. */
1775 if (!VECTOR_MODE_P (compute_mode
))
1776 new_rhs
= expand_vector_parallel (gsi
, do_cond
, type
, rhs2
, rhs3
,
1779 new_rhs
= expand_vector_piecewise (gsi
, do_cond
, type
, compute_type
,
1780 rhs2
, rhs3
, COND_EXPR
, false);
1781 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1782 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1785 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1786 way to do it is change expand_vector_operation and its callees to
1787 return a tree_code, RHS1 and RHS2 instead of a tree. */
1788 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1789 update_stmt (gsi_stmt (*gsi
));
1792 /* Callback for expand_vector_piecewise to do VEC_CONVERT ifn call
1793 lowering. If INNER_TYPE is not a vector type, this is a scalar
1797 do_vec_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1798 tree decl
, tree bitpos
, tree bitsize
,
1799 enum tree_code code
, tree type
)
1801 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1802 if (!VECTOR_TYPE_P (inner_type
))
1803 return gimplify_build1 (gsi
, code
, TREE_TYPE (type
), a
);
1804 if (code
== CALL_EXPR
)
1806 gimple
*g
= gimple_build_call (decl
, 1, a
);
1807 tree lhs
= make_ssa_name (TREE_TYPE (TREE_TYPE (decl
)));
1808 gimple_call_set_lhs (g
, lhs
);
1809 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1814 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1815 TYPE_VECTOR_SUBPARTS (inner_type
));
1816 return gimplify_build1 (gsi
, code
, outer_type
, a
);
1820 /* Similarly, but for narrowing conversion. */
1823 do_vec_narrow_conversion (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
1824 tree
, tree bitpos
, tree
, enum tree_code code
,
1827 tree itype
= build_vector_type (TREE_TYPE (inner_type
),
1828 exact_div (TYPE_VECTOR_SUBPARTS (inner_type
),
1830 tree b
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
), bitpos
);
1831 tree c
= tree_vec_extract (gsi
, itype
, a
, TYPE_SIZE (itype
),
1832 int_const_binop (PLUS_EXPR
, bitpos
,
1833 TYPE_SIZE (itype
)));
1834 tree outer_type
= build_vector_type (TREE_TYPE (type
),
1835 TYPE_VECTOR_SUBPARTS (inner_type
));
1836 return gimplify_build2 (gsi
, code
, outer_type
, b
, c
);
1839 /* Expand VEC_CONVERT ifn call. */
1842 expand_vector_conversion (gimple_stmt_iterator
*gsi
)
1844 gimple
*stmt
= gsi_stmt (*gsi
);
1846 tree lhs
= gimple_call_lhs (stmt
);
1847 if (lhs
== NULL_TREE
)
1849 g
= gimple_build_nop ();
1850 gsi_replace (gsi
, g
, false);
1853 tree arg
= gimple_call_arg (stmt
, 0);
1854 tree ret_type
= TREE_TYPE (lhs
);
1855 tree arg_type
= TREE_TYPE (arg
);
1856 tree new_rhs
, new_lhs
, compute_type
= TREE_TYPE (arg_type
);
1857 enum tree_code code
= NOP_EXPR
;
1858 enum tree_code code1
= ERROR_MARK
;
1859 enum { NARROW
, NONE
, WIDEN
} modifier
= NONE
;
1860 optab optab1
= unknown_optab
;
1862 gcc_checking_assert (VECTOR_TYPE_P (ret_type
) && VECTOR_TYPE_P (arg_type
));
1863 if (INTEGRAL_TYPE_P (TREE_TYPE (ret_type
))
1864 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (arg_type
)))
1865 code
= FIX_TRUNC_EXPR
;
1866 else if (INTEGRAL_TYPE_P (TREE_TYPE (arg_type
))
1867 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (ret_type
)))
1869 unsigned int ret_elt_bits
= vector_element_bits (ret_type
);
1870 unsigned int arg_elt_bits
= vector_element_bits (arg_type
);
1871 if (ret_elt_bits
< arg_elt_bits
)
1873 else if (ret_elt_bits
> arg_elt_bits
)
1876 auto_vec
<std::pair
<tree
, tree_code
> > converts
;
1877 if (supportable_indirect_convert_operation (code
,
1883 for (unsigned int i
= 0; i
< converts
.length () - 1; i
++)
1885 new_lhs
= make_ssa_name (converts
[i
].first
);
1886 g
= gimple_build_assign (new_lhs
, converts
[i
].second
, new_rhs
);
1888 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1890 g
= gimple_build_assign (lhs
,
1891 converts
[converts
.length() - 1].second
,
1893 gsi_replace (gsi
, g
, false);
1897 if (modifier
== NONE
&& (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
))
1899 /* Can't use get_compute_type here, as supportable_convert_operation
1900 doesn't necessarily use an optab and needs two arguments. */
1901 tree vec_compute_type
1902 = type_for_widest_vector_mode (arg_type
, mov_optab
);
1903 if (vec_compute_type
1904 && VECTOR_MODE_P (TYPE_MODE (vec_compute_type
)))
1906 unsigned HOST_WIDE_INT nelts
1907 = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vec_compute_type
));
1910 tree ret1_type
= build_vector_type (TREE_TYPE (ret_type
), nelts
);
1911 tree arg1_type
= build_vector_type (TREE_TYPE (arg_type
), nelts
);
1912 if (supportable_convert_operation (code
, ret1_type
, arg1_type
,
1915 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
,
1916 ret_type
, arg1_type
, arg
,
1917 NULL_TREE
, code1
, false);
1918 g
= gimple_build_assign (lhs
, new_rhs
);
1919 gsi_replace (gsi
, g
, false);
1926 else if (modifier
== NARROW
)
1931 code1
= VEC_PACK_TRUNC_EXPR
;
1932 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1934 case FIX_TRUNC_EXPR
:
1935 code1
= VEC_PACK_FIX_TRUNC_EXPR
;
1936 /* The signedness is determined from output operand. */
1937 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
1940 code1
= VEC_PACK_FLOAT_EXPR
;
1941 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
1948 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
1949 enum insn_code icode1
;
1950 if (VECTOR_TYPE_P (compute_type
)
1951 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
1952 != CODE_FOR_nothing
)
1953 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
))
1956 = build_vector_type (TREE_TYPE (ret_type
),
1957 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1958 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
1960 if (compute_type
== arg_type
)
1962 new_rhs
= gimplify_build2 (gsi
, code1
, cretd_type
,
1963 arg
, build_zero_cst (arg_type
));
1964 new_rhs
= tree_vec_extract (gsi
, ret_type
, new_rhs
,
1965 TYPE_SIZE (ret_type
),
1967 g
= gimple_build_assign (lhs
, new_rhs
);
1968 gsi_replace (gsi
, g
, false);
1972 = build_vector_type (TREE_TYPE (compute_type
),
1973 TYPE_VECTOR_SUBPARTS (compute_type
) * 2);
1974 if (TYPE_MAIN_VARIANT (dcompute_type
)
1975 == TYPE_MAIN_VARIANT (arg_type
))
1976 new_rhs
= do_vec_narrow_conversion (gsi
, dcompute_type
, arg
,
1977 NULL_TREE
, bitsize_int (0),
1981 new_rhs
= expand_vector_piecewise (gsi
,
1982 do_vec_narrow_conversion
,
1983 arg_type
, dcompute_type
,
1984 arg
, NULL_TREE
, code1
,
1986 g
= gimple_build_assign (lhs
, new_rhs
);
1987 gsi_replace (gsi
, g
, false);
1992 else if (modifier
== WIDEN
)
1994 enum tree_code code2
= ERROR_MARK
;
1995 optab optab2
= unknown_optab
;
1999 code1
= VEC_UNPACK_LO_EXPR
;
2000 code2
= VEC_UNPACK_HI_EXPR
;
2002 case FIX_TRUNC_EXPR
:
2003 code1
= VEC_UNPACK_FIX_TRUNC_LO_EXPR
;
2004 code2
= VEC_UNPACK_FIX_TRUNC_HI_EXPR
;
2007 code1
= VEC_UNPACK_FLOAT_LO_EXPR
;
2008 code2
= VEC_UNPACK_FLOAT_HI_EXPR
;
2013 if (BYTES_BIG_ENDIAN
)
2014 std::swap (code1
, code2
);
2016 if (code
== FIX_TRUNC_EXPR
)
2018 /* The signedness is determined from output operand. */
2019 optab1
= optab_for_tree_code (code1
, ret_type
, optab_default
);
2020 optab2
= optab_for_tree_code (code2
, ret_type
, optab_default
);
2024 optab1
= optab_for_tree_code (code1
, arg_type
, optab_default
);
2025 optab2
= optab_for_tree_code (code2
, arg_type
, optab_default
);
2028 if (optab1
&& optab2
)
2029 compute_type
= get_compute_type (code1
, optab1
, arg_type
);
2031 enum insn_code icode1
, icode2
;
2032 if (VECTOR_TYPE_P (compute_type
)
2033 && ((icode1
= optab_handler (optab1
, TYPE_MODE (compute_type
)))
2034 != CODE_FOR_nothing
)
2035 && ((icode2
= optab_handler (optab2
, TYPE_MODE (compute_type
)))
2036 != CODE_FOR_nothing
)
2037 && VECTOR_MODE_P (insn_data
[icode1
].operand
[0].mode
)
2038 && (insn_data
[icode1
].operand
[0].mode
2039 == insn_data
[icode2
].operand
[0].mode
))
2042 = exact_div (TYPE_VECTOR_SUBPARTS (compute_type
), 2);
2043 tree cretd_type
= build_vector_type (TREE_TYPE (ret_type
), nunits
);
2044 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (cretd_type
))
2046 vec
<constructor_elt
, va_gc
> *v
;
2047 tree part_width
= TYPE_SIZE (compute_type
);
2048 tree index
= bitsize_int (0);
2049 int nunits
= nunits_for_known_piecewise_op (arg_type
);
2050 int delta
= tree_to_uhwi (part_width
) / arg_elt_bits
;
2052 location_t loc
= gimple_location (gsi_stmt (*gsi
));
2054 if (compute_type
!= arg_type
)
2056 if (!warning_suppressed_p (gsi_stmt (*gsi
),
2057 OPT_Wvector_operation_performance
))
2058 warning_at (loc
, OPT_Wvector_operation_performance
,
2059 "vector operation will be expanded piecewise");
2067 vec_alloc (v
, (nunits
+ delta
- 1) / delta
* 2);
2068 bool constant_p
= true;
2069 for (i
= 0; i
< nunits
;
2070 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
,
2074 if (compute_type
!= arg_type
)
2075 a
= tree_vec_extract (gsi
, compute_type
, a
, part_width
,
2077 tree result
= gimplify_build1 (gsi
, code1
, cretd_type
, a
);
2078 constructor_elt ce
= { NULL_TREE
, result
};
2079 if (!CONSTANT_CLASS_P (ce
.value
))
2082 ce
.value
= gimplify_build1 (gsi
, code2
, cretd_type
, a
);
2083 if (!CONSTANT_CLASS_P (ce
.value
))
2089 new_rhs
= build_vector_from_ctor (ret_type
, v
);
2091 new_rhs
= build_constructor (ret_type
, v
);
2092 g
= gimple_build_assign (lhs
, new_rhs
);
2093 gsi_replace (gsi
, g
, false);
2099 new_rhs
= expand_vector_piecewise (gsi
, do_vec_conversion
, arg_type
,
2100 TREE_TYPE (arg_type
), arg
,
2101 NULL_TREE
, code
, false, ret_type
);
2102 g
= gimple_build_assign (lhs
, new_rhs
);
2103 gsi_replace (gsi
, g
, false);
2106 /* Process one statement. If we identify a vector operation, expand it. */
2109 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
,
2110 bitmap dce_ssa_names
)
2112 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
= NULL_TREE
;
2113 enum tree_code code
;
2114 optab op
= unknown_optab
;
2115 enum gimple_rhs_class rhs_class
;
2118 /* Only consider code == GIMPLE_ASSIGN. */
2119 gassign
*stmt
= dyn_cast
<gassign
*> (gsi_stmt (*gsi
));
2122 if (gimple_call_internal_p (gsi_stmt (*gsi
), IFN_VEC_CONVERT
))
2123 expand_vector_conversion (gsi
);
2127 code
= gimple_assign_rhs_code (stmt
);
2128 rhs_class
= get_gimple_rhs_class (code
);
2129 lhs
= gimple_assign_lhs (stmt
);
2131 if (code
== VEC_PERM_EXPR
)
2133 lower_vec_perm (gsi
);
2137 if (code
== VEC_COND_EXPR
)
2139 expand_vector_condition (gsi
, dce_ssa_names
);
2143 if (code
== COND_EXPR
2144 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == VECTOR_TYPE
2145 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == BLKmode
)
2147 expand_vector_scalar_condition (gsi
);
2151 if (code
== CONSTRUCTOR
2152 && TREE_CODE (lhs
) == SSA_NAME
2153 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs
)))
2154 && !gimple_clobber_p (stmt
)
2157 optimize_vector_constructor (gsi
);
2161 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
2164 rhs1
= gimple_assign_rhs1 (stmt
);
2165 if (rhs_class
== GIMPLE_BINARY_RHS
)
2166 rhs2
= gimple_assign_rhs2 (stmt
);
2168 type
= TREE_TYPE (lhs
);
2169 if (!VECTOR_TYPE_P (type
)
2170 || !VECTOR_TYPE_P (TREE_TYPE (rhs1
)))
2173 /* A scalar operation pretending to be a vector one. */
2174 if (VECTOR_BOOLEAN_TYPE_P (type
)
2175 && !VECTOR_MODE_P (TYPE_MODE (type
))
2176 && TYPE_MODE (type
) != BLKmode
2177 && (TREE_CODE_CLASS (gimple_assign_rhs_code (stmt
)) != tcc_comparison
2178 || (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1
))
2179 && !VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (rhs1
)))
2180 && TYPE_MODE (TREE_TYPE (rhs1
)) != BLKmode
)))
2183 /* If the vector operation is operating on all same vector elements
2184 implement it with a scalar operation and a splat if the target
2185 supports the scalar operation. */
2186 tree srhs1
, srhs2
= NULL_TREE
;
2187 if ((srhs1
= ssa_uniform_vector_p (rhs1
)) != NULL_TREE
2188 && (rhs2
== NULL_TREE
2189 || (! VECTOR_TYPE_P (TREE_TYPE (rhs2
))
2191 || (srhs2
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2192 /* As we query direct optabs restrict to non-convert operations. */
2193 && TYPE_MODE (TREE_TYPE (type
)) == TYPE_MODE (TREE_TYPE (srhs1
)))
2195 op
= optab_for_tree_code (code
, TREE_TYPE (type
), optab_scalar
);
2196 if (op
>= FIRST_NORM_OPTAB
&& op
<= LAST_NORM_OPTAB
2197 && optab_handler (op
, TYPE_MODE (TREE_TYPE (type
))) != CODE_FOR_nothing
)
2199 tree stype
= TREE_TYPE (TREE_TYPE (lhs
));
2200 tree slhs
= (rhs2
!= NULL_TREE
)
2201 ? gimplify_build2 (gsi
, code
, stype
, srhs1
, srhs2
)
2202 : gimplify_build1 (gsi
, code
, stype
, srhs1
);
2203 gimple_assign_set_rhs_from_tree (gsi
,
2204 build_vector_from_val (type
, slhs
));
2210 /* Plain moves do not need lowering. */
2211 if (code
== SSA_NAME
2212 || code
== VIEW_CONVERT_EXPR
2213 || code
== PAREN_EXPR
)
2216 if (CONVERT_EXPR_CODE_P (code
)
2217 || code
== FLOAT_EXPR
2218 || code
== FIX_TRUNC_EXPR
)
2221 /* The signedness is determined from input argument. */
2222 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
2223 || code
== VEC_UNPACK_FLOAT_LO_EXPR
2224 || code
== VEC_PACK_FLOAT_EXPR
)
2226 /* We do not know how to scalarize those. */
2230 /* For widening/narrowing vector operations, the relevant type is of the
2231 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
2232 calculated in the same way above. */
2233 if (code
== WIDEN_SUM_EXPR
2234 || code
== VEC_WIDEN_MULT_HI_EXPR
2235 || code
== VEC_WIDEN_MULT_LO_EXPR
2236 || code
== VEC_WIDEN_MULT_EVEN_EXPR
2237 || code
== VEC_WIDEN_MULT_ODD_EXPR
2238 || code
== VEC_UNPACK_HI_EXPR
2239 || code
== VEC_UNPACK_LO_EXPR
2240 || code
== VEC_UNPACK_FIX_TRUNC_HI_EXPR
2241 || code
== VEC_UNPACK_FIX_TRUNC_LO_EXPR
2242 || code
== VEC_PACK_TRUNC_EXPR
2243 || code
== VEC_PACK_SAT_EXPR
2244 || code
== VEC_PACK_FIX_TRUNC_EXPR
2245 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
2246 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
2248 /* We do not know how to scalarize those. */
2252 /* Choose between vector shift/rotate by vector and vector shift/rotate by
2254 if (code
== LSHIFT_EXPR
2255 || code
== RSHIFT_EXPR
2256 || code
== LROTATE_EXPR
2257 || code
== RROTATE_EXPR
)
2261 /* Check whether we have vector <op> {x,x,x,x} where x
2262 could be a scalar variable or a constant. Transform
2263 vector <op> {x,x,x,x} ==> vector <op> scalar. */
2264 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2268 if ((first
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
2270 gimple_assign_set_rhs2 (stmt
, first
);
2276 opv
= optab_for_tree_code (code
, type
, optab_vector
);
2277 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2281 op
= optab_for_tree_code (code
, type
, optab_scalar
);
2283 compute_type
= get_compute_type (code
, op
, type
);
2284 if (compute_type
== type
)
2286 /* The rtl expander will expand vector/scalar as vector/vector
2287 if necessary. Pick one with wider vector type. */
2288 tree compute_vtype
= get_compute_type (code
, opv
, type
);
2289 if (subparts_gt (compute_vtype
, compute_type
))
2291 compute_type
= compute_vtype
;
2296 if (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
2298 if (compute_type
== NULL_TREE
)
2299 compute_type
= get_compute_type (code
, op
, type
);
2300 if (compute_type
== type
)
2302 /* Before splitting vector rotates into scalar rotates,
2303 see if we can't use vector shifts and BIT_IOR_EXPR
2304 instead. For vector by vector rotates we'd also
2305 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
2306 for now, fold doesn't seem to create such rotates anyway. */
2307 if (compute_type
== TREE_TYPE (type
)
2308 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
2310 optab oplv
= vashl_optab
, opl
= ashl_optab
;
2311 optab oprv
= vlshr_optab
, opr
= lshr_optab
, opo
= ior_optab
;
2312 tree compute_lvtype
= get_compute_type (LSHIFT_EXPR
, oplv
, type
);
2313 tree compute_rvtype
= get_compute_type (RSHIFT_EXPR
, oprv
, type
);
2314 tree compute_otype
= get_compute_type (BIT_IOR_EXPR
, opo
, type
);
2315 tree compute_ltype
= get_compute_type (LSHIFT_EXPR
, opl
, type
);
2316 tree compute_rtype
= get_compute_type (RSHIFT_EXPR
, opr
, type
);
2317 /* The rtl expander will expand vector/scalar as vector/vector
2318 if necessary. Pick one with wider vector type. */
2319 if (subparts_gt (compute_lvtype
, compute_ltype
))
2321 compute_ltype
= compute_lvtype
;
2324 if (subparts_gt (compute_rvtype
, compute_rtype
))
2326 compute_rtype
= compute_rvtype
;
2329 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
2331 compute_type
= compute_ltype
;
2332 if (subparts_gt (compute_type
, compute_rtype
))
2333 compute_type
= compute_rtype
;
2334 if (subparts_gt (compute_type
, compute_otype
))
2335 compute_type
= compute_otype
;
2336 /* Verify all 3 operations can be performed in that type. */
2337 if (compute_type
!= TREE_TYPE (type
))
2339 if (optab_handler (opl
, TYPE_MODE (compute_type
))
2341 || optab_handler (opr
, TYPE_MODE (compute_type
))
2343 || optab_handler (opo
, TYPE_MODE (compute_type
))
2344 == CODE_FOR_nothing
)
2345 compute_type
= TREE_TYPE (type
);
2351 op
= optab_for_tree_code (code
, type
, optab_default
);
2353 /* Optabs will try converting a negation into a subtraction, so
2354 look for it as well. TODO: negation of floating-point vectors
2355 might be turned into an exclusive OR toggling the sign bit. */
2356 if (op
== unknown_optab
2357 && code
== NEGATE_EXPR
2358 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
2359 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
2361 if (compute_type
== NULL_TREE
)
2362 compute_type
= get_compute_type (code
, op
, type
);
2363 if (compute_type
== type
)
2366 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
,
2369 /* Leave expression untouched for later expansion. */
2370 if (new_rhs
== NULL_TREE
)
2373 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
2374 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
2377 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
2378 way to do it is change expand_vector_operation and its callees to
2379 return a tree_code, RHS1 and RHS2 instead of a tree. */
2380 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
2381 update_stmt (gsi_stmt (*gsi
));
2384 /* Use this to lower vector operations introduced by the vectorizer,
2385 if it may need the bit-twiddling tricks implemented in this file. */
2388 expand_vector_operations (void)
2390 gimple_stmt_iterator gsi
;
2392 bool cfg_changed
= false;
2394 auto_bitmap dce_ssa_names
;
2396 FOR_EACH_BB_FN (bb
, cfun
)
2398 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2400 expand_vector_operations_1 (&gsi
, dce_ssa_names
);
2401 /* ??? If we do not cleanup EH then we will ICE in
2402 verification. But in reality we have created wrong-code
2403 as we did not properly transition EH info and edges to
2404 the piecewise computations. */
2405 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
2406 && gimple_purge_dead_eh_edges (bb
))
2408 /* If a .LOOP_DIST_ALIAS call prevailed loops got elided
2409 before vectorization got a chance to get at them. Simply
2410 fold as if loop distribution wasn't performed. */
2411 if (gimple_call_internal_p (gsi_stmt (gsi
), IFN_LOOP_DIST_ALIAS
))
2413 fold_loop_internal_call (gsi_stmt (gsi
), boolean_false_node
);
2419 simple_dce_from_worklist (dce_ssa_names
);
2421 return cfg_changed
? TODO_cleanup_cfg
: 0;
2426 const pass_data pass_data_lower_vector
=
2428 GIMPLE_PASS
, /* type */
2429 "veclower", /* name */
2430 OPTGROUP_VEC
, /* optinfo_flags */
2431 TV_NONE
, /* tv_id */
2432 PROP_cfg
, /* properties_required */
2433 PROP_gimple_lvec
, /* properties_provided */
2434 0, /* properties_destroyed */
2435 0, /* todo_flags_start */
2436 TODO_update_ssa
, /* todo_flags_finish */
2439 class pass_lower_vector
: public gimple_opt_pass
2442 pass_lower_vector (gcc::context
*ctxt
)
2443 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
2446 /* opt_pass methods: */
2447 bool gate (function
*fun
) final override
2449 return !(fun
->curr_properties
& PROP_gimple_lvec
);
2452 unsigned int execute (function
*) final override
2454 return expand_vector_operations ();
2457 }; // class pass_lower_vector
2462 make_pass_lower_vector (gcc::context
*ctxt
)
2464 return new pass_lower_vector (ctxt
);
2469 const pass_data pass_data_lower_vector_ssa
=
2471 GIMPLE_PASS
, /* type */
2472 "veclower2", /* name */
2473 OPTGROUP_VEC
, /* optinfo_flags */
2474 TV_NONE
, /* tv_id */
2475 PROP_cfg
, /* properties_required */
2476 PROP_gimple_lvec
, /* properties_provided */
2477 0, /* properties_destroyed */
2478 0, /* todo_flags_start */
2480 | TODO_cleanup_cfg
), /* todo_flags_finish */
2483 class pass_lower_vector_ssa
: public gimple_opt_pass
2486 pass_lower_vector_ssa (gcc::context
*ctxt
)
2487 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
2490 /* opt_pass methods: */
2491 opt_pass
* clone () final override
2493 return new pass_lower_vector_ssa (m_ctxt
);
2495 unsigned int execute (function
*) final override
2497 return expand_vector_operations ();
2500 }; // class pass_lower_vector_ssa
2505 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
2507 return new pass_lower_vector_ssa (ctxt
);
2510 #include "gt-tree-vect-generic.h"