| blob | 43358767934612a9788c587f35f66e007faa8863 |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
23 #define INCLUDE_MEMORY
62 /* For lang_hooks.types.type_for_mode. */
65 /* Return the vectorized type for the given statement. */
67 tree
69 {
71 }
73 /* Return TRUE iff the given statement is in an inner loop relative to
74 the loop being vectorized. */
75 bool
77 {
89 }
91 /* Record the cost of a statement, either by directly informing the
92 target model or by saving it in a vector for later processing.
93 Return a preliminary estimate of the statement's cost. */
95 static unsigned
101 {
109 stmt_info_for_cost si
115 }
117 unsigned
122 {
125 }
127 unsigned
132 {
135 }
137 unsigned
141 {
146 }
148 /* Return a variable of type ELEM_TYPE[NELEMS]. */
150 static tree
152 {
155 }
157 /* ARRAY is an array of vectors created by create_vector_array.
158 Return an SSA_NAME for the vector in index N. The reference
159 is part of the vectorization of STMT_INFO and the vector is associated
160 with scalar destination SCALAR_DEST. */
162 static tree
166 {
183 }
185 /* ARRAY is an array of vectors created by create_vector_array.
186 Emit code to store SSA_NAME VECT in index N of the array.
187 The store is part of the vectorization of STMT_INFO. */
189 static void
193 {
194 tree array_ref;
203 }
205 /* PTR is a pointer to an array of type TYPE. Return a representation
206 of *PTR. The memory reference replaces those in FIRST_DR
207 (and its group). */
209 static tree
211 {
212 tree mem_ref;
215 /* Arrays have the same alignment as their type. */
218 }
220 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
221 Emit the clobber before *GSI. */
223 static void
226 {
230 }
232 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
234 /* Function vect_mark_relevant.
236 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
238 static void
241 {
250 /* If this stmt is an original stmt in a pattern, we might need to mark its
251 related pattern stmt instead of the original stmt. However, such stmts
252 may have their own uses that are not in any pattern, in such cases the
253 stmt itself should be marked. */
255 {
256 /* This is the last stmt in a sequence that was detected as a
257 pattern that can potentially be vectorized. Don't mark the stmt
258 as relevant/live because it's not going to be vectorized.
259 Instead mark the pattern-stmt that replaces it. */
263 "last stmt in pattern. don't mark"
273 {
276 "vec_stmt_relevant_p: forcing live pattern stmt "
279 }
285 }
293 {
298 }
301 }
304 /* Function is_simple_and_all_uses_invariant
306 Return true if STMT_INFO is simple and all uses of it are invariant. */
308 bool
310 loop_vec_info loop_vinfo)
311 {
312 tree op;
313 ssa_op_iter iter;
320 {
324 {
329 }
333 }
335 }
337 /* Function vect_stmt_relevant_p.
339 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
340 is "relevant for vectorization".
342 A stmt is considered "relevant for vectorization" if:
343 - it has uses outside the loop.
344 - it has vdefs (it alters memory).
345 - control stmts in the loop (except for the exit condition).
346 - it is an induction and we have multiple exits.
348 CHECKME: what other side effects would the vectorizer allow? */
350 static bool
353 {
355 ssa_op_iter op_iter;
356 imm_use_iterator imm_iter;
357 use_operand_p use_p;
358 def_operand_p def_p;
363 /* cond stmt other than loop exit cond. */
370 /* changing memory. */
374 {
379 }
381 /* uses outside the loop. */
383 {
385 {
388 {
396 /* We expect all such uses to be in the loop exit phis
397 (because of loop closed form) */
401 }
402 }
403 }
405 /* Check if it's an induction and multiple exits. In this case there will be
406 a usage later on after peeling which is needed for the alternate exit. */
409 {
412 "vec_stmt_relevant_p: induction forced for "
416 }
420 {
425 }
428 }
431 /* Function exist_non_indexing_operands_for_use_p
433 USE is one of the uses attached to STMT_INFO. Check if USE is
434 used in STMT_INFO for anything other than indexing an array. */
436 static bool
438 {
439 tree operand;
441 /* USE corresponds to some operand in STMT. If there is no data
442 reference in STMT, then any operand that corresponds to USE
443 is not indexing an array. */
447 /* STMT has a data_ref. FORNOW this means that its of one of
448 the following forms:
449 -1- ARRAY_REF = var
450 -2- var = ARRAY_REF
451 (This should have been verified in analyze_data_refs).
453 'var' in the second case corresponds to a def, not a use,
454 so USE cannot correspond to any operands that are not used
455 for array indexing.
457 Therefore, all we need to check is if STMT falls into the
458 first case, and whether var corresponds to USE. */
462 {
465 {
478 }
480 }
492 }
495 /*
496 Function process_use.
498 Inputs:
499 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
500 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
501 that defined USE. This is done by calling mark_relevant and passing it
502 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
503 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
504 be performed.
506 Outputs:
507 Generally, LIVE_P and RELEVANT are used to define the liveness and
508 relevance info of the DEF_STMT of this USE:
509 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
510 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
511 Exceptions:
512 - case 1: If USE is used only for address computations (e.g. array indexing),
513 which does not need to be directly vectorized, then the liveness/relevance
514 of the respective DEF_STMT is left unchanged.
515 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
516 we skip DEF_STMT cause it had already been processed.
517 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
518 "relevant" will be modified accordingly.
520 Return true if everything is as expected. Return false otherwise. */
522 static opt_result
526 {
527 stmt_vec_info dstmt_vinfo;
530 /* case 1: we are only interested in uses that need to be vectorized. Uses
531 that are used for address computation are not considered relevant. */
537 "not vectorized:"
546 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
547 We have to force the stmt live since the epilogue loop needs it to
548 continue computing the reduction. */
554 {
560 }
562 /* case 3a: outer-loop stmt defining an inner-loop stmt:
563 outer-loop-header-bb:
564 d = dstmt_vinfo
565 inner-loop:
566 stmt # use (d)
567 outer-loop-tail-bb:
568 ... */
570 {
576 {
597 }
598 }
600 /* case 3b: inner-loop stmt defining an outer-loop stmt:
601 outer-loop-header-bb:
602 ...
603 inner-loop:
604 d = dstmt_vinfo
605 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
606 stmt # use (d) */
608 {
614 {
632 }
633 }
634 /* We are also not interested in uses on loop PHI backedges that are
635 inductions. Otherwise we'll needlessly vectorize the IV increment
636 and cause hybrid SLP for SLP inductions. Unless the PHI is live
637 of course. */
644 {
649 }
654 }
657 /* Function vect_mark_stmts_to_be_vectorized.
659 Not all stmts in the loop need to be vectorized. For example:
661 for i...
662 for j...
663 1. T0 = i + j
664 2. T1 = a[T0]
666 3. j = j + 1
668 Stmt 1 and 3 do not need to be vectorized, because loop control and
669 addressing of vectorized data-refs are handled differently.
671 This pass detects such stmts. */
673 opt_result
675 {
679 gimple_stmt_iterator si;
681 basic_block bb;
689 /* 1. Init worklist. */
691 {
694 {
702 }
704 {
714 }
715 }
717 /* 2. Process_worklist */
719 {
720 use_operand_p use_p;
721 ssa_op_iter iter;
728 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
729 (DEF_STMT) as relevant/irrelevant according to the relevance property
730 of STMT. */
733 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
734 propagated as is to the DEF_STMTs of its USEs.
736 One exception is when STMT has been identified as defining a reduction
737 variable; in this case we set the relevance to vect_used_by_reduction.
738 This is because we distinguish between two kinds of relevant stmts -
739 those that are used by a reduction computation, and those that are
740 (also) used by a regular computation. This allows us later on to
741 identify stmts that are used solely by a reduction, and therefore the
742 order of the results that they produce does not have to be kept. */
745 {
774 }
777 {
778 /* Pattern statements are not inserted into the code, so
779 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
780 have to scan the RHS or function arguments instead. */
782 {
788 {
789 opt_result res
799 }
801 {
804 {
805 opt_result res
810 }
811 }
812 }
814 {
817 opt_result res
826 }
828 {
830 {
832 opt_result res
837 }
838 }
839 else
841 }
842 else
844 {
846 opt_result res
851 }
854 {
855 gather_scatter_info gs_info;
858 opt_result res
862 {
866 }
867 }
871 }
873 /* Function vect_model_simple_cost.
875 Models cost for simple operations, i.e. those that only emit ncopies of a
876 single op. Right now, this does not account for multiple insns that could
877 be generated for the single vector op. We will handle that shortly. */
879 static void
884 slp_tree node,
887 {
892 /* ??? Somehow we need to fix this at the callers. */
897 /* Cost the "broadcast" of a scalar operand in to a vector operand.
898 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
899 cost model. */
905 /* Pass the inside-of-loop statements to the target-specific cost model. */
911 "vect_model_simple_cost: inside_cost = %d, "
913 }
916 /* Model cost for type demotion and promotion operations. PWR is
917 normally zero for single-step promotions and demotions. It will be
918 one if two-step promotion/demotion is required, and so on. NCOPIES
919 is the number of vector results (and thus number of instructions)
920 for the narrowest end of the operation chain. Each additional
921 step doubles the number of instructions required. If WIDEN_ARITH
922 is true the stmt is doing widening arithmetic. */
924 static void
930 {
935 {
937 widen_arith
941 }
943 /* FORNOW: Assuming maximum 2 args per stmts. */
951 "vect_model_promotion_demotion_cost: inside_cost = %d, "
953 }
955 /* Returns true if the current function returns DECL. */
957 static bool
959 {
960 edge_iterator ei;
961 edge e;
963 {
969 /* We often end up with an aggregate copy to the result decl,
970 handle that case as well. First skip intermediate clobbers
971 though. */
973 do
974 {
976 }
982 }
984 }
986 /* Calculate cost of DR's memory access. */
987 void
989 dr_alignment_support alignment_support_scheme,
993 {
995 {
997 {
1000 vect_body);
1006 }
1009 {
1010 /* Here, we assign an additional cost for the unaligned store. */
1016 "vect_model_store_cost: unaligned supported by "
1019 }
1022 {
1029 }
1033 }
1034 }
1036 /* Calculate cost of DR's memory access. */
1037 void
1039 dr_alignment_support alignment_support_scheme,
1046 {
1048 {
1050 {
1059 }
1061 {
1062 /* Here, we assign an additional cost for the unaligned load. */
1069 "vect_model_load_cost: unaligned supported by "
1073 }
1075 {
1081 /* FIXME: If the misalignment remains fixed across the iterations of
1082 the containing loop, the following cost should be added to the
1083 prologue costs. */
1093 }
1095 {
1098 "vect_model_load_cost: unaligned software "
1101 /* Unaligned software pipeline has a load of an address, an initial
1102 load, and possibly a mask operation to "prime" the loop. However,
1103 if this is an access in a group of loads, which provide grouped
1104 access, then the above cost should only be considered for one
1105 access in the group. Inside the loop, there is a load op
1106 and a realignment op. */
1109 {
1117 }
1126 "vect_model_load_cost: explicit realign optimized"
1130 }
1133 {
1140 }
1144 }
1145 }
1147 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1148 the loop preheader for the vectorized stmt STMT_VINFO. */
1150 static void
1153 {
1156 else
1162 }
1164 /* Function vect_init_vector.
1166 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1167 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1168 vector type a vector with all elements equal to VAL is created first.
1169 Place the initialization at GSI if it is not NULL. Otherwise, place the
1170 initialization at the loop preheader.
1171 Return the DEF of INIT_STMT.
1172 It will be used in the vectorization of STMT_INFO. */
1174 tree
1177 {
1179 tree new_temp;
1181 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1183 {
1186 {
1187 /* Scalar boolean value should be transformed into
1188 all zeros or all ones value before building a vector. */
1190 {
1196 else
1197 {
1203 }
1204 }
1205 else
1206 {
1211 else
1212 /* ??? Condition vectorization expects us to do
1213 promotion of invariant/external defs. */
1217 {
1221 }
1222 }
1223 }
1225 }
1231 }
1234 /* Function vect_get_vec_defs_for_operand.
1236 OP is an operand in STMT_VINFO. This function returns a vector of
1237 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1239 In the case that OP is an SSA_NAME which is defined in the loop, then
1240 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1242 In case OP is an invariant or constant, a new stmt that creates a vector def
1243 needs to be introduced. VECTYPE may be used to specify a required type for
1244 vector invariant. */
1246 void
1250 {
1260 stmt_vec_info def_stmt_info;
1269 {
1271 tree vector_type;
1278 else
1285 }
1286 else
1287 {
1293 }
1294 }
1297 /* Get vectorized definitions for OP0 and OP1. */
1299 void
1306 {
1308 {
1317 }
1318 else
1319 {
1332 }
1333 }
1335 void
1342 {
1348 }
1350 /* Helper function called by vect_finish_replace_stmt and
1351 vect_finish_stmt_generation. Set the location of the new
1352 statement and create and return a stmt_vec_info for it. */
1354 static void
1357 {
1362 {
1365 /* While EH edges will generally prevent vectorization, stmt might
1366 e.g. be in a must-not-throw region. Ensure newly created stmts
1367 that could throw are part of the same region. */
1371 }
1372 else
1374 }
1376 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1377 which sets the same scalar result as STMT_INFO did. Create and return a
1378 stmt_vec_info for VEC_STMT. */
1380 void
1383 {
1391 }
1393 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1394 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1396 void
1400 {
1405 {
1409 {
1413 /* If we have an SSA vuse and insert a store, update virtual
1414 SSA form to avoid triggering the renamer. Do so only
1415 if we can easily see all uses - which is what almost always
1416 happens with the way vectorized stmts are inserted. */
1425 {
1429 }
1430 }
1431 }
1434 }
1436 /* We want to vectorize a call to combined function CFN with function
1437 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1438 as the types of all inputs. Check whether this is possible using
1439 an internal function, returning its code if so or IFN_LAST if not. */
1441 static internal_fn
1444 {
1445 internal_fn ifn;
1448 else
1451 {
1454 {
1459 /* The type size of both the vectype_in and vectype_out should be
1460 exactly the same when vectype_out isn't participating the optab.
1461 While there is no restriction for type size when vectype_out
1462 is part of the optab query. */
1467 OPTIMIZE_FOR_SPEED))
1469 }
1470 }
1472 }
1476 gimple_stmt_iterator *);
1478 /* Check whether a load or store statement in the loop described by
1479 LOOP_VINFO is possible in a loop using partial vectors. This is
1480 testing whether the vectorizer pass has the appropriate support,
1481 as well as whether the target does.
1483 VLS_TYPE says whether the statement is a load or store and VECTYPE
1484 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1485 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1486 says how the load or store is going to be implemented and GROUP_SIZE
1487 is the number of load or store statements in the containing group.
1488 If the access is a gather load or scatter store, GS_INFO describes
1489 its arguments. If the load or store is conditional, SCALAR_MASK is the
1490 condition under which it occurs.
1492 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1493 vectors is not supported, otherwise record the required rgroup control
1494 types. */
1496 static void
1498 slp_tree slp_node,
1499 vec_load_store_type vls_type,
1501 vect_memory_access_type
1502 memory_access_type,
1504 tree scalar_mask)
1505 {
1506 /* Invariant loads need no special support. */
1516 {
1519 internal_fn ifn
1526 scalar_mask);
1527 else
1528 {
1531 "can't operate on partial vectors because"
1532 " the target doesn't have an appropriate"
1535 }
1537 }
1540 {
1541 internal_fn ifn = (is_load
1542 ? IFN_MASK_GATHER_LOAD
1544 internal_fn len_ifn = (is_load
1545 ? IFN_MASK_LEN_GATHER_LOAD
1557 scalar_mask);
1558 else
1559 {
1562 "can't operate on partial vectors because"
1563 " the target doesn't have an appropriate"
1566 }
1568 }
1572 {
1573 /* Element X of the data must come from iteration i * VF + X of the
1574 scalar loop. We need more work to support other mappings. */
1577 "can't operate on partial vectors because an"
1581 }
1584 {
1587 "can't operate on partial vectors when emulating"
1591 }
1593 /* We might load more scalars than we need for permuting SLP loads.
1594 We checked in get_group_load_store_type that the extra elements
1595 don't leak into a new vector. */
1597 {
1602 };
1606 machine_mode mask_mode;
1607 machine_mode vmode;
1610 {
1615 }
1618 {
1622 }
1625 {
1628 "can't operate on partial vectors because the"
1629 " target doesn't have the appropriate partial"
1632 }
1633 }
1635 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1636 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1637 that needs to be applied to all loads and stores in a vectorized loop.
1638 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1639 otherwise return VEC_MASK & LOOP_MASK.
1641 MASK_TYPE is the type of both masks. If new statements are needed,
1642 insert them before GSI. */
1644 tree
1647 {
1663 }
1665 /* Determine whether we can use a gather load or scatter store to vectorize
1666 strided load or store STMT_INFO by truncating the current offset to a
1667 smaller width. We need to be able to construct an offset vector:
1669 { 0, X, X*2, X*3, ... }
1671 without loss of precision, where X is STMT_INFO's DR_STEP.
1673 Return true if this is possible, describing the gather load or scatter
1674 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1676 static bool
1680 {
1685 {
1686 /* ??? Perhaps we could use range information here? */
1691 }
1693 /* Get the number of bits in an element. */
1698 /* Set COUNT to the upper limit on the number of elements - 1.
1699 Start with the maximum vectorization factor. */
1702 /* Try lowering COUNT to the number of scalar latch iterations. */
1704 widest_int max_iters;
1709 /* Try scales of 1 and the element size. */
1713 {
1715 widest_int factor;
1719 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1726 /* Find the narrowest viable offset type. */
1731 /* See whether the target supports the operation with an offset
1732 no narrower than OFFSET_TYPE. */
1741 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1742 but we don't need to store that here. */
1750 }
1754 "truncating gather/scatter offset to %d bits"
1758 }
1760 /* Return true if we can use gather/scatter internal functions to
1761 vectorize STMT_INFO, which is a grouped or strided load or store.
1762 MASKED_P is true if load or store is conditional. When returning
1763 true, fill in GS_INFO with the information required to perform the
1764 operation. */
1766 static bool
1770 {
1785 "using gather/scatter for strided/grouped access,"
1789 }
1791 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1792 elements with a known constant step. Return -1 if that step
1793 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1795 int
1797 {
1800 size_zero_node);
1801 }
1803 /* If the target supports a permute mask that reverses the elements in
1804 a vector of type VECTYPE, return that mask, otherwise return null. */
1806 tree
1808 {
1811 /* The encoding has a single stepped pattern. */
1818 indices))
1821 }
1823 /* A subroutine of get_load_store_type, with a subset of the same
1824 arguments. Handle the case where STMT_INFO is a load or store that
1825 accesses consecutive elements with a negative step. Sets *POFFSET
1826 to the offset to be applied to the DR for the first access. */
1828 static vect_memory_access_type
1831 vec_load_store_type vls_type,
1833 {
1835 dr_alignment_support alignment_support_scheme;
1838 {
1843 }
1845 /* For backward running DRs the first access in vectype actually is
1846 N-1 elements before the address of the DR. */
1851 alignment_support_scheme
1855 {
1861 }
1864 {
1867 "negative step with invariant source;"
1870 }
1873 {
1879 }
1882 }
1884 /* STMT_INFO is either a masked or unconditional store. Return the value
1885 being stored. */
1887 tree
1889 {
1891 {
1894 }
1896 {
1901 }
1903 }
1905 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1907 This function returns a vector type which can be composed with NETLS pieces,
1908 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1909 same vector size as the return vector. It checks target whether supports
1910 pieces-size vector mode for construction firstly, if target fails to, check
1911 pieces-size scalar mode for construction further. It returns NULL_TREE if
1912 fails to find the available composition.
1914 For example, for (vtype=V16QI, nelts=4), we can probably get:
1915 - V16QI with PTYPE V4QI.
1916 - V4SI with PTYPE SI.
1917 - NULL_TREE. */
1919 static tree
1921 {
1929 /* When we are asked to compose the vector from its components let
1930 that happen directly. */
1932 {
1935 }
1940 {
1941 /* First check if vec_init optab supports construction from
1942 vector pieces directly. */
1945 machine_mode rmode;
1949 {
1952 }
1954 /* Otherwise check if exists an integer type of the same piece size and
1955 if vec_init optab supports construction from it directly. */
1960 {
1963 }
1964 }
1967 }
1969 /* A subroutine of get_load_store_type, with a subset of the same
1970 arguments. Handle the case where STMT_INFO is part of a grouped load
1971 or store.
1973 For stores, the statements in the group are all consecutive
1974 and there is no gap at the end. For loads, the statements in the
1975 group might not be consecutive; there can be gaps between statements
1976 as well as at the end. */
1978 static bool
1988 {
1991 stmt_vec_info first_stmt_info;
1995 {
1999 }
2000 else
2001 {
2005 }
2011 /* True if the vectorized statements would access beyond the last
2012 statement in the group. */
2015 /* True if we can cope with such overrun by peeling for gaps, so that
2016 there is at least one final scalar iteration after the vector loop. */
2019 && loop_vinfo
2022 /* There can only be a gap at the end of the group if the stride is
2023 known at compile time. */
2026 /* Stores can't yet have gaps. */
2030 {
2031 /* For SLP vectorization we directly vectorize a subchain
2032 without permutation. */
2034 first_dr_info
2037 /* Try to use consecutive accesses of as many elements as possible,
2038 separated by the stride, until we have a complete vector.
2039 Fall back to scalar accesses if that isn't possible. */
2041 else
2042 {
2045 {
2047 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2048 only correct for single element "interleaving" SLP. */
2051 else
2052 {
2053 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2054 separated by the stride, until we have a complete vector.
2055 Fall back to scalar accesses if that isn't possible. */
2058 else
2060 }
2061 }
2063 {
2066 /* Invariant accesses perform only component accesses, alignment
2067 is irrelevant for them. */
2069 }
2070 /* Try using LOAD/STORE_LANES. */
2072 && (*lanes_ifn
2078 else
2083 {
2085 "Grouped store with gaps requires"
2088 }
2089 /* An overrun is fine if the trailing elements are smaller
2090 than the alignment boundary B. Every vector access will
2091 be a multiple of B and so we are guaranteed to access a
2092 non-gap element in the same B-sized block. */
2095 vectype)
2099 /* When we have a contiguous access across loop iterations
2100 but the access in the loop doesn't cover the full vector
2101 we can end up with no gap recorded but still excess
2102 elements accessed, see PR103116. Make sure we peel for
2103 gaps if necessary and sufficient and give up if not.
2105 If there is a combination of the access not covering the full
2106 vector and a gap recorded then we may need to peel twice. */
2112 nunits))
2115 /* If the gap splits the vector in half and the target
2116 can do half-vector operations avoid the epilogue peeling
2117 by simply loading half of the vector only. Usually
2118 the construction with an upper zero half will be elided. */
2119 dr_alignment_support alss;
2121 tree half_vtype;
2122 poly_uint64 remain;
2125 && !masked_p
2129 == dr_aligned
2143 {
2148 }
2149 /* Peeling for gaps assumes that a single scalar iteration
2150 is enough to make sure the last vector iteration doesn't
2151 access excess elements. */
2157 {
2158 /* But peeling a single scalar iteration is enough if
2159 we can use the next power-of-two sized partial
2160 access and that is sufficiently small to be covered
2161 by the single scalar iteration. */
2168 || vector_vector_composition_type
2171 {
2174 "peeling for gaps insufficient for "
2177 }
2178 }
2180 /* If this is single-element interleaving with an element
2181 distance that leaves unused vector loads around punt - we
2182 at least create very sub-optimal code in that case (and
2183 blow up memory, see PR65518). */
2186 && single_element_p
2188 {
2190 {
2194 "single-element interleaving not supported "
2195 "for not adjacent vector loads, using "
2197 }
2198 else
2199 {
2202 "single-element interleaving not supported "
2205 }
2206 }
2207 }
2208 }
2209 else
2210 {
2211 /* We can always handle this case using elementwise accesses,
2212 but see if something more efficient is available. */
2215 /* If there is a gap at the end of the group then these optimizations
2216 would access excess elements in the last iteration. */
2218 /* An overrun is fine if the trailing elements are smaller than the
2219 alignment boundary B. Every vector access will be a multiple of B
2220 and so we are guaranteed to access a non-gap element in the
2221 same B-sized block. */
2223 && !masked_p
2231 {
2232 /* First cope with the degenerate case of a single-element
2233 vector. */
2235 ;
2237 else
2238 {
2239 /* Otherwise try using LOAD/STORE_LANES. */
2240 *lanes_ifn
2244 masked_p);
2246 {
2249 }
2251 /* If that fails, try using permuting loads. */
2254 single_element_p,
2255 group_size)
2257 {
2260 }
2261 }
2262 }
2263 }
2265 /* As a last resort, trying using a gather load or scatter store.
2267 ??? Although the code can handle all group sizes correctly,
2268 it probably isn't a win to use separate strided accesses based
2269 on nearby locations. Or, even if it's a win over scalar code,
2270 it might not be a win over vectorizing at a lower VF, if that
2271 allows us to use contiguous accesses. */
2273 && single_element_p
2274 && loop_vinfo
2281 {
2284 }
2285 else
2286 {
2288 *alignment_support_scheme
2291 }
2294 {
2295 /* STMT is the leader of the group. Check the operands of all the
2296 stmts of the group. */
2299 {
2303 {
2308 }
2310 }
2311 }
2314 {
2318 "Data access with gaps requires scalar "
2321 }
2324 }
2326 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2327 if there is a memory access type that the vectorized form can use,
2328 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2329 or scatters, fill in GS_INFO accordingly. In addition
2330 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2331 the target does not support the alignment scheme. *MISALIGNMENT
2332 is set according to the alignment of the access (including
2333 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2335 SLP says whether we're performing SLP rather than loop vectorization.
2336 MASKED_P is true if the statement is conditional on a vectorized mask.
2337 VECTYPE is the vector type that the vectorized statements will use.
2338 NCOPIES is the number of vector statements that will be needed. */
2340 static bool
2351 {
2357 {
2361 /* When using internal functions, we rely on pattern recognition
2362 to convert the type of the offset to the type that the target
2363 requires, with the result being a call to an internal function.
2364 If that failed for some reason (e.g. because another pattern
2365 took priority), just handle cases in which the offset already
2366 has the right type. */
2371 {
2377 }
2381 {
2387 }
2389 {
2395 {
2398 "unsupported vector types for emulated "
2401 }
2402 }
2403 /* Gather-scatter accesses perform only component accesses, alignment
2404 is irrelevant for them. */
2406 }
2408 {
2410 masked_p,
2412 alignment_support_scheme,
2415 }
2417 {
2423 else
2425 /* Alignment is irrelevant here. */
2427 }
2428 else
2429 {
2432 {
2435 /* Invariant accesses perform only component accesses, alignment
2436 is irrelevant for them. */
2438 }
2439 else
2440 {
2444 else
2448 *alignment_support_scheme
2452 }
2453 }
2458 {
2461 "Not using elementwise accesses due to variable "
2464 }
2467 {
2472 }
2474 /* FIXME: At the moment the cost model seems to underestimate the
2475 cost of using elementwise accesses. This check preserves the
2476 traditional behavior until that can be fixed. */
2485 {
2490 }
2492 }
2494 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2495 conditional operation STMT_INFO. When returning true, store the mask
2496 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2497 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2498 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2500 static bool
2505 {
2507 tree mask_vectype;
2508 slp_tree mask_node_1;
2511 {
2516 }
2519 {
2524 }
2526 /* If the caller is not prepared for adjusting an external/constant
2527 SLP mask vector type fail. */
2529 && !mask_node
2531 {
2536 }
2541 mask_node_1);
2544 {
2549 }
2553 {
2556 "vector mask type %T"
2561 }
2568 }
2570 /* Return true if stored value is suitable for vectorizing store
2571 statement STMT_INFO. When returning true, store the scalar stored
2572 in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2573 the type of the vectorized store value in
2574 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2576 static bool
2581 {
2584 {
2588 }
2590 op_no = vect_slp_child_index_for_operand
2594 tree rhs_vectype;
2597 {
2602 }
2604 /* In the case this is a store from a constant make sure
2605 native_encode_expr can handle it. */
2608 {
2613 }
2617 {
2622 }
2628 else
2631 }
2633 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2634 Note that we support masks with floating-point type, in which case the
2635 floats are interpreted as a bitmask. */
2637 static tree
2640 {
2645 {
2649 }
2651 {
2652 REAL_VALUE_TYPE r;
2660 }
2662 }
2664 /* Build an all-zero merge value of type VECTYPE while vectorizing
2665 STMT_INFO as a gather load. */
2667 static tree
2670 {
2671 tree merge;
2675 {
2676 REAL_VALUE_TYPE r;
2682 }
2683 else
2687 }
2689 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2690 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2691 the gather load operation. If the load is conditional, MASK is the
2692 vectorized condition, otherwise MASK is null. PTR is the base
2693 pointer and OFFSET is the vectorized offset. */
2700 {
2709 tree var;
2711 && (!mask
2717 {
2725 }
2730 {
2732 {
2737 else
2741 gassign *new_stmt
2746 {
2753 }
2756 }
2757 else
2758 {
2761 }
2762 }
2763 else
2764 {
2767 }
2774 {
2782 }
2785 }
2787 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2788 instructions before GSI. GS_INFO describes the scatter store operation.
2789 PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
2790 vectorized data to store.
2791 If the store is conditional, MASK is the vectorized condition, otherwise
2792 MASK is null. */
2799 {
2812 {
2815 tree utype;
2818 else
2822 gassign *new_stmt
2827 {
2833 }
2834 }
2835 else
2836 {
2839 }
2843 {
2851 }
2855 {
2863 }
2866 gcall *new_stmt
2869 }
2871 /* Prepare the base and offset in GS_INFO for vectorization.
2872 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2873 to the vectorized offset argument for the first copy of STMT_INFO.
2874 STMT_INFO is the statement described by GS_INFO and LOOP is the
2875 containing loop. */
2877 static void
2882 {
2886 {
2887 basic_block new_bb;
2891 }
2894 else
2895 {
2896 unsigned ncopies
2901 }
2902 }
2904 /* Prepare to implement a grouped or strided load or store using
2905 the gather load or scatter store operation described by GS_INFO.
2906 STMT_INFO is the load or store statement.
2908 Set *DATAREF_BUMP to the amount that should be added to the base
2909 address after each copy of the vectorized statement. Set *VEC_OFFSET
2910 to an invariant offset vector in which element I has the value
2911 I * DR_STEP / SCALE. */
2913 static void
2915 loop_vec_info loop_vinfo,
2920 {
2925 {
2926 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2927 ivtmp_8 = _31 * 16 (step in bytes);
2928 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2929 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2930 tree loop_len
2932 tree tmp
2935 loop_len);
2937 GSI_SAME_STMT);
2938 }
2939 else
2940 {
2941 tree bump
2946 }
2948 /* The offset given in GS_INFO can have pointer type, so use the element
2949 type of the vector instead. */
2952 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2957 /* Create {0, X, X*2, X*3, ...}. */
2961 }
2963 /* Prepare the pointer IVs which needs to be updated by a variable amount.
2964 Such variable amount is the outcome of .SELECT_VL. In this case, we can
2965 allow each iteration process the flexible number of elements as long as
2966 the number <= vf elments.
2968 Return data reference according to SELECT_VL.
2969 If new statements are needed, insert them before GSI. */
2971 static tree
2975 vect_memory_access_type memory_access_type)
2976 {
2980 /* gather/scatter never reach here. */
2983 /* When we support SELECT_VL pattern, we dynamic adjust
2984 the memory address by .SELECT_VL result.
2986 The result of .SELECT_VL is the number of elements to
2987 be processed of each iteration. So the memory address
2988 adjustment operation should be:
2990 addr = addr + .SELECT_VL (ARG..) * step;
2991 */
2992 tree loop_len
2995 /* Since the outcome of .SELECT_VL is element size, we should adjust
2996 it into bytesize so that it can be used in address pointer variable
2997 amount IVs adjustment. */
3004 }
3006 /* Return the amount that should be added to a vector pointer to move
3007 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3008 being vectorized and MEMORY_ACCESS_TYPE describes the type of
3009 vectorization. */
3011 static tree
3014 vect_memory_access_type memory_access_type,
3016 {
3024 memory_access_type);
3031 }
3033 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3035 static bool
3041 {
3051 /* Multiple types in SLP are handled by creating the appropriate number of
3052 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3053 case of SLP. */
3056 else
3062 {
3068 }
3079 /* The encoding uses one stepped pattern for each byte in the word. */
3091 {
3094 {
3099 }
3106 slp_node
3110 }
3114 /* Transform. */
3118 /* Arguments are ready. create the new vector stmt. */
3120 tree vop;
3122 {
3138 else
3140 }
3147 }
3149 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3150 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3151 in a single step. On success, store the binary pack code in
3152 *CONVERT_CODE. */
3154 static bool
3157 {
3162 code_helper code;
3172 }
3174 /* Function vectorizable_call.
3176 Check if STMT_INFO performs a function call that can be vectorized.
3177 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3178 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3179 Return true if STMT_INFO is vectorizable in this way. */
3181 static bool
3186 {
3188 tree vec_dest;
3189 tree scalar_dest;
3190 tree op;
3193 poly_uint64 nunits_in;
3194 poly_uint64 nunits_out;
3200 vect_unknown_def_type };
3208 tree lhs;
3218 /* Is STMT_INFO a vectorizable call? */
3226 /* Handled by vectorizable_load and vectorizable_store. */
3237 /* Process function arguments. */
3242 /* Bail out if the function has more than four arguments, we do not have
3243 interesting builtin functions to vectorize with more than two arguments
3244 except for fma. No arguments is also not good. */
3248 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3251 {
3254 }
3255 /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3256 argument just says whether it is well-defined at zero or not and what
3257 value should be returned for it. */
3259 {
3262 }
3269 {
3271 {
3276 }
3280 {
3285 }
3287 /* We can only handle calls with arguments of the same type. */
3290 {
3295 }
3303 {
3308 }
3309 }
3310 /* If all arguments are external or constant defs, infer the vector type
3311 from the scalar type. */
3317 {
3323 }
3327 {
3332 }
3335 {
3340 }
3342 /* FORNOW */
3351 else
3354 /* We only handle functions that do not read or clobber memory. */
3356 {
3361 }
3363 /* For now, we only vectorize functions if a target specific builtin
3364 is available. TODO -- in some cases, it might be profitable to
3365 insert the calls for pieces of the vector, in order to be able
3366 to vectorize other operations in the loop. */
3371 /* First try using an internal function. */
3379 vectype_in);
3381 /* If that fails, try asking for a target-specific built-in function. */
3383 {
3390 }
3393 {
3396 && loop_vinfo
3401 {
3402 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3403 { 0, 1, 2, ... vf - 1 } vector. */
3405 }
3413 else
3414 {
3419 }
3420 }
3426 else
3429 /* Sanity check: make sure that at least one copy of the vectorized stmt
3430 needs to be generated. */
3440 {
3446 {
3451 }
3463 {
3467 OPTIMIZE_FOR_SPEED))
3470 OPTIMIZE_FOR_SPEED)))
3471 {
3474 "can't use a fully-masked loop because no"
3477 }
3478 else
3479 {
3480 unsigned int nvectors
3481 = (slp_node
3489 OPTIMIZE_FOR_SPEED))
3492 else
3494 scalar_mask);
3495 }
3496 }
3498 }
3500 /* Transform. */
3505 /* Handle def. */
3513 {
3515 {
3517 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3519 }
3522 }
3524 {
3527 }
3532 {
3537 {
3538 /* Build argument list for the vectorized call. */
3540 {
3542 {
3544 {
3545 /* ??? For multi-lane SLP we'd need to build
3546 { 0, 0, .., 1, 1, ... }. */
3549 tree new_var
3555 gimple *new_stmt
3558 gsi);
3560 }
3562 }
3568 /* Arguments are ready. Create the new vector stmt. */
3570 {
3573 {
3575 /* Always true for SLP. */
3580 }
3583 {
3586 }
3594 {
3595 /* We don't define any narrowing conditional functions
3596 at present. */
3599 gcall *call
3605 {
3608 }
3614 }
3615 else
3616 {
3618 {
3620 /* Always true for SLP. */
3622 tree len
3625 signed char biasval
3630 }
3632 {
3634 /* Always true for SLP. */
3642 }
3647 else
3654 }
3656 }
3658 }
3665 {
3668 {
3673 }
3675 }
3682 {
3685 signed char biasval
3690 }
3692 {
3698 }
3702 {
3704 tree new_var
3711 }
3713 {
3714 /* We don't define any narrowing conditional functions at
3715 present. */
3723 {
3726 }
3729 half_res);
3731 }
3732 else
3733 {
3737 else
3744 }
3749 }
3751 {
3754 }
3755 }
3757 {
3759 /* We don't define any narrowing conditional functions at present. */
3762 {
3763 /* Build argument list for the vectorized call. */
3766 else
3770 {
3776 /* Arguments are ready. Create the new vector stmt. */
3778 {
3782 {
3786 }
3790 else
3797 }
3799 }
3802 {
3805 {
3809 }
3815 }
3823 }
3829 {
3832 }
3833 }
3834 else
3835 /* No current target implements this case. */
3840 /* The call in STMT might prevent it from being removed in dce.
3841 We however cannot remove it here, due to the way the ssa name
3842 it defines is mapped to the new definition. So just replace
3843 rhs of the statement with something harmless. */
3851 gassign *new_stmt
3856 }
3859 struct simd_call_arg_info
3860 {
3861 tree vectype;
3862 tree op;
3863 HOST_WIDE_INT linear_step;
3867 };
3869 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3870 is linear within simd lane (but not within whole loop), note it in
3871 *ARGINFO. */
3873 static void
3876 {
3888 {
3889 tree t;
3893 {
3908 CASE_CONVERT:
3920 }
3926 {
3933 }
3934 }
3935 }
3937 /* Function vectorizable_simd_clone_call.
3939 Check if STMT_INFO performs a function call that can be vectorized
3940 by calling a simd clone of the function.
3941 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3942 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3943 Return true if STMT_INFO is vectorizable in this way. */
3945 static bool
3949 stmt_vector_for_cost *)
3950 {
3951 tree vec_dest;
3952 tree scalar_dest;
3955 tree vectype;
3956 poly_uint64 nunits;
3969 /* Is STMT a vectorizable call? */
3977 {
3983 }
4009 /* Process function arguments. */
4012 /* Bail out if the function has zero arguments. */
4025 {
4026 simd_call_arg_info thisarginfo;
4027 affine_iv iv;
4041 {
4046 }
4050 {
4051 /* With SLP we determine the vector type of constants/externals
4052 at analysis time, handling conflicts via
4053 vect_maybe_update_slp_op_vectype. At transform time
4054 we have a vector type recorded for SLP. */
4056 || !slp_node
4061 slp_node);
4062 }
4063 else
4066 /* For linear arguments, the analyze phase should have saved
4067 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
4071 {
4074 thisarginfo.simd_lane_linear
4076 /* If loop has been peeled for alignment, we need to adjust it. */
4080 {
4086 thisarginfo.op
4090 }
4091 }
4095 && loop_vinfo
4100 {
4103 }
4108 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4109 linear too. */
4112 && !vec_stmt
4115 && loop_vinfo
4120 }
4128 else
4131 {
4134 /* The number of arguments in the call and the number of parameters in
4135 the simdclone should match. However, when the simdclone is
4136 'inbranch', it could have one more paramater than nargs when using
4137 an inbranch simdclone to call a non-inbranch call, either in a
4138 non-masked loop using a all true constant mask, or inside a masked
4139 loop using it's mask. */
4142 simd_nargs--;
4157 {
4159 {
4190 /* FORNOW */
4194 /* While we can create a traditional data vector from
4195 an incoming integer mode mask we have no good way to
4196 force generate an integer mode mask from a traditional
4197 boolean vector input. */
4205 }
4209 {
4212 }
4216 }
4222 {
4224 SIMD_CLONE_ARG_TYPE_MASK);
4225 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4226 not in a branch, as we'd have to construct an all-true mask. */
4229 }
4231 {
4234 }
4235 }
4244 num_mask_args++;
4247 {
4251 {
4255 slp_node);
4260 }
4264 {
4269 }
4272 {
4275 {
4278 {
4279 /* FORNOW we only have partial support for vector-type masks
4280 that can't hold all of simdlen. */
4283 vect_location,
4284 "in-branch vector clones are not yet"
4287 }
4290 {
4293 vect_location,
4294 "cannot compute mask argument for"
4297 }
4298 }
4300 {
4303 num_mask_args),
4305 {
4306 /* FORNOW we only have partial support for integer-type masks
4307 that represent the same number of lanes as the
4308 vectorized mask inputs. */
4311 vect_location,
4312 "in-branch vector clones are not yet "
4315 }
4316 }
4317 else
4318 {
4321 vect_location,
4322 "in-branch vector clones not supported"
4325 }
4326 }
4327 }
4333 else
4336 /* If the function isn't const, only allow it in simd loops where user
4337 has asserted that at least nunits consecutive iterations can be
4338 performed using SIMD instructions. */
4343 /* Sanity check: make sure that at least one copy of the vectorized stmt
4344 needs to be generated. */
4348 {
4352 {
4357 }
4358 /* When the original call is pure or const but the SIMD ABI dictates
4359 an aggregate return we will have to use a virtual definition and
4360 in a loop eventually even need to add a virtual PHI. That's
4361 not straight-forward so allow to fix this up via renaming. */
4366 /* ??? For SLP code-gen we end up inserting after the last
4367 vector argument def rather than at the original call position
4368 so automagic virtual operand updating doesn't work. */
4373 {
4375 {
4380 {
4390 }
4395 {
4396 unsigned nmasks
4402 }
4405 }
4406 }
4409 {
4413 "can't use a fully-masked loop because a"
4416 }
4420 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4421 dt, slp_node, cost_vec); */
4423 }
4425 /* Transform. */
4430 /* Handle def. */
4436 {
4440 {
4443 }
4444 }
4450 {
4453 }
4454 else
4457 {
4458 poly_uint64 callee_nelements;
4459 poly_uint64 caller_nelements;
4460 /* Build argument list for the vectorized call. */
4463 else
4467 {
4469 tree atype;
4472 {
4479 {
4481 {
4489 {
4496 }
4497 else
4498 {
4502 }
4504 vec_oprnd0
4508 gassign *new_stmt
4510 vec_oprnd0);
4514 }
4515 else
4516 {
4524 else
4527 {
4529 {
4533 op,
4537 }
4538 else
4544 vec_oprnd0);
4545 }
4548 atype))
4549 {
4551 vec_oprnd0);
4552 gassign *new_stmt
4554 vec_oprnd0);
4558 }
4559 else
4561 else
4562 {
4564 gassign *new_stmt
4566 vec_oprnd0);
4570 }
4571 }
4572 }
4576 {
4585 {
4587 {
4588 /* The mask type has fewer elements than simdlen. */
4590 /* FORNOW */
4592 }
4594 {
4595 /* The SIMD clone function has the same number of
4596 elements as the current function. */
4598 {
4602 op,
4605 }
4609 {
4610 vec_loop_masks *loop_masks
4612 tree loop_mask
4616 vec_oprnd0
4620 gsi);
4622 loop_mask });
4624 }
4625 vec_oprnd0
4629 gassign *new_stmt
4631 vec_oprnd0);
4635 }
4636 else
4637 {
4638 /* The mask type has more elements than simdlen. */
4640 /* FORNOW */
4642 }
4643 }
4644 }
4646 {
4648 /* Guess the number of lanes represented by atype. */
4649 poly_uint64 atype_subparts
4651 num_mask_args);
4654 {
4656 {
4660 op,
4663 }
4666 {
4667 /* The mask argument has fewer elements than the
4668 input vector. */
4669 /* FORNOW */
4671 }
4674 {
4675 /* The vector mask argument matches the input
4676 in the number of lanes, but not necessarily
4677 in the mode. */
4682 vec_oprnd0);
4683 gassign *new_stmt
4685 vec_oprnd0);
4689 {
4690 new_stmt
4692 NOP_EXPR,
4693 gimple_assign_lhs
4697 }
4699 }
4700 else
4701 {
4702 /* The mask argument has more elements than the
4703 input vector. */
4704 /* FORNOW */
4706 }
4707 }
4708 }
4709 else
4718 {
4719 gimple_seq stmts;
4724 {
4725 basic_block new_bb;
4729 }
4731 {
4734 }
4739 enum tree_code code
4744 poly_widest_int cst
4749 gassign *new_stmt
4754 UNKNOWN_LOCATION);
4757 }
4758 else
4759 {
4760 enum tree_code code
4765 poly_widest_int cst
4770 gassign *new_stmt
4775 }
4785 }
4786 }
4791 {
4794 tree mask;
4796 SIMD_CLONE_ARG_TYPE_MASK);
4800 /* Guess the number of lanes represented by masktype. */
4803 else
4807 {
4809 {
4813 }
4814 else
4819 {
4820 /* This means we are dealing with integer mask modes.
4821 First convert to an integer type with the same size as
4822 the current vector type. */
4823 unsigned HOST_WIDE_INT intermediate_size
4825 tree mid_int_type =
4828 new_stmt
4830 mask);
4832 /* Then zero-extend to the mask mode. */
4835 }
4837 {
4839 integer_one_node);
4841 integer_zero_node);
4845 }
4846 else
4854 }
4855 }
4859 {
4866 else
4869 }
4874 {
4876 {
4884 {
4885 tree t;
4887 {
4891 }
4892 else
4903 else
4905 }
4910 }
4912 {
4921 {
4926 {
4930 tem);
4935 }
4937 }
4938 else
4943 new_stmt
4952 else
4955 }
4957 {
4964 }
4966 {
4968 new_stmt
4971 }
4972 }
4978 else
4980 }
4983 {
4986 }
4989 /* Mark the clone as no longer being a candidate for GC. */
4992 /* The call in STMT might prevent it from being removed in dce.
4993 We however cannot remove it here, due to the way the ssa name
4994 it defines is mapped to the new definition. So just replace
4995 rhs of the statement with something harmless. */
5002 {
5006 }
5007 else
5013 }
5016 /* Function vect_gen_widened_results_half
5018 Create a vector stmt whose code, type, number of arguments, and result
5019 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5020 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5021 In the case that CODE is a CALL_EXPR, this means that a call to DECL
5022 needs to be created (DECL is a function-decl of a target-builtin).
5023 STMT_INFO is the original scalar stmt that we are vectorizing. */
5029 stmt_vec_info stmt_info)
5030 {
5032 tree new_temp;
5034 /* Generate half of the widened result: */
5043 }
5046 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5047 For multi-step conversions store the resulting vectors and call the function
5048 recursively. When NARROW_SRC_P is true, there's still a conversion after
5049 narrowing, don't store the vectors in the SLP_NODE or in vector info of
5050 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5052 static void
5055 stmt_vec_info stmt_info,
5060 {
5067 {
5068 /* Create demotion operation. */
5076 /* Store the resulting vector for next recursive call,
5077 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5079 else
5080 {
5081 /* This is the last step of the conversion sequence. Store the
5082 vectors in SLP_NODE or in vector info of the scalar statement
5083 (or in STMT_VINFO_RELATED_STMT chain). */
5086 else
5088 }
5089 }
5091 /* For multi-step demotion operations we first generate demotion operations
5092 from the source type to the intermediate types, and then combine the
5093 results (stored in VEC_OPRNDS) in demotion operation to the destination
5094 type. */
5096 {
5097 /* At each level of recursion we have half of the operands we had at the
5098 previous level. */
5104 narrow_src_p);
5105 }
5108 }
5111 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5112 and VEC_OPRNDS1, for a binary operation associated with scalar statement
5113 STMT_INFO. For multi-step conversions store the resulting vectors and
5114 call the function recursively. */
5116 static void
5122 code_helper ch1,
5124 {
5132 {
5135 else
5138 /* Generate the two halves of promotion operation. */
5141 stmt_info);
5144 stmt_info);
5146 {
5149 }
5150 else
5151 {
5154 }
5156 /* Store the results for the next step. */
5159 }
5163 }
5165 /* Create vectorized promotion stmts for widening stmts using only half the
5166 potential vector size for input. */
5167 static void
5173 code_helper code1,
5175 {
5185 {
5191 /* Widen the first vector input. */
5197 {
5198 /* Widen the second vector input. */
5202 /* Perform the operation. With both vector inputs widened. */
5204 }
5205 else
5206 {
5207 /* Perform the operation. With the single vector input widened. */
5209 }
5215 /* Store the results for the next step. */
5217 }
5221 }
5224 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5225 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5226 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5227 Return true if STMT_INFO is vectorizable in this way. */
5229 static bool
5234 {
5236 tree scalar_dest;
5239 tree_code tc1;
5242 tree new_temp;
5245 poly_uint64 nunits_in;
5246 poly_uint64 nunits_out;
5250 /* For conversions between floating point and integer, there're 2 NARROW
5251 cases. NARROW_SRC is for FLOAT_EXPR, means
5252 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5253 This is safe when the range of the source integer can fit into the lower
5254 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5255 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5256 For other conversions, when there's narrowing, NARROW_DST is used as
5257 default. */
5261 tree vop0;
5269 /* Is STMT a vectorizable conversion? */
5290 {
5293 }
5295 {
5298 }
5299 else
5312 /* Check types of lhs and rhs. */
5317 /* Check the operands of the operation. */
5321 {
5326 }
5341 {
5344 "type conversion to/from bit-precision unsupported."
5347 }
5350 {
5357 tree vectype1_in;
5360 {
5365 }
5366 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5367 OP1. */
5370 }
5372 /* If op0 is an external or constant def, infer the vector type
5373 from the scalar type. */
5379 {
5385 }
5389 {
5392 "can't convert between boolean and non "
5396 }
5403 else
5407 else
5408 {
5411 }
5413 /* Multiple types in SLP are handled by creating the appropriate number of
5414 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5415 case of SLP. */
5420 else
5423 /* Sanity check: make sure that at least one copy of the vectorized stmt
5424 needs to be generated. */
5430 opt_scalar_mode rhs_mode_iter;
5433 /* Supportable by target? */
5435 {
5443 vectype_out,
5444 vectype_in,
5446 op0))
5447 {
5451 else
5452 {
5458 }
5460 }
5462 /* FALLTHRU */
5463 unsupported:
5471 {
5480 }
5485 {
5486 /* Binary widening operation can only be supported directly by the
5487 architecture. */
5490 }
5498 {
5503 cvt_type
5510 {
5517 }
5524 else
5528 cvt_type,
5532 {
5535 }
5536 }
5543 else
5544 {
5545 multi_step_cvt++;
5548 }
5562 {
5563 cvt_type
5571 else
5577 }
5578 /* If op0 can be represented with low precision integer,
5579 truncate it to cvt_type and the do FLOAT_EXPR. */
5581 {
5586 cvt_type
5604 {
5608 }
5609 }
5615 }
5618 {
5622 {
5627 }
5630 {
5635 }
5637 {
5639 /* The final packing step produces one vector result per copy. */
5640 unsigned int nvectors
5644 widen_arith);
5645 }
5646 else
5647 {
5649 /* The initial unpacking step produces two vector results
5650 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5651 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5652 unsigned int nvectors
5653 = (slp_node
5658 widen_arith);
5659 }
5662 }
5664 /* Transform. */
5670 {
5675 }
5677 /* In case of multi-step conversion, we first generate conversion operations
5678 to the intermediate types, and then from that types to the final one.
5679 We create vector destinations for the intermediate type (TYPES) received
5680 from supportable_*_operation, and store them in the correct order
5681 for future use in vect_create_vectorized_*_stmts (). */
5683 bool widen_or_narrow_float_p
5686 widen_or_narrow_float_p
5691 {
5694 {
5696 intermediate_type);
5698 }
5699 }
5703 widen_or_narrow_float_p
5708 {
5710 ;
5712 {
5716 }
5717 }
5720 {
5724 /* vec_dest is intermediate type operand when multi_step_cvt. */
5726 {
5729 }
5732 {
5733 /* Arguments are ready, create the new vector stmt. */
5736 {
5743 }
5751 else
5753 }
5757 /* In case the vectorization factor (VF) is bigger than the number
5758 of elements that we can fit in a vectype (nunits), we have to
5759 generate more than one vector stmt - i.e - we need to "unroll"
5760 the vector stmt by a factor VF/nunits. */
5766 {
5771 }
5772 /* Arguments are ready. Create the new vector stmts. */
5774 {
5778 {
5781 }
5785 op_type);
5786 else
5791 }
5794 {
5797 {
5801 }
5802 else
5807 else
5809 }
5814 /* In case the vectorization factor (VF) is bigger than the number
5815 of elements that we can fit in a vectype (nunits), we have to
5816 generate more than one vector stmt - i.e - we need to "unroll"
5817 the vector stmt by a factor VF/nunits. */
5820 /* Arguments are ready. Create the new vector stmts. */
5823 {
5828 }
5831 multi_step_cvt,
5835 /* After demoting op0 to cvt_type, convert it to dest. */
5837 {
5839 {
5840 /* Arguments are ready, create the new vector stmt. */
5842 gimple *new_stmt
5848 /* This is the last step of the conversion sequence. Store the
5849 vectors in SLP_NODE or in vector info of the scalar statement
5850 (or in STMT_VINFO_RELATED_STMT chain). */
5853 else
5855 }
5856 }
5858 }
5867 }
5869 /* Return true if we can assume from the scalar form of STMT_INFO that
5870 neither the scalar nor the vector forms will generate code. STMT_INFO
5871 is known not to involve a data reference. */
5873 bool
5875 {
5891 }
5893 /* Function vectorizable_assignment.
5895 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5896 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5897 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5898 Return true if STMT_INFO is vectorizable in this way. */
5900 static bool
5905 {
5906 tree vec_dest;
5907 tree scalar_dest;
5908 tree op;
5910 tree new_temp;
5916 tree vop;
5919 tree vectype_in;
5928 /* Is vectorizable assignment? */
5949 /* Multiple types in SLP are handled by creating the appropriate number of
5950 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5951 case of SLP. */
5954 else
5959 slp_tree slp_op;
5962 {
5967 }
5971 /* We can handle VIEW_CONVERT conversions that do not change the number
5972 of elements or the vector size or other conversions when the component
5973 types are nop-convertible. */
5985 {
5988 "can't convert between boolean and non "
5992 }
5994 /* We do not handle bit-precision changes. */
6001 /* But a conversion that does not change the bit-pattern is ok. */
6009 {
6012 "type conversion to/from bit-precision "
6015 }
6018 {
6021 {
6026 }
6031 cost_vec);
6033 }
6035 /* Transform. */
6039 /* Handle def. */
6042 /* Handle use. */
6045 /* Arguments are ready. create the new vector stmt. */
6047 {
6057 else
6059 }
6065 }
6068 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6069 either as shift by a scalar or by a vector. */
6071 bool
6073 {
6075 machine_mode vec_mode;
6076 optab optab;
6078 tree vectype;
6087 {
6093 }
6101 }
6104 /* Function vectorizable_shift.
6106 Check if STMT_INFO performs a shift operation that can be vectorized.
6107 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6108 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6109 Return true if STMT_INFO is vectorizable in this way. */
6111 static bool
6116 {
6117 tree vec_dest;
6118 tree scalar_dest;
6121 tree vectype;
6124 machine_mode vec_mode;
6125 tree new_temp;
6126 optab optab;
6128 machine_mode optab_op2_mode;
6131 poly_uint64 nunits_in;
6132 poly_uint64 nunits_out;
6133 tree vectype_out;
6134 tree op1_vectype;
6153 /* Is STMT a vectorizable binary/unary operation? */
6170 {
6175 }
6177 slp_tree slp_op0;
6180 {
6185 }
6186 /* If op0 is an external or constant def, infer the vector type
6187 from the scalar type. */
6193 {
6198 }
6205 stmt_vec_info op1_def_stmt_info;
6206 slp_tree slp_op1;
6209 {
6214 }
6216 /* Multiple types in SLP are handled by creating the appropriate number of
6217 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6218 case of SLP. */
6221 else
6226 /* Determine whether the shift amount is a vector, or scalar. If the
6227 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6237 {
6238 /* In SLP, need to check whether the shift count is the same,
6239 in loops if it is a constant or invariant, it is always
6240 a scalar shift. */
6242 {
6244 stmt_vec_info slpstmt_info;
6248 {
6252 }
6254 /* For internal SLP defs we have to make sure we see scalar stmts
6255 for all vector elements.
6256 ??? For different vectors we could resort to a different
6257 scalar shift operand but code-generation below simply always
6258 takes the first. */
6263 }
6265 /* If the shift amount is computed by a pattern stmt we cannot
6266 use the scalar amount directly thus give up and use a vector
6267 shift. */
6270 }
6271 else
6272 {
6277 }
6279 /* Vector shifted by vector. */
6282 {
6290 slp_op1);
6291 incompatible_op1_vectype_p
6297 && (!slp_node
6300 {
6303 "unusable type for last operand in"
6306 }
6307 }
6308 /* See if the machine has a vector shifted by scalar insn and if not
6309 then see if it has a vector shifted by vector insn. */
6310 else
6311 {
6315 {
6319 }
6320 else
6321 {
6326 {
6336 slp_op1);
6338 /* Unlike the other binary operators, shifts/rotates have
6339 the rhs being int, instead of the same type as the lhs,
6340 so make sure the scalar is the right type if we are
6341 dealing with vectors of long long/long/short/char. */
6342 incompatible_op1_vectype_p
6343 = (!op1_vectype
6348 {
6351 "unusable type for last operand in"
6354 }
6355 }
6356 }
6357 }
6359 /* Supportable by target? */
6361 {
6366 }
6370 {
6375 }
6376 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6381 {
6385 && (!incompatible_op1_vectype_p
6387 && !vect_maybe_update_slp_op_vectype
6390 {
6395 }
6396 /* Now adjust the constant shift amount in place. */
6398 && incompatible_op1_vectype_p
6400 {
6403 {
6409 }
6410 }
6416 }
6418 /* Transform. */
6425 {
6431 }
6433 /* Handle def. */
6437 {
6438 /* Vector shl and shr insn patterns can be defined with scalar
6439 operand 2 (shift operand). In this case, use constant or loop
6440 invariant op1 directly, without extending it to vector mode
6441 first. */
6444 {
6451 /* Store vec_oprnd1 for every vector stmt to be created.
6452 We check during the analysis that all the shift arguments
6453 are the same.
6454 TODO: Allow different constants for different vector
6455 stmts generated for an SLP instance. */
6459 }
6460 }
6462 {
6464 {
6465 /* If the argument was the same in all lanes create
6466 the correctly typed vector shift amount directly. */
6475 }
6477 /* The constant shift amount has been adjusted in place. */
6478 ;
6479 else
6481 }
6483 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6484 (a special case for certain kind of vector shifts); otherwise,
6485 operand 1 should be of a vector type (the usual case). */
6490 /* Arguments are ready. Create the new vector stmt. */
6492 {
6493 /* For internal defs where we need to use a scalar shift arg
6494 extract the first lane. */
6496 {
6499 gassign *new_stmt
6502 vop1,
6504 bitsize_zero_node));
6507 }
6508 else
6516 else
6518 }
6527 }
6529 /* Function vectorizable_operation.
6531 Check if STMT_INFO performs a binary, unary or ternary operation that can
6532 be vectorized.
6533 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6534 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6535 Return true if STMT_INFO is vectorizable in this way. */
6537 static bool
6542 {
6543 tree vec_dest;
6544 tree scalar_dest;
6546 tree vectype;
6549 machine_mode vec_mode;
6550 tree new_temp;
6552 optab optab;
6557 poly_uint64 nunits_in;
6558 poly_uint64 nunits_out;
6559 tree vectype_out;
6576 /* Is STMT a vectorizable binary/unary operation? */
6581 /* Loads and stores are handled in vectorizable_{load,store}. */
6587 /* Shifts are handled in vectorizable_shift. */
6594 /* Comparisons are handled in vectorizable_comparison. */
6598 /* Conditions are handled in vectorizable_condition. */
6602 /* For pointer addition and subtraction, we should use the normal
6603 plus and minus for the vector operation. */
6609 /* Support only unary or binary operations. */
6612 {
6616 op_type);
6618 }
6623 /* Most operations cannot handle bit-precision types without extra
6624 truncations. */
6628 /* Exception are bitwise binary operations. */
6632 {
6637 }
6639 slp_tree slp_op0;
6642 {
6647 }
6650 /* If op0 is an external or constant def, infer the vector type
6651 from the scalar type. */
6653 {
6654 /* For boolean type we cannot determine vectype by
6655 invariant value (don't know whether it is a vector
6656 of booleans or vector of integers). We use output
6657 vectype because operations on boolean don't change
6658 type. */
6660 {
6662 {
6667 }
6669 }
6670 else
6672 slp_node);
6673 }
6677 {
6684 }
6695 {
6698 {
6703 }
6711 }
6713 {
6716 {
6721 }
6729 }
6731 /* Multiple types in SLP are handled by creating the appropriate number of
6732 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6733 case of SLP. */
6735 {
6738 }
6739 else
6740 {
6743 }
6747 /* Reject attempts to combine mask types with nonmask types, e.g. if
6748 we have an AND between a (nonmask) boolean loaded from memory and
6749 a (mask) boolean result of a comparison.
6751 TODO: We could easily fix these cases up using pattern statements. */
6755 {
6760 }
6762 /* Supportable by target? */
6767 else
6768 {
6771 {
6776 }
6779 }
6783 {
6787 /* When vec_mode is not a vector mode and we verified ops we
6788 do not have to lower like AND are natively supported let
6789 those through even when the mode isn't word_mode. For
6790 ops we have to lower the lowering code assumes we are
6791 dealing with word_mode. */
6796 /* Check only during analysis. */
6798 {
6802 }
6807 }
6815 /* If operating on inactive elements could generate spurious traps,
6816 we need to restrict the operation to active lanes. Note that this
6817 specifically doesn't apply to unhoisted invariants, since they
6818 operate on the same value for every lane.
6820 Similarly, if this operation is part of a reduction, a fully-masked
6821 loop should only change the active lanes of the reduction chain,
6822 keeping the inactive lanes as-is. */
6827 {
6831 {
6834 OPTIMIZE_FOR_SPEED))
6839 OPTIMIZE_FOR_SPEED))
6842 else
6843 {
6846 "can't use a fully-masked loop because no"
6849 }
6850 }
6852 /* Put types on constant and invariant SLP children. */
6857 {
6862 }
6869 {
6870 /* The above vect_model_simple_cost call handles constants
6871 in the prologue and (mis-)costs one of the stmts as
6872 vector stmt. See below for the actual lowering that will
6873 be applied. */
6874 unsigned n
6877 {
6888 /* Bit operations do not have extra cost and are accounted
6889 as vector stmt by vect_model_simple_cost. */
6892 }
6894 {
6895 /* We also need to materialize two large constants. */
6900 }
6901 }
6903 }
6905 /* Transform. */
6914 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6915 vectors with unsigned elements, but the result is signed. So, we
6916 need to compute the MINUS_EXPR into vectype temporary and
6917 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6920 {
6923 }
6924 /* Handle def. */
6925 else
6928 /* In case the vectorization factor (VF) is bigger than the number
6929 of elements that we can fit in a vectype (nunits), we have to generate
6930 more than one vector stmt - i.e - we need to "unroll" the
6931 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6932 from one copy of the vector stmt to the next, in the field
6933 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6934 stages to find the correct vector defs to be used when vectorizing
6935 stmts that use the defs of the current stmt. The example below
6936 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6937 we need to create 4 vectorized stmts):
6939 before vectorization:
6940 RELATED_STMT VEC_STMT
6941 S1: x = memref - -
6942 S2: z = x + 1 - -
6944 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6945 there):
6946 RELATED_STMT VEC_STMT
6947 VS1_0: vx0 = memref0 VS1_1 -
6948 VS1_1: vx1 = memref1 VS1_2 -
6949 VS1_2: vx2 = memref2 VS1_3 -
6950 VS1_3: vx3 = memref3 - -
6951 S1: x = load - VS1_0
6952 S2: z = x + 1 - -
6954 step2: vectorize stmt S2 (done here):
6955 To vectorize stmt S2 we first need to find the relevant vector
6956 def for the first operand 'x'. This is, as usual, obtained from
6957 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
6958 that defines 'x' (S1). This way we find the stmt VS1_0, and the
6959 relevant vector def 'vx0'. Having found 'vx0' we can generate
6960 the vector stmt VS2_0, and as usual, record it in the
6961 STMT_VINFO_VEC_STMT of stmt S2.
6962 When creating the second copy (VS2_1), we obtain the relevant vector
6963 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
6964 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
6965 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
6966 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
6967 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
6968 chain of stmts and pointers:
6969 RELATED_STMT VEC_STMT
6970 VS1_0: vx0 = memref0 VS1_1 -
6971 VS1_1: vx1 = memref1 VS1_2 -
6972 VS1_2: vx2 = memref2 VS1_3 -
6973 VS1_3: vx3 = memref3 - -
6974 S1: x = load - VS1_0
6975 VS2_0: vz0 = vx0 + v1 VS2_1 -
6976 VS2_1: vz1 = vx1 + v1 VS2_2 -
6977 VS2_2: vz2 = vx2 + v1 VS2_3 -
6978 VS2_3: vz3 = vx3 + v1 - -
6979 S2: z = x + 1 - VS2_0 */
6983 /* Arguments are ready. Create the new vector stmt. */
6985 {
6992 {
6993 /* Lower the operation. This follows vector lowering. */
6996 tree word_type
7000 tree high_bits
7009 {
7027 else
7032 {
7038 }
7047 }
7048 else
7049 {
7066 }
7068 signs);
7078 }
7080 {
7081 tree mask;
7085 else
7086 /* Dummy mask. */
7096 {
7097 /* Perform the operation on active elements only and take
7098 inactive elements from the reduction chain input. */
7101 }
7102 else
7103 {
7107 }
7109 {
7112 signed char biasval
7117 }
7118 gcall *call
7121 vops);
7127 }
7128 else
7129 {
7131 /* When combining two masks check if either of them is elsewhere
7132 combined with a loop mask, if that's the case we can mark that the
7133 new combined mask doesn't need to be combined with a loop mask. */
7137 {
7139 ncopies}))
7140 {
7146 }
7149 ncopies }))
7150 {
7156 }
7157 }
7166 /* Enter the combined value into the vector cond hash so we don't
7167 AND it with a loop mask again. */
7170 }
7173 {
7176 new_temp);
7181 }
7185 else
7187 }
7197 }
7199 /* A helper function to ensure data reference DR_INFO's base alignment. */
7201 static void
7203 {
7204 /* Alignment is only analyzed for the first element of a DR group,
7205 use that to look at base alignment we need to enforce. */
7212 {
7215 // We should only be able to increase the alignment of a base object if
7216 // we know what its new alignment should be at compile time.
7223 {
7226 }
7228 }
7229 }
7232 /* Function get_group_alias_ptr_type.
7234 Return the alias type for the group starting at FIRST_STMT_INFO. */
7236 static tree
7238 {
7244 {
7248 {
7253 }
7255 }
7257 }
7260 /* Function scan_operand_equal_p.
7262 Helper function for check_scan_store. Compare two references
7263 with .GOMP_SIMD_LANE bases. */
7265 static bool
7267 {
7272 {
7273 machine_mode mode;
7283 {
7289 {
7294 }
7295 }
7296 }
7303 {
7307 {
7310 {
7313 {
7319 {
7322 }
7323 }
7325 {
7328 }
7331 {
7335 }
7344 }
7348 }
7349 }
7351 }
7355 /* Normal permutation. */
7356 scan_store_kind_perm,
7358 /* Whole vector left shift permutation with zero init. */
7359 scan_store_kind_lshift_zero,
7361 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7362 scan_store_kind_lshift_cond
7363 };
7365 /* Function check_scan_store.
7367 Verify if we can perform the needed permutations or whole vector shifts.
7368 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7369 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7370 to do at each step. */
7372 static int
7375 {
7387 {
7393 {
7396 }
7397 else
7398 {
7403 }
7406 {
7411 {
7415 /* Whole vector shifts shift in zeros, so if init is all zero
7416 constant, there is no need to do anything further. */
7420 {
7425 }
7426 }
7428 }
7430 {
7435 }
7436 }
7439 }
7442 /* Function check_scan_store.
7444 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7446 static bool
7449 vect_memory_access_type memory_access_type)
7450 {
7453 tree ref_type;
7457 || mask
7469 {
7474 }
7476 /* We need to pattern match code built by OpenMP lowering and simplified
7477 by following optimizations into something we can handle.
7478 #pragma omp simd reduction(inscan,+:r)
7479 for (...)
7480 {
7481 r += something ();
7482 #pragma omp scan inclusive (r)
7483 use (r);
7484 }
7485 shall have body with:
7486 // Initialization for input phase, store the reduction initializer:
7487 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7488 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7489 D.2042[_21] = 0;
7490 // Actual input phase:
7491 ...
7492 r.0_5 = D.2042[_20];
7493 _6 = _4 + r.0_5;
7494 D.2042[_20] = _6;
7495 // Initialization for scan phase:
7496 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7497 _26 = D.2043[_25];
7498 _27 = D.2042[_25];
7499 _28 = _26 + _27;
7500 D.2043[_25] = _28;
7501 D.2042[_25] = _28;
7502 // Actual scan phase:
7503 ...
7504 r.1_8 = D.2042[_20];
7505 ...
7506 The "omp simd array" variable D.2042 holds the privatized copy used
7507 inside of the loop and D.2043 is another one that holds copies of
7508 the current original list item. The separate GOMP_SIMD_LANE ifn
7509 kinds are there in order to allow optimizing the initializer store
7510 and combiner sequence, e.g. if it is originally some C++ish user
7511 defined reduction, but allow the vectorizer to pattern recognize it
7512 and turn into the appropriate vectorized scan.
7514 For exclusive scan, this is slightly different:
7515 #pragma omp simd reduction(inscan,+:r)
7516 for (...)
7517 {
7518 use (r);
7519 #pragma omp scan exclusive (r)
7520 r += something ();
7521 }
7522 shall have body with:
7523 // Initialization for input phase, store the reduction initializer:
7524 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7525 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7526 D.2042[_21] = 0;
7527 // Actual input phase:
7528 ...
7529 r.0_5 = D.2042[_20];
7530 _6 = _4 + r.0_5;
7531 D.2042[_20] = _6;
7532 // Initialization for scan phase:
7533 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7534 _26 = D.2043[_25];
7535 D.2044[_25] = _26;
7536 _27 = D.2042[_25];
7537 _28 = _26 + _27;
7538 D.2043[_25] = _28;
7539 // Actual scan phase:
7540 ...
7541 r.1_8 = D.2044[_20];
7542 ... */
7545 {
7546 /* Match the D.2042[_21] = 0; store above. Just require that
7547 it is a constant or external definition store. */
7549 {
7550 fail_init:
7555 }
7565 /* These stores can be vectorized normally. */
7567 }
7570 {
7571 fail:
7576 }
7585 bool inscan_var_store
7589 {
7591 {
7592 use_operand_p use_p;
7593 imm_use_iterator iter;
7595 {
7602 || other_store_stmt
7606 }
7612 }
7613 }
7615 {
7616 use_operand_p use_p;
7617 imm_use_iterator iter;
7619 {
7626 }
7627 }
7628 else
7638 /* For pointer addition, we should use the normal plus for the vector
7639 operation. */
7641 {
7649 }
7677 {
7683 tree lrhs;
7686 else
7688 use_operand_p use_p;
7689 imm_use_iterator iter;
7691 {
7698 }
7699 }
7707 stmt_vec_info other_store_stmt_info
7720 {
7724 }
7735 {
7746 }
7765 {
7770 }
7778 /* The IL is as expected, now check if we can actually vectorize it.
7779 Inclusive scan:
7780 _26 = D.2043[_25];
7781 _27 = D.2042[_25];
7782 _28 = _26 + _27;
7783 D.2043[_25] = _28;
7784 D.2042[_25] = _28;
7785 should be vectorized as (where _40 is the vectorized rhs
7786 from the D.2042[_21] = 0; store):
7787 _30 = MEM <vector(8) int> [(int *)&D.2043];
7788 _31 = MEM <vector(8) int> [(int *)&D.2042];
7789 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7790 _33 = _31 + _32;
7791 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7792 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7793 _35 = _33 + _34;
7794 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7795 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7796 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7797 _37 = _35 + _36;
7798 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7799 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7800 _38 = _30 + _37;
7801 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7802 MEM <vector(8) int> [(int *)&D.2043] = _39;
7803 MEM <vector(8) int> [(int *)&D.2042] = _38;
7804 Exclusive scan:
7805 _26 = D.2043[_25];
7806 D.2044[_25] = _26;
7807 _27 = D.2042[_25];
7808 _28 = _26 + _27;
7809 D.2043[_25] = _28;
7810 should be vectorized as (where _40 is the vectorized rhs
7811 from the D.2042[_21] = 0; store):
7812 _30 = MEM <vector(8) int> [(int *)&D.2043];
7813 _31 = MEM <vector(8) int> [(int *)&D.2042];
7814 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7815 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7816 _34 = _32 + _33;
7817 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7818 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7819 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7820 _36 = _34 + _35;
7821 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7822 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7823 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7824 _38 = _36 + _37;
7825 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7826 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7827 _39 = _30 + _38;
7828 _50 = _31 + _39;
7829 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7830 MEM <vector(8) int> [(int *)&D.2044] = _39;
7831 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7842 }
7845 /* Function vectorizable_scan_store.
7847 Helper of vectorizable_score, arguments like on vectorizable_store.
7848 Handle only the transformation, checking is done in check_scan_store. */
7850 static bool
7854 {
7869 bool inscan_var_store
7873 {
7874 use_operand_p use_p;
7875 imm_use_iterator iter;
7877 {
7883 }
7884 }
7905 {
7909 }
7924 {
7931 else
7932 {
7937 }
7941 {
7948 }
7949 else
7951 }
7964 /* The initialization is invariant. */
7969 {
7970 /* We want to lookup the vector operands of the reduction, not those
7971 of the store - for SLP we have to use the proper SLP node for the
7972 lookup, which should be the single child of the scan store. */
7975 /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
7976 them in SLP child order. So we have to swap here with logic
7977 similar to above. */
7978 stmt_vec_info load
7986 }
7987 else
7991 {
8001 {
8005 dataref_offset);
8010 {
8013 }
8014 }
8018 {
8021 (zero_vec
8028 {
8031 }
8034 {
8035 /* Whole vector shift shifted in zero bits, but if *init
8036 is not initializer_zerop, we need to replace those elements
8037 with elements from vec_oprnd1. */
8051 }
8053 /* For exclusive scan, perform the perms[i] permutation once
8054 more. */
8058 {
8062 }
8071 }
8080 /* For exclusive scan, new_temp computed above is the exclusive scan
8081 prefix sum. Turn it into inclusive prefix sum for the broadcast
8082 of the last element into orig. */
8084 {
8090 }
8100 {
8103 dataref_offset);
8109 }
8110 }
8114 {
8120 dataref_offset);
8126 }
8128 }
8131 /* Function vectorizable_store.
8133 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8134 that can be vectorized.
8135 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8136 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8137 Return true if STMT_INFO is vectorizable in this way. */
8139 static bool
8144 {
8145 tree data_ref;
8147 tree elem_type;
8150 machine_mode vec_mode;
8151 tree dummy;
8159 stmt_vec_info first_stmt_info;
8165 tree aggr_type;
8166 gather_scatter_info gs_info;
8167 poly_uint64 vf;
8168 vec_load_store_type vls_type;
8169 tree ref_type;
8178 /* Is vectorizable store? */
8183 {
8196 }
8197 else
8198 {
8209 mask_index = vect_slp_child_index_for_operand
8216 }
8218 /* Cannot have hybrid store SLP -- that would mean storing to the
8219 same location twice. */
8226 {
8229 }
8230 else
8233 /* Multiple types in SLP are handled by creating the appropriate number of
8234 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8235 case of SLP. */
8238 else
8243 /* FORNOW. This restriction should be relaxed. */
8247 {
8252 }
8254 tree op;
8255 slp_tree op_node;
8266 vect_memory_access_type memory_access_type;
8269 poly_int64 poffset;
8270 internal_fn lanes_ifn;
8280 {
8285 }
8288 {
8290 {
8295 }
8299 {
8304 }
8308 {
8313 }
8314 }
8315 else
8316 {
8317 /* FORNOW. In some cases can vectorize even if data-type not supported
8318 (e.g. - array initialization with 0). */
8321 }
8328 {
8332 }
8333 else
8334 {
8338 }
8341 {
8343 memory_access_type))
8345 }
8349 {
8357 mask);
8361 || (mask
8363 mask_vectype))))
8364 {
8369 }
8380 /* As function vect_transform_stmt shows, for interleaving stores
8381 the whole chain is vectorized when the last store in the chain
8382 is reached, the other stores in the group are skipped. So we
8383 want to only cost the last one here, but it's not trivial to
8384 get the last, as it's equivalent to use the first one for
8385 costing, use the first one instead. */
8387 && !slp
8390 }
8393 /* Transform. */
8398 {
8402 {
8413 "vect_model_store_cost: inside_cost = %d, "
8418 }
8421 }
8424 {
8425 /* FORNOW */
8427 || !loop
8431 {
8433 /* VEC_NUM is the number of vect stmts to be created for this
8434 group. */
8442 }
8443 else
8444 /* VEC_NUM is the number of vect stmts to be created for this
8445 group. */
8449 }
8450 else
8455 ncopies);
8457 /* Check if we need to update prologue cost for invariant,
8458 and update it accordingly if so. If it's not for
8459 interleaving store, we can just check vls_type; but if
8460 it's for interleaving store, need to check the def_type
8461 of the stored value since the current vls_type is just
8462 for first_stmt_info. */
8464 {
8469 {
8475 }
8480 };
8484 {
8486 gimple_stmt_iterator incr_gsi;
8489 tree offvar;
8490 tree ivstep;
8491 tree running_off;
8494 tree dr_offset;
8496 /* Checked by get_load_store_type. */
8503 stride_base
8504 = fold_build_pointer_plus
8511 /* For a store with loop-invariant (but other than power-of-2)
8512 stride (i.e. not a grouped access) like so:
8514 for (i = 0; i < n; i += stride)
8515 array[i] = ...;
8517 we generate a new induction variable and new stores from
8518 the components of the (vectorized) rhs:
8520 for (j = 0; ; j += VF*stride)
8521 vectemp = ...;
8522 tmp1 = vectemp[0];
8523 array[j] = tmp1;
8524 tmp2 = vectemp[1];
8525 array[j + stride] = tmp2;
8526 ...
8527 */
8534 {
8537 {
8539 dr_alignment_support dr_align
8541 mis_align);
8544 {
8551 }
8552 }
8554 {
8560 /* First check if vec_extract optab doesn't support extraction
8561 of vector elts directly. */
8563 machine_mode vmode;
8570 {
8571 /* Try to avoid emitting an extract of vector elements
8572 by performing the extracts using an integer type of the
8573 same size, extracting from a vector of those and then
8574 re-interpreting it as the original vector type if
8575 supported. */
8576 unsigned lsize
8579 /* If we can't construct such a vector fall back to
8580 element extracts from the original vector type and
8581 element size stores. */
8589 {
8594 }
8595 /* Else fall back to vector extraction anyway.
8596 Fewer stores are more important than avoiding spilling
8597 of the vector we extract from. Compared to the
8598 construction case in vectorizable_load no store-forwarding
8599 issue exists here for reasonable archs. */
8600 }
8601 }
8604 }
8607 {
8621 }
8626 /* For costing some adjacent vector stores, we'd like to cost with
8627 the total number of them once instead of cost each one by one. */
8630 {
8633 {
8635 {
8637 tree pos
8644 }
8645 }
8651 else
8654 unsigned HOST_WIDE_INT
8657 {
8659 {
8661 /* Pun the vector to extract from if necessary. */
8663 {
8665 tree cvt
8670 }
8671 }
8673 {
8675 {
8676 /* Only need vector extracting when there are more
8677 than one stores. */
8679 inside_cost
8682 /* Take a single lane vector type store as scalar
8683 store to avoid ICE like 110776. */
8686 n_adjacent_stores++;
8687 else
8688 inside_cost
8692 }
8696 /* Extract the i'th component. */
8704 GSI_SAME_STMT);
8712 /* And store it to *running_off. */
8719 {
8727 }
8730 {
8734 }
8735 }
8736 }
8741 }
8744 {
8751 "vect_model_store_cost: inside_cost = %d, "
8754 }
8757 }
8760 vec_loop_masks *loop_masks
8764 vec_loop_lens *loop_lens
8769 /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8770 are some difference here. We cannot enable both the lens and masks
8771 during transform but it is allowed during analysis.
8772 Shouldn't go with length-based approach if fully masked. */
8774 /* The cost_vec is NULL during transfrom. */
8777 /* Targets with store-lane instructions must not require explicit
8778 realignment. vect_supportable_dr_alignment always returns either
8779 dr_aligned or dr_unaligned_supported for masked operations. */
8781 && !mask
8790 tree bump;
8793 {
8796 }
8798 {
8803 }
8804 else
8805 {
8808 else
8813 }
8818 /* In case the vectorization factor (VF) is bigger than the number
8819 of elements that we can fit in a vectype (nunits), we have to generate
8820 more than one vector stmt - i.e - we need to "unroll" the
8821 vector stmt by a factor VF/nunits. */
8823 /* In case of interleaving (non-unit grouped access):
8825 S1: &base + 2 = x2
8826 S2: &base = x0
8827 S3: &base + 1 = x1
8828 S4: &base + 3 = x3
8830 We create vectorized stores starting from base address (the access of the
8831 first stmt in the chain (S2 in the above example), when the last store stmt
8832 of the chain (S4) is reached:
8834 VS1: &base = vx2
8835 VS2: &base + vec_size*1 = vx0
8836 VS3: &base + vec_size*2 = vx1
8837 VS4: &base + vec_size*3 = vx3
8839 Then permutation statements are generated:
8841 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8842 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8843 ...
8845 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8846 (the order of the data-refs in the output of vect_permute_store_chain
8847 corresponds to the order of scalar stmts in the interleaving chain - see
8848 the documentation of vect_permute_store_chain()).
8850 In case of both multiple types and interleaving, above vector stores and
8851 permutation stmts are created for every copy. The result vector stmts are
8852 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8853 STMT_VINFO_RELATED_STMT for the next copies.
8854 */
8864 {
8866 /* Update all incoming store operand nodes, the general handling
8867 above only handles the mask and the first store operand node. */
8871 {
8876 }
8878 /* For costing some adjacent vector stores, we'd like to cost with
8879 the total number of them once instead of cost each one by one. */
8884 {
8887 {
8888 /* For interleaved stores we collect vectorized defs for all
8889 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8890 as an input to vect_permute_store_chain(). */
8893 {
8894 /* Since gaps are not supported for interleaved stores,
8895 DR_GROUP_SIZE is the exact number of stmts in the
8896 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8901 {
8907 }
8909 }
8912 {
8914 {
8917 else
8920 mask_vectype);
8922 }
8924 dataref_ptr
8928 }
8929 }
8931 {
8933 /* DR_CHAIN is then used as an input to
8934 vect_permute_store_chain(). */
8936 {
8937 /* We should have caught mismatched types earlier. */
8941 {
8944 }
8945 }
8950 }
8953 {
8956 }
8958 /* Get an array into which we can store the individual vectors. */
8961 /* Invalidate the current contents of VEC_ARRAY. This should
8962 become an RTL clobber too, which prevents the vector registers
8963 from being upward-exposed. */
8966 /* Store the individual vectors into the array. */
8968 {
8970 {
8971 slp_tree child;
8974 else
8977 }
8978 else
8981 i);
8982 }
8995 {
8999 else
9001 signed char biasval
9005 {
9008 }
9009 }
9013 {
9014 /* Emit:
9015 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
9016 LEN, BIAS, VEC_ARRAY). */
9022 vec_array);
9023 }
9025 {
9026 /* Emit:
9027 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
9028 VEC_ARRAY). */
9034 }
9035 else
9036 {
9037 /* Emit:
9038 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
9042 }
9047 /* Record that VEC_ARRAY is now dead. */
9053 }
9056 {
9063 "vect_model_store_cost: inside_cost = %d, "
9066 }
9069 }
9072 {
9077 {
9080 {
9085 {
9086 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
9087 DR_CHAIN is of size 1. */
9091 else
9095 {
9098 else
9100 ncopies,
9102 mask_vectype);
9103 }
9109 else
9110 dataref_ptr
9114 bump);
9115 }
9116 }
9118 {
9123 }
9127 {
9129 {
9133 /* We should have catched mismatched types earlier. */
9136 }
9142 {
9150 }
9153 {
9155 {
9157 inside_cost
9161 }
9168 {
9173 else
9175 signed char biasval
9179 {
9182 }
9183 }
9187 call = gimple_build_call_internal
9192 call = gimple_build_call_internal
9195 else
9202 }
9204 {
9205 /* The builtin decls path for scatter is legacy, x86 only. */
9207 && (!final_mask
9208 || SCALAR_INT_MODE_P
9211 {
9213 inside_cost
9217 }
9218 poly_uint64 offset_nunits
9221 {
9222 new_stmt = vect_build_one_scatter_store_call
9228 }
9230 {
9231 /* We have a offset vector with half the number of
9232 lanes but the builtins will store full vectype
9233 data from the lower lanes. */
9234 new_stmt = vect_build_one_scatter_store_call
9236 dataref_ptr,
9247 tree perm_mask
9251 perm_mask);
9257 {
9259 VEC_UNPACK_HI_EXPR,
9260 final_mask);
9261 final_mask = make_ssa_name
9266 }
9267 new_stmt = vect_build_one_scatter_store_call
9269 dataref_ptr,
9274 }
9276 {
9277 /* We have a offset vector with double the number of
9278 lanes. Select the low/high part accordingly. */
9281 {
9288 tree perm_mask = vect_gen_perm_mask_checked
9291 VEC_PERM_EXPR,
9292 vec_offset,
9293 vec_offset,
9294 perm_mask);
9299 }
9300 new_stmt = vect_build_one_scatter_store_call
9306 }
9307 else
9309 }
9310 else
9311 {
9312 /* Emulated scatter. */
9315 {
9317 /* For emulated scatter N offset vector element extracts
9318 (we assume the scalar scaling and ptr + offset add is
9319 consumed by the load). */
9320 inside_cost
9323 /* N scalar stores plus extracting the elements. */
9324 inside_cost
9327 inside_cost
9331 }
9334 unsigned HOST_WIDE_INT const_offset_nunits
9340 unsigned HOST_WIDE_INT elt_size
9342 /* We support offset vectors with more elements
9343 than the data vector for now. */
9344 unsigned HOST_WIDE_INT factor
9347 unsigned elt_offset
9354 {
9355 /* Compute the offsetted pointer. */
9358 tree idx
9364 tree ptr
9369 /* Extract the element to be stored. */
9370 tree elt
9377 tree ref
9383 }
9386 }
9387 }
9390 }
9397 "vect_model_store_cost: inside_cost = %d, "
9402 }
9410 /* For costing some adjacent vector stores, we'd like to cost with
9411 the total number of them once instead of cost each one by one. */
9416 {
9419 {
9421 {
9422 /* Get vectorized arguments for SLP_NODE. */
9428 }
9429 else
9430 {
9431 /* For interleaved stores we collect vectorized defs for all the
9432 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9433 input to vect_permute_store_chain().
9435 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9436 is of size 1. */
9439 {
9440 /* Since gaps are not supported for interleaved stores,
9441 DR_GROUP_SIZE is the exact number of stmts in the chain.
9442 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9443 that there is no interleaving, DR_GROUP_SIZE is 1,
9444 and only one iteration of the loop will be executed. */
9448 else
9449 {
9455 }
9457 }
9459 {
9462 mask_vectype);
9464 }
9465 }
9467 /* We should have catched mismatched types earlier. */
9471 bool simd_lane_access_p
9474 && simd_lane_access_p
9475 && !loop_masks
9482 {
9485 }
9487 dataref_ptr
9492 }
9494 {
9496 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9497 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9498 of size 1. */
9500 {
9503 }
9508 else
9511 }
9515 {
9516 /* Permute. */
9519 {
9526 "vect_model_store_cost: "
9528 group_size);
9529 }
9530 else
9533 }
9537 {
9539 {
9543 /* For grouped stores vectorized defs are interleaved in
9544 vect_permute_store_chain(). */
9546 }
9549 {
9553 else
9554 {
9560 /* Generate the permute statement. */
9561 gimple *perm_stmt
9565 gsi);
9569 }
9570 }
9573 {
9574 n_adjacent_stores++;
9577 {
9581 }
9584 }
9600 /* Bump the vector pointer. */
9610 {
9613 }
9614 else
9619 misalign);
9622 /* Compute IFN when LOOP_LENS or final_mask valid. */
9627 {
9628 opt_machine_mode new_ovmode
9631 unsigned factor
9636 }
9638 {
9643 }
9646 {
9648 {
9649 /* Pass VF value to 'len' argument of
9650 MASK_LEN_STORE if LOOP_LENS is invalid. */
9652 }
9654 {
9655 /* Pass all ones value to 'mask' argument of
9656 MASK_LEN_STORE if final_mask is invalid. */
9659 }
9660 }
9662 {
9663 signed char biasval
9667 }
9669 /* Arguments are ready. Create the new vector stmt. */
9671 {
9674 /* Need conversion if it's wrapped with VnQI. */
9676 {
9677 tree new_vtype
9679 new_vmode);
9682 gassign *new_stmt
9686 }
9692 else
9699 }
9701 {
9703 gcall *call
9709 }
9710 else
9711 {
9712 data_ref
9714 dataref_offset ? dataref_offset
9717 ;
9718 else
9725 }
9733 }
9735 {
9739 }
9740 }
9743 {
9749 /* When vectorizing a store into the function result assign
9750 a penalty if the function returns in a multi-register location.
9751 In this case we assume we'll end up with having to spill the
9752 vector result and do piecewise loads as a conservative estimate. */
9758 {
9760 /* ??? Handle PARALLEL in some way. */
9762 {
9764 /* Assume that a single reg-reg move is possible and cheap,
9765 do not account for vector to gp register move cost. */
9767 {
9768 /* Spill. */
9769 prologue_cost
9772 /* Loads. */
9773 prologue_cost
9776 }
9777 }
9778 }
9781 "vect_model_store_cost: inside_cost = %d, "
9784 }
9787 }
9789 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9790 VECTOR_CST mask. No checks are made that the target platform supports the
9791 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9792 vect_gen_perm_mask_checked. */
9794 tree
9796 {
9797 tree mask_type;
9804 }
9806 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9807 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9809 tree
9811 {
9815 }
9817 /* Given a vector variable X and Y, that was generated for the scalar
9818 STMT_INFO, generate instructions to permute the vector elements of X and Y
9819 using permutation mask MASK_VEC, insert them at *GSI and return the
9820 permuted vector variable. */
9822 static tree
9826 {
9834 else
9838 /* Generate the permute statement. */
9843 }
9845 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9846 inserting them on the loops preheader edge. Returns true if we
9847 were successful in doing so (and thus STMT_INFO can be moved then),
9848 otherwise returns false. HOIST_P indicates if we want to hoist the
9849 definitions of all SSA uses, it would be false when we are costing. */
9851 static bool
9853 {
9854 ssa_op_iter i;
9855 use_operand_p use_p;
9859 {
9863 {
9864 /* Make sure we don't need to recurse. While we could do
9865 so in simple cases when there are more complex use webs
9866 we don't have an easy way to preserve stmt order to fulfil
9867 dependencies within them. */
9868 tree op2;
9869 ssa_op_iter i2;
9875 {
9880 }
9882 }
9883 }
9891 /* Instead of moving defs we copy them so we can zero their UID to not
9892 confuse dominance queries in the preheader. */
9895 {
9906 }
9909 }
9911 /* vectorizable_load.
9913 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9914 that can be vectorized.
9915 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9916 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9917 Return true if STMT_INFO is vectorizable in this way. */
9919 static bool
9924 {
9925 tree scalar_dest;
9932 tree elem_type;
9933 /* Avoid false positive uninitialized warning, see PR110652. */
9935 machine_mode mode;
9936 tree dummy;
9943 poly_uint64 group_gap_adj;
9949 stmt_vec_info first_stmt_info;
9957 poly_uint64 vf;
9958 tree aggr_type;
9959 gather_scatter_info gs_info;
9960 tree ref_type;
9977 {
9992 }
9993 else
9994 {
10009 mask_index = vect_slp_child_index_for_operand
10015 }
10021 {
10025 }
10026 else
10029 /* Multiple types in SLP are handled by creating the appropriate number of
10030 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
10031 case of SLP. */
10034 else
10039 /* FORNOW. This restriction should be relaxed. */
10042 {
10047 }
10049 /* Invalidate assumptions made by dependence analysis when vectorization
10050 on the unrolled body effectively re-orders stmts. */
10055 {
10058 "cannot perform implicit CSE when unrolling "
10061 }
10066 /* FORNOW. In some cases can vectorize even if data-type not supported
10067 (e.g. - data copies). */
10069 {
10074 }
10076 /* Check if the load is a part of an interleaving chain. */
10078 {
10080 /* FORNOW */
10087 /* Refuse non-SLP vectorization of SLP-only groups. */
10089 {
10094 }
10096 /* Invalidate assumptions made by dependence analysis when vectorization
10097 on the unrolled body effectively re-orders stmts. */
10101 {
10104 "cannot perform implicit CSE when performing "
10107 }
10108 }
10109 else
10112 vect_memory_access_type memory_access_type;
10115 poly_int64 poffset;
10116 internal_fn lanes_ifn;
10123 /* ??? The following checks should really be part of
10124 get_group_load_store_type. */
10130 {
10134 {
10135 /* In BB vectorization we may not actually use a loaded vector
10136 accessing elements in excess of DR_GROUP_SIZE. */
10147 {
10150 "BB vectorization with gaps at the end of "
10153 }
10154 }
10160 {
10163 vect_location,
10166 }
10167 }
10172 {
10177 }
10180 {
10182 {
10188 }
10191 {
10196 }
10200 {
10205 }
10208 {
10213 }
10214 }
10219 {
10221 && mask
10223 mask_vectype))
10224 {
10229 }
10239 mask);
10252 }
10262 /* Transform. */
10268 {
10270 /* If we have versioned for aliasing or the loop doesn't
10271 have any data dependencies that would preclude this,
10272 then we are sure this is a loop invariant load and
10273 thus we can insert it on the preheader edge.
10274 TODO: hoist_defs_of_uses should ideally be computed
10275 once at analysis time, remembered and used in the
10276 transform time. */
10278 && !nested_in_vect_loop
10281 {
10282 enum vect_cost_model_location cost_loc
10287 cost_loc);
10292 "vect_model_load_cost: inside_cost = %d, "
10296 }
10298 {
10308 tree vuse;
10311 else
10317 }
10318 /* These copies are all equivalent. */
10322 else
10323 {
10328 }
10333 else
10334 {
10338 }
10340 }
10344 {
10345 gimple_stmt_iterator incr_gsi;
10347 tree offvar;
10348 tree ivstep;
10349 tree running_off;
10352 /* Checked by get_load_store_type. */
10355 tree dr_offset;
10362 {
10365 }
10366 else
10367 {
10370 }
10374 {
10377 }
10378 else
10379 {
10381 cst_offset
10384 first_stmt_info));
10387 }
10390 {
10398 /* For a load with loop-invariant (but other than power-of-2)
10399 stride (i.e. not a grouped access) like so:
10401 for (i = 0; i < n; i += stride)
10402 ... = array[i];
10404 we generate a new induction variable and new accesses to
10405 form a new vector (or vectors, depending on ncopies):
10407 for (j = 0; ; j += VF*stride)
10408 tmp1 = array[j];
10409 tmp2 = array[j + stride];
10410 ...
10411 vectemp = {tmp1, tmp2, ...}
10412 */
10426 }
10436 {
10438 /* Use the target vector type if the group size is a multiple
10439 of it. */
10441 {
10445 }
10446 /* Else use the biggest vector we can load the group without
10447 accessing excess elements. */
10449 {
10450 tree ptype;
10451 tree vtype
10455 {
10460 }
10461 }
10462 /* Else fall back to the default element-wise access. */
10464 }
10465 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10470 {
10471 /* For SLP permutation support we need to load the whole group,
10472 not only the number of vector stmts the permutation result
10473 fits in. */
10475 {
10476 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10477 variable VF. */
10481 }
10482 else
10484 }
10486 unsigned HOST_WIDE_INT
10489 /* For costing some adjacent vector loads, we'd like to cost with
10490 the total number of them once instead of cost each one by one. */
10493 {
10498 {
10500 {
10501 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10502 avoid ICE, see PR110776. */
10505 n_adjacent_loads++;
10506 else
10510 }
10524 {
10525 n_groups++;
10526 /* When doing SLP make sure to not load elements from
10527 the next vector iteration, those will not be accessed
10528 so just use the last element again. See PR107451. */
10530 {
10532 gimple *incr
10537 }
10539 }
10540 }
10543 {
10547 else
10548 {
10554 {
10555 new_stmt
10557 VIEW_CONVERT_EXPR,
10561 gsi);
10562 }
10563 }
10564 }
10567 {
10569 {
10572 else
10574 }
10575 else
10576 {
10580 }
10581 }
10582 }
10584 {
10587 {
10593 }
10594 else
10597 }
10600 {
10608 "vect_model_load_cost: inside_cost = %u, "
10610 inside_cost);
10611 }
10614 }
10622 {
10624 {
10627 }
10628 else
10629 {
10632 }
10633 /* For SLP vectorization we directly vectorize a subchain
10634 without permutation. */
10637 /* For BB vectorization always use the first stmt to base
10638 the data ref pointer on. */
10640 first_stmt_info_for_drptr
10643 /* Check if the chain of loads is already vectorized. */
10645 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10646 ??? But we can only do so if there is exactly one
10647 as we have no way to get at the rest. Leave the CSE
10648 opportunity alone.
10649 ??? With the group load eventually participating
10650 in multiple different permutations (having multiple
10651 slp nodes which refer to the same group) the CSE
10652 is even wrong code. See PR56270. */
10654 {
10657 }
10661 /* VEC_NUM is the number of vect stmts to be created for this group. */
10663 {
10665 /* If an SLP permutation is from N elements to N elements,
10666 and if one vector holds a whole number of N, we can load
10667 the inputs to the permutation in the same way as an
10668 unpermuted sequence. In other cases we need to load the
10669 whole group, not only the number of vector stmts the
10670 permutation result fits in. */
10673 /* We do not support grouped accesses in a nested loop,
10674 instead the access is contiguous but it might be
10675 permuted. No gap adjustment is needed though. */
10680 {
10681 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10682 variable VF; see vect_transform_slp_perm_load. */
10687 }
10688 else
10689 {
10691 group_gap_adj
10693 }
10694 }
10695 else
10699 }
10700 else
10701 {
10709 }
10712 vec_loop_masks *loop_masks
10716 vec_loop_lens *loop_lens
10721 /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10722 are some difference here. We cannot enable both the lens and masks
10723 during transform but it is allowed during analysis.
10724 Shouldn't go with length-based approach if fully masked. */
10726 /* The cost_vec is NULL during transfrom. */
10729 /* Targets with store-lane instructions must not require explicit
10730 realignment. vect_supportable_dr_alignment always returns either
10731 dr_aligned or dr_unaligned_supported for masked operations. */
10733 && !mask
10738 /* In case the vectorization factor (VF) is bigger than the number
10739 of elements that we can fit in a vectype (nunits), we have to generate
10740 more than one vector stmt - i.e - we need to "unroll" the
10741 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10742 from one copy of the vector stmt to the next, in the field
10743 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10744 stages to find the correct vector defs to be used when vectorizing
10745 stmts that use the defs of the current stmt. The example below
10746 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10747 need to create 4 vectorized stmts):
10749 before vectorization:
10750 RELATED_STMT VEC_STMT
10751 S1: x = memref - -
10752 S2: z = x + 1 - -
10754 step 1: vectorize stmt S1:
10755 We first create the vector stmt VS1_0, and, as usual, record a
10756 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10757 Next, we create the vector stmt VS1_1, and record a pointer to
10758 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10759 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10760 stmts and pointers:
10761 RELATED_STMT VEC_STMT
10762 VS1_0: vx0 = memref0 VS1_1 -
10763 VS1_1: vx1 = memref1 VS1_2 -
10764 VS1_2: vx2 = memref2 VS1_3 -
10765 VS1_3: vx3 = memref3 - -
10766 S1: x = load - VS1_0
10767 S2: z = x + 1 - -
10768 */
10770 /* In case of interleaving (non-unit grouped access):
10772 S1: x2 = &base + 2
10773 S2: x0 = &base
10774 S3: x1 = &base + 1
10775 S4: x3 = &base + 3
10777 Vectorized loads are created in the order of memory accesses
10778 starting from the access of the first stmt of the chain:
10780 VS1: vx0 = &base
10781 VS2: vx1 = &base + vec_size*1
10782 VS3: vx3 = &base + vec_size*2
10783 VS4: vx4 = &base + vec_size*3
10785 Then permutation statements are generated:
10787 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10788 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10789 ...
10791 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10792 (the order of the data-refs in the output of vect_permute_load_chain
10793 corresponds to the order of scalar stmts in the interleaving chain - see
10794 the documentation of vect_permute_load_chain()).
10795 The generation of permutation stmts and recording them in
10796 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10798 In case of both multiple types and interleaving, the vector loads and
10799 permutation stmts above are created for every copy. The result vector
10800 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10801 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10803 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10804 on a target that supports unaligned accesses (dr_unaligned_supported)
10805 we generate the following code:
10806 p = initial_addr;
10807 indx = 0;
10808 loop {
10809 p = p + indx * vectype_size;
10810 vec_dest = *(p);
10811 indx = indx + 1;
10812 }
10814 Otherwise, the data reference is potentially unaligned on a target that
10815 does not support unaligned accesses (dr_explicit_realign_optimized) -
10816 then generate the following code, in which the data in each iteration is
10817 obtained by two vector loads, one from the previous iteration, and one
10818 from the current iteration:
10819 p1 = initial_addr;
10820 msq_init = *(floor(p1))
10821 p2 = initial_addr + VS - 1;
10822 realignment_token = call target_builtin;
10823 indx = 0;
10824 loop {
10825 p2 = p2 + indx * vectype_size
10826 lsq = *(floor(p2))
10827 vec_dest = realign_load (msq, lsq, realignment_token)
10828 indx = indx + 1;
10829 msq = lsq;
10830 } */
10832 /* If the misalignment remains the same throughout the execution of the
10833 loop, we can create the init_addr and permutation mask at the loop
10834 preheader. Otherwise, it needs to be created inside the loop.
10835 This can only occur when vectorizing memory accesses in the inner-loop
10836 nested within an outer-loop that is being vectorized. */
10841 {
10844 }
10846 bool diff_first_stmt_info
10853 {
10854 /* If we have different first_stmt_info, we can't set up realignment
10855 here, since we can't guarantee first_stmt_info DR has been
10856 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10857 distance from first_stmt_info DR instead as below. */
10859 {
10866 {
10869 size_one_node);
10871 }
10872 }
10873 }
10874 else
10878 offset = (offset
10882 tree bump;
10885 {
10888 }
10890 {
10895 }
10896 else
10897 {
10900 else
10905 }
10910 {
10914 else
10917 }
10921 {
10926 /* For costing some adjacent vector loads, we'd like to cost with
10927 the total number of them once instead of cost each one by one. */
10932 {
10934 {
10935 /* An IFN_LOAD_LANES will load all its vector results,
10936 regardless of which ones we actually need. Account
10937 for the cost of unused results. */
10939 {
10942 do
10943 {
10946 }
10949 {
10952 "vect_model_load_cost: %d "
10954 gaps);
10956 alignment_support_scheme,
10960 }
10961 }
10962 n_adjacent_loads++;
10964 }
10966 /* 1. Create the vector or array pointer update chain. */
10968 dataref_ptr
10972 else
10973 {
10977 }
10994 {
10998 else
11000 signed char biasval
11004 {
11007 }
11008 }
11012 {
11013 /* Emit:
11014 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
11015 VEC_MASK, LEN, BIAS). */
11021 }
11023 {
11024 /* Emit:
11025 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
11026 VEC_MASK). */
11031 final_mask);
11032 }
11033 else
11034 {
11035 /* Emit:
11036 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
11039 }
11046 /* Extract each vector into an SSA_NAME. */
11048 {
11053 else
11055 }
11058 /* Record the mapping between SSA_NAMEs and statements. */
11061 /* Record that VEC_ARRAY is now dead. */
11069 }
11072 {
11080 "vect_model_load_cost: inside_cost = %u, "
11083 }
11086 }
11089 {
11096 {
11097 /* 1. Create the vector or array pointer update chain. */
11099 {
11104 else
11105 dataref_ptr
11109 }
11111 {
11116 }
11120 {
11125 {
11129 final_mask
11140 }
11142 /* 2. Create the vector-load in the loop. */
11145 {
11147 {
11149 inside_cost
11153 }
11160 {
11162 final_len
11166 else
11167 final_len
11170 signed char biasval
11174 {
11177 }
11178 }
11182 call
11191 else
11198 }
11200 {
11201 /* The builtin decls path for gather is legacy, x86 only. */
11204 {
11206 inside_cost
11210 }
11211 poly_uint64 offset_nunits
11214 {
11215 new_stmt = vect_build_one_gather_load_call
11218 final_mask);
11220 }
11222 {
11223 /* We have a offset vector with half the number of
11224 lanes but the builtins will produce full vectype
11225 data with just the lower lanes filled. */
11226 new_stmt = vect_build_one_gather_load_call
11229 final_mask);
11235 /* now put upper half of final_mask in final_mask low. */
11237 && !SCALAR_INT_MODE_P
11239 {
11246 tree perm_mask = vect_gen_perm_mask_checked
11249 VEC_PERM_EXPR,
11250 final_mask,
11251 final_mask,
11252 perm_mask);
11257 }
11259 {
11261 VEC_UNPACK_HI_EXPR,
11262 final_mask);
11263 final_mask = make_ssa_name
11268 }
11270 new_stmt = vect_build_one_gather_load_call
11272 dataref_ptr,
11274 final_mask);
11280 /* compose low + high. */
11287 tree perm_mask
11290 VEC_PERM_EXPR,
11293 }
11295 {
11296 /* We have a offset vector with double the number of
11297 lanes. Select the low/high part accordingly. */
11300 {
11307 tree perm_mask = vect_gen_perm_mask_checked
11310 VEC_PERM_EXPR,
11311 vec_offset,
11312 vec_offset,
11313 perm_mask);
11318 }
11319 new_stmt = vect_build_one_gather_load_call
11323 }
11324 else
11326 }
11327 else
11328 {
11329 /* Emulated gather-scatter. */
11333 {
11334 /* For emulated gathers N offset vector element
11335 offset add is consumed by the load). */
11339 /* N scalar loads plus gathering them into a
11340 vector. */
11341 inside_cost
11344 inside_cost
11348 }
11349 unsigned HOST_WIDE_INT const_offset_nunits
11355 /* We support offset vectors with more elements
11356 than the data vector for now. */
11357 unsigned HOST_WIDE_INT factor
11360 unsigned elt_offset
11367 {
11370 tree idx
11375 scale);
11387 }
11392 }
11395 /* DATA_REF is null if we've already built the statement. */
11397 {
11400 }
11405 /* Store vector loads in the corresponding SLP_NODE. */
11408 }
11412 }
11419 "vect_model_load_cost: inside_cost = %u, "
11423 }
11427 /* For costing some adjacent vector loads, we'd like to cost with
11428 the total number of them once instead of cost each one by one. */
11431 {
11432 /* 1. Create the vector or array pointer update chain. */
11434 {
11435 bool simd_lane_access_p
11446 {
11449 }
11451 {
11452 dataref_ptr
11456 bump);
11457 /* Adjust the pointer by the difference to first_stmt. */
11458 data_reference_p ptrdr
11460 tree diff
11468 {
11472 alignment_support_scheme,
11475 }
11476 }
11477 else
11478 dataref_ptr
11480 at_loop,
11483 }
11485 {
11489 bump);
11490 else
11493 }
11500 {
11505 {
11519 }
11521 /* 2. Create the vector-load in the loop. */
11523 {
11526 {
11536 {
11537 align
11540 }
11541 else
11546 misalign);
11549 /* Compute IFN when LOOP_LENS or final_mask valid. */
11554 {
11555 opt_machine_mode new_ovmode
11558 unsigned factor
11563 }
11565 {
11570 }
11573 {
11575 {
11576 /* Pass VF value to 'len' argument of
11577 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11579 }
11581 {
11582 /* Pass all ones value to 'mask' argument of
11583 MASK_LEN_LOAD if final_mask is invalid. */
11586 }
11587 }
11589 {
11590 signed char biasval
11594 }
11597 {
11604 bias);
11605 else
11613 /* Need conversion if it's wrapped with VnQI. */
11615 {
11618 tree var
11622 gsi);
11626 }
11627 }
11629 {
11633 final_mask);
11637 }
11638 else
11639 {
11643 unsigned int vect_align
11645 /* Try to use a single smaller load when we are about
11646 to load excess elements compared to the unrolled
11647 scalar loop. */
11650 {
11655 /* DR will be unused. */
11658 tree_to_poly_uint64
11660 /* Aligned access to excess elements is OK if
11661 at least one element is accessed in the
11662 scalar loop. */
11663 ;
11666 * vect_get_scalar_dr_size
11668 /* Aligned access to the gap area when there's
11669 at least one element in it is OK. */
11670 ;
11671 else
11672 {
11673 /* remain should now be > 0 and < nunits. */
11677 {
11678 tree ptype;
11679 new_vtype
11681 num,
11685 }
11686 /* Else use multiple loads or a masked load? */
11687 /* For loop vectorization we now should have
11688 an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11689 set. */
11692 || LOOP_VINFO_PEELING_FOR_GAPS
11694 /* But still reduce the access size to the next
11695 required power-of-two so peeling a single
11696 scalar iteration is sufficient. */
11699 {
11700 unsigned HOST_WIDE_INT cpart_size
11705 {
11706 tree ptype;
11707 new_vtype
11709 num,
11713 }
11714 }
11715 }
11716 }
11717 tree offset
11721 ;
11724 {
11725 poly_uint64 gap_offset
11730 }
11732 {
11733 data_ref
11736 ;
11737 else
11741 }
11745 {
11751 gsi);
11754 /* We've computed 'num' above to statically two
11755 or via constant_multiple_p. */
11756 unsigned num
11759 tree_to_poly_uint64
11764 {
11769 }
11770 else
11771 {
11776 }
11781 else
11782 {
11789 vec_dest,
11791 }
11792 }
11793 }
11795 }
11797 {
11807 dr_explicit_realign,
11812 else
11814 // For explicit realign the target alignment should be
11815 // known at compile time.
11816 unsigned HOST_WIDE_INT align
11823 data_ref
11837 bump);
11843 else
11847 data_ref
11850 }
11852 {
11857 else
11859 // We should only be doing this if we know the target
11860 // alignment at compile time.
11861 unsigned HOST_WIDE_INT align
11871 }
11874 }
11876 /* One common place to cost the above vect load for different
11877 alignment support schemes. */
11879 {
11880 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11881 only need to take care of the first stmt, whose
11882 stmt_info is first_stmt_info, vec_num iterating on it
11883 will cover the cost for the remaining, it's consistent
11884 with transforming. For the prologue cost for realign,
11885 we only need to count it once for the whole group. */
11892 {
11893 /* Leave realign cases alone to keep them simple. */
11901 else
11902 n_adjacent_loads++;
11903 }
11904 }
11905 else
11906 {
11908 /* DATA_REF is null if we've already built the statement. */
11910 {
11913 }
11917 }
11919 /* 3. Handle explicit realignment if necessary/supported.
11920 Create in loop:
11921 vec_dest = realign_load (msq, lsq, realignment_token) */
11925 {
11937 {
11941 UNKNOWN_LOCATION);
11943 }
11944 }
11947 {
11951 else
11952 {
11957 }
11958 }
11960 /* Collect vector loads and later create their permutation in
11961 vect_transform_grouped_load (). */
11965 /* Store vector loads in the corresponding SLP_NODE. */
11969 /* With SLP permutation we load the gaps as well, without
11970 we need to skip the gaps after we manage to fully load
11971 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11975 && !slp_perm
11977 {
11978 poly_wide_int bump_val
11987 }
11988 }
11989 /* Bump the vector pointer to account for a gap or for excess
11990 elements loaded for a permuted SLP load. */
11994 {
11995 poly_wide_int bump_val
12002 }
12008 {
12010 /* For SLP we know we've seen all possible uses of dr_chain so
12011 direct vect_transform_slp_perm_load to DCE the unused parts.
12012 ??? This is a hack to prevent compile-time issues as seen
12013 in PR101120 and friends. */
12015 {
12020 }
12021 else
12022 {
12027 }
12028 }
12029 else
12030 {
12032 {
12034 /* We assume that the cost of a single load-lanes instruction
12035 is equivalent to the cost of DR_GROUP_SIZE separate loads.
12036 If a grouped access is instead being provided by a
12037 load-and-permute operation, include the cost of the
12038 permutes. */
12040 {
12041 /* Uses an even and odd extract operations or shuffle
12042 operations for each needed permute. */
12050 "vect_model_load_cost:"
12052 group_size);
12053 }
12055 {
12059 }
12060 }
12063 }
12065 }
12070 {
12081 "vect_model_load_cost: inside_cost = %u, "
12084 }
12087 }
12089 /* Function vect_is_simple_cond.
12091 Input:
12092 LOOP - the loop that is being vectorized.
12093 COND - Condition that is checked for simple use.
12095 Output:
12096 *COMP_VECTYPE - the vector type for the comparison.
12097 *DTS - The def types for the arguments of the comparison
12099 Returns whether a COND can be vectorized. Checks whether
12100 condition operands are supportable using vec_is_simple_use. */
12102 static bool
12106 {
12109 slp_tree slp_op;
12111 /* Mask case. */
12114 {
12117 || !*comp_vectype
12121 }
12130 {
12134 }
12138 else
12142 {
12146 }
12150 else
12159 /* Invariant comparison. */
12161 {
12165 else
12166 {
12167 /* If we can widen the comparison to match vectype do so. */
12169 && !slp_node
12172 scalar_type = build_nonstandard_integer_type
12175 slp_node);
12176 }
12177 }
12180 }
12182 /* vectorizable_condition.
12184 Check if STMT_INFO is conditional modify expression that can be vectorized.
12185 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12186 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
12187 at GSI.
12189 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12191 Return true if STMT_INFO is vectorizable in this way. */
12193 static bool
12198 {
12206 tree vec_compare;
12207 tree new_temp;
12222 tree vec_cmp_type;
12228 /* Is vectorizable conditional operation? */
12240 bool for_reduction
12243 {
12251 }
12252 else
12253 {
12256 }
12262 {
12265 }
12266 else
12267 {
12270 }
12306 {
12309 }
12311 /* For conditional reductions, the "then" value needs to be the candidate
12312 value calculated by this iteration while the "else" value needs to be
12313 the result carried over from previous iterations. If the COND_EXPR
12314 is the other way around, we need to swap it. */
12317 {
12320 else
12321 {
12326 else
12327 {
12329 /* Make sure we don't accidentally use the old condition. */
12331 }
12332 }
12333 /* ??? The vectorized operand query below doesn't allow swapping
12334 this way for SLP. */
12338 }
12341 {
12342 /* Boolean values may have another representation in vectors
12343 and therefore we prefer bit operations over comparison for
12344 them (which also works for scalar masks). We store opcodes
12345 to use in bitop1 and bitop2. Statement is vectorized as
12346 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12347 depending on bitop1 and bitop2 arity. */
12349 {
12377 }
12379 }
12384 {
12389 }
12392 {
12394 {
12396 optab optab;
12403 {
12405 optab_default);
12408 }
12409 }
12413 /* Count one reduction-like operation per vector. */
12416 && (masked
12418 cond_code)
12420 ERROR_MARK))))
12424 && (!vect_maybe_update_slp_op_vectype
12427 && !vect_maybe_update_slp_op_vectype
12431 {
12436 }
12440 {
12442 {
12448 else
12452 }
12453 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12455 {
12458 "conditional reduction prevents the use"
12461 }
12462 }
12468 }
12470 /* Transform. */
12472 /* Handle def. */
12479 /* See whether another part of the vectorized code applies a loop
12480 mask to the condition, or to its inverse. */
12485 {
12488 }
12490 {
12493 else
12494 {
12498 else
12499 {
12504 {
12508 }
12509 else
12510 {
12511 /* Try the inverse of the current mask. We check if the
12512 inverse mask is live and if so we generate a negate of
12513 the current mask such that we still honor NaNs. */
12517 {
12522 }
12523 }
12524 }
12525 }
12526 }
12528 /* Handle cond expr. */
12533 reduction_type != EXTRACT_LAST_REDUCTION
12535 else
12540 reduction_type != EXTRACT_LAST_REDUCTION
12546 /* Arguments are ready. Create the new vector stmt. */
12548 {
12558 else
12559 {
12562 {
12567 }
12568 else
12569 {
12574 vec_cond_rhs);
12575 else
12576 new_stmt
12578 vec_cond_rhs);
12584 {
12585 /* Instead of doing ~x ? y : z do x ? z : y. */
12588 }
12589 else
12590 {
12593 new_stmt
12595 else
12596 new_stmt
12601 }
12602 }
12603 }
12605 /* If we decided to apply a loop mask to the result of the vector
12606 comparison, AND the comparison with the mask now. Later passes
12607 should then be able to reuse the AND results between mulitple
12608 vector statements.
12610 For example:
12611 for (int i = 0; i < 100; ++i)
12612 x[i] = y[i] ? z[i] : 10;
12614 results in following optimized GIMPLE:
12616 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12617 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12618 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12619 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12620 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12621 vect_iftmp.11_47, { 10, ... }>;
12623 instead of using a masked and unmasked forms of
12624 vec != { 0, ... } (masked in the MASK_LOAD,
12625 unmasked in the VEC_COND_EXPR). */
12627 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12628 in cases where that's necessary. */
12632 {
12634 {
12637 vec_compare);
12640 }
12643 {
12646 BIT_NOT_EXPR,
12647 vec_compare);
12650 }
12654 {
12656 {
12659 signed char biasval
12662 }
12663 else
12664 {
12667 }
12668 }
12670 {
12671 tree loop_mask
12675 gassign *g
12677 loop_mask);
12680 }
12681 }
12685 {
12691 new_stmt = gimple_build_call_internal
12694 else
12695 new_stmt = gimple_build_call_internal
12697 vec_then_clause);
12701 {
12704 }
12707 else
12708 {
12709 /* In this case we're moving the definition to later in the
12710 block. That doesn't matter because the only uses of the
12711 lhs are in phi statements. */
12715 }
12716 }
12717 else
12718 {
12723 }
12726 else
12728 }
12739 }
12741 /* Helper of vectorizable_comparison.
12743 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12744 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12745 comparison, put it in VEC_STMT, and insert it at GSI.
12747 Return true if STMT_INFO is vectorizable in this way. */
12749 static bool
12754 {
12758 tree new_temp;
12762 poly_uint64 nunits;
12769 tree mask_type;
12783 else
12807 /* Invariant comparison. */
12809 {
12813 }
12817 /* Can't compare mask and non-mask types. */
12822 /* Boolean values may have another representation in vectors
12823 and therefore we prefer bit operations over comparison for
12824 them (which also works for scalar masks). We store opcodes
12825 to use in bitop1 and bitop2. Statement is vectorized as
12826 BITOP2 (rhs1 BITOP1 rhs2) or
12827 rhs1 BITOP2 (BITOP1 rhs2)
12828 depending on bitop1 and bitop2 arity. */
12831 {
12833 {
12836 }
12838 {
12841 }
12843 {
12847 }
12849 {
12853 }
12854 else
12855 {
12859 }
12860 }
12863 {
12865 {
12868 }
12869 else
12870 {
12872 optab optab;
12879 {
12883 }
12884 }
12886 /* Put types on constant and invariant SLP children. */
12890 {
12895 }
12901 }
12903 /* Transform. */
12905 /* Handle def. */
12916 /* Arguments are ready. Create the new vector stmt. */
12918 {
12924 else
12927 {
12931 }
12932 else
12933 {
12936 else
12938 vec_rhs2);
12941 {
12945 else
12947 new_temp);
12949 }
12950 }
12953 else
12955 }
12964 }
12966 /* vectorizable_comparison.
12968 Check if STMT_INFO is comparison expression that can be vectorized.
12969 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12970 comparison, put it in VEC_STMT, and insert it at GSI.
12972 Return true if STMT_INFO is vectorizable in this way. */
12974 static bool
12979 {
13002 }
13004 /* Check to see if the current early break given in STMT_INFO is valid for
13005 vectorization. */
13007 static bool
13011 {
13028 slp_tree slp_op0;
13029 tree op0;
13033 {
13038 }
13048 else
13056 /* Now build the new conditional. Pattern gimple_conds get dropped during
13057 codegen so we must replace the original insn. */
13060 /* When vectorizing we assume that if the branch edge is taken that we're
13061 exiting the loop. This is not however always the case as the compiler will
13062 rewrite conditions to always be a comparison against 0. To do this it
13063 sometimes flips the edges. This is fine for scalar, but for vector we
13064 then have to flip the test, as we're still assuming that if you take the
13065 branch edge that we found the exit condition. i.e. we need to know whether
13066 we are generating a `forall` or an `exist` condition. */
13077 {
13082 }
13084 /* Analyze only. */
13086 {
13088 {
13091 "can't vectorize early exit because the "
13092 "target doesn't support flag setting vector "
13095 }
13099 {
13102 "can't vectorize early exit because the "
13103 "target does not support boolean vector %s "
13106 vectype);
13108 }
13115 {
13117 OPTIMIZE_FOR_SPEED))
13119 else
13121 }
13124 }
13126 /* Tranform. */
13146 else
13147 {
13152 }
13154 /* Determine if we need to reduce the final value. */
13156 {
13157 /* We build the reductions in a way to maintain as much parallelism as
13158 possible. */
13161 /* Mask the statements as we queue them up. Normally we loop over
13162 vec_num, but since we inspect the exact results of vectorization
13163 we don't need to and instead can just use the stmts themselves. */
13166 {
13167 tree stmt_mask
13169 i);
13170 stmt_mask
13174 }
13177 {
13183 }
13184 else
13188 {
13196 }
13197 }
13198 else
13199 {
13202 {
13203 tree mask
13207 }
13211 }
13220 else
13227 }
13229 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13230 can handle all live statements in the node. Otherwise return true
13231 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13232 VEC_STMT_P is as for vectorizable_live_operation. */
13234 static bool
13239 {
13242 {
13243 stmt_vec_info slp_stmt_info;
13246 {
13249 || (loop_vinfo
13257 }
13258 }
13268 }
13270 /* Make sure the statement is vectorizable. */
13272 opt_result
13277 {
13281 gimple_seq pattern_def_seq;
13289 "not vectorized:"
13296 {
13297 gimple_stmt_iterator si;
13300 {
13301 stmt_vec_info pattern_def_stmt_info
13305 {
13306 /* Analyze def stmt of STMT if it's a pattern stmt. */
13312 opt_result res
13315 cost_vec);
13318 }
13319 }
13320 }
13322 /* Skip stmts that do not need to be vectorized. In loops this is expected
13323 to include:
13324 - the COND_EXPR which is the loop exit condition
13325 - any LABEL_EXPRs in the loop
13326 - computations that are used only for array indexing or loop control.
13327 In basic blocks we only analyze statements that are a part of some SLP
13328 instance, therefore, all the statements are relevant.
13330 Pattern statement needs to be analyzed instead of the original statement
13331 if the original statement is not relevant. Otherwise, we analyze both
13332 statements. In basic blocks we are called from some SLP instance
13333 traversal, don't analyze pattern stmts instead, the pattern stmts
13334 already will be part of SLP instance. */
13339 {
13341 && pattern_stmt_info
13344 {
13345 /* Analyze PATTERN_STMT instead of the original stmt. */
13351 }
13352 else
13353 {
13359 "not vectorized:"
13360 " irrelevant stmt as SLP node %p "
13364 }
13365 }
13368 && pattern_stmt_info
13371 {
13372 /* Analyze PATTERN_STMT too. */
13378 opt_result res
13383 }
13386 {
13415 }
13422 {
13428 }
13431 {
13436 }
13438 /* When we arrive here with a non-SLP statement and we are supposed
13439 to use SLP for everything fail vectorization. */
13448 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13449 -mveclibabi= takes preference over library functions with
13450 the simd attribute. */
13453 cost_vec)
13472 cost_vec)
13478 cost_vec));
13479 else
13480 {
13486 cost_vec)
13492 cost_vec)
13500 cost_vec)
13503 cost_vec));
13505 }
13512 "not vectorized:"
13516 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13517 need extra handling, except for vectorizable reductions. */
13525 "not vectorized:"
13530 }
13533 /* Function vect_transform_stmt.
13535 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13537 bool
13541 {
13553 {
13565 NULL);
13577 NULL);
13589 NULL);
13595 && !slp_node
13598 /* In case of interleaving, the whole chain is vectorized when the
13599 last store in the chain is reached. Store stmts before the last
13600 one are skipped, and there vec_stmt_info shouldn't be freed
13601 meanwhile. */
13602 ;
13603 else
13604 {
13609 }
13671 {
13676 }
13678 }
13684 {
13685 /* Handle stmts whose DEF is used outside the loop-nest that is
13686 being vectorized. */
13690 }
13696 }
13699 /* Remove a group of stores (for SLP or interleaving), free their
13700 stmt_vec_info. */
13702 void
13704 {
13708 {
13711 /* Free the attached stmt_vec_info and remove the stmt. */
13714 }
13715 }
13717 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13718 elements of type SCALAR_TYPE, or null if the target doesn't support
13719 such a type.
13721 If NUNITS is zero, return a vector type that contains elements of
13722 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13724 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13725 for this vectorization region and want to "autodetect" the best choice.
13726 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13727 and we want the new type to be interoperable with it. PREVAILING_MODE
13728 in this case can be a scalar integer mode or a vector mode; when it
13729 is a vector mode, the function acts like a tree-level version of
13730 related_vector_mode. */
13732 tree
13735 {
13737 scalar_mode inner_mode;
13738 machine_mode simd_mode;
13739 tree vectype;
13750 /* Interoperability between modes requires one to be a constant multiple
13751 of the other, so that the number of vectors required for each operation
13752 is a compile-time constant. */
13760 /* For vector types of elements whose mode precision doesn't
13761 match their types precision we use a element type of mode
13762 precision. The vectorization routines will have to make sure
13763 they support the proper result truncation/extension.
13764 We also make sure to build vector types with INTEGER_TYPE
13765 component type only. */
13772 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13773 When the component mode passes the above test simply use a type
13774 corresponding to that mode. The theory is that any use that
13775 would cause problems with this will disable vectorization anyway. */
13780 /* We can't build a vector type of elements with alignment bigger than
13781 their size. */
13786 /* If we felt back to using the mode fail if there was
13787 no scalar type for it. */
13791 /* If no prevailing mode was supplied, use the mode the target prefers.
13792 Otherwise lookup a vector mode based on the prevailing mode. */
13794 {
13798 {
13799 /* Traditional behavior is not to take the integer mode
13800 literally, but simply to use it as a way of determining
13801 the vector size. It is up to mode_for_vector to decide
13802 what the TYPE_MODE should be.
13804 Note that nunits == 1 is allowed in order to support single
13805 element vector types. */
13809 }
13810 }
13814 {
13815 /* Fall back to using mode_for_vector, mostly in the hope of being
13816 able to use an integer mode. */
13823 }
13827 /* In cases where the mode was chosen by mode_for_vector, check that
13828 the target actually supports the chosen mode, or that it at least
13829 allows the vector mode to be replaced by a like-sized integer. */
13834 /* Re-attach the address-space qualifier if we canonicalized the scalar
13835 type. */
13837 return build_qualified_type
13841 }
13843 /* Function get_vectype_for_scalar_type.
13845 Returns the vector type corresponding to SCALAR_TYPE as supported
13846 by the target. If GROUP_SIZE is nonzero and we're performing BB
13847 vectorization, make sure that the number of elements in the vector
13848 is no bigger than GROUP_SIZE. */
13850 tree
13853 {
13854 /* For BB vectorization, we should always have a group size once we've
13855 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13856 are tentative requests during things like early data reference
13857 analysis and pattern recognition. */
13860 else
13864 scalar_type);
13868 /* Register the natural choice of vector type, before the group size
13869 has been applied. */
13873 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13874 try again with an explicit number of elements. */
13876 && group_size
13878 {
13879 /* Start with the biggest number of units that fits within
13880 GROUP_SIZE and halve it until we find a valid vector type.
13881 Usually either the first attempt will succeed or all will
13882 fail (in the latter case because GROUP_SIZE is too small
13883 for the target), but it's possible that a target could have
13884 a hole between supported vector types.
13886 If GROUP_SIZE is not a power of 2, this has the effect of
13887 trying the largest power of 2 that fits within the group,
13888 even though the group is not a multiple of that vector size.
13889 The BB vectorizer will then try to carve up the group into
13890 smaller pieces. */
13892 do
13893 {
13897 }
13899 }
13902 }
13904 /* Return the vector type corresponding to SCALAR_TYPE as supported
13905 by the target. NODE, if nonnull, is the SLP tree node that will
13906 use the returned vector type. */
13908 tree
13910 {
13915 }
13917 /* Function get_mask_type_for_scalar_type.
13919 Returns the mask type corresponding to a result of comparison
13920 of vectors of specified SCALAR_TYPE as supported by target.
13921 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13922 make sure that the number of elements in the vector is no bigger
13923 than GROUP_SIZE. */
13925 tree
13928 {
13935 }
13937 /* Function get_mask_type_for_scalar_type.
13939 Returns the mask type corresponding to a result of comparison
13940 of vectors of specified SCALAR_TYPE as supported by target.
13941 NODE, if nonnull, is the SLP tree node that will use the returned
13942 vector type. */
13944 tree
13946 slp_tree node)
13947 {
13954 }
13956 /* Function get_same_sized_vectype
13958 Returns a vector type corresponding to SCALAR_TYPE of size
13959 VECTOR_TYPE if supported by the target. */
13961 tree
13963 {
13967 poly_uint64 nunits;
13974 }
13976 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
13977 would not change the chosen vector modes. */
13979 bool
13981 {
13988 }
13990 /* Function vect_is_simple_use.
13992 Input:
13993 VINFO - the vect info of the loop or basic block that is being vectorized.
13994 OPERAND - operand in the loop or bb.
13995 Output:
13996 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
13997 case OPERAND is an SSA_NAME that is defined in the vectorizable region
13998 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
13999 the definition could be anywhere in the function
14000 DT - the type of definition
14002 Returns whether a stmt with OPERAND can be vectorized.
14003 For loops, supportable operands are constants, loop invariants, and operands
14004 that are defined by the current iteration of the loop. Unsupportable
14005 operands are those that are defined by a previous iteration of the loop (as
14006 is the case in reduction/induction computations).
14007 For basic blocks, supportable operands are constants and bb invariants.
14008 For now, operands defined outside the basic block are not supported. */
14010 bool
14013 {
14021 {
14027 else
14029 }
14039 else
14040 {
14045 else
14046 {
14052 }
14055 }
14058 {
14061 {
14095 }
14096 }
14099 {
14104 }
14107 }
14109 /* Function vect_is_simple_use.
14111 Same as vect_is_simple_use but also determines the vector operand
14112 type of OPERAND and stores it to *VECTYPE. If the definition of
14113 OPERAND is vect_uninitialized_def, vect_constant_def or
14114 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
14115 is responsible to compute the best suited vector type for the
14116 scalar operand. */
14118 bool
14122 {
14123 stmt_vec_info def_stmt_info;
14133 /* Now get a vector type if the def is internal, otherwise supply
14134 NULL_TREE and leave it up to the caller to figure out a proper
14135 type for the use stmt. */
14142 {
14148 }
14153 else
14157 }
14159 /* Function vect_is_simple_use.
14161 Same as vect_is_simple_use but determines the operand by operand
14162 position OPERAND from either STMT or SLP_NODE, filling in *OP
14163 and *SLP_DEF (when SLP_NODE is not NULL). */
14165 bool
14170 {
14172 {
14177 {
14178 /* ??? VEC_PERM nodes might be intermediate and their lane value
14179 have no representative (nor do we build a VEC_PERM stmt for
14180 the actual operation). Note for two-operator nodes we set
14181 a representative but leave scalar stmts empty as we'd only
14182 have one for a subset of lanes. Ideally no caller would
14183 require *op for internal defs. */
14185 {
14188 }
14189 else
14190 {
14197 }
14198 }
14199 else
14200 {
14206 }
14207 }
14208 else
14209 {
14212 {
14215 {
14218 else
14220 }
14223 else
14225 }
14230 else
14233 }
14234 }
14236 /* If OP is not NULL and is external or constant update its vector
14237 type with VECTYPE. Returns true if successful or false if not,
14238 for example when conflicting vector types are present. */
14240 bool
14242 {
14247 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
14248 should be handled by patters. Allow vect_constant_def for now. */
14254 }
14256 /* Function supportable_widening_operation
14258 Check whether an operation represented by the code CODE is a
14259 widening operation that is supported by the target platform in
14260 vector form (i.e., when operating on arguments of type VECTYPE_IN
14261 producing a result of type VECTYPE_OUT).
14263 Widening operations we currently support are NOP (CONVERT), FLOAT,
14264 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
14265 are supported by the target platform either directly (via vector
14266 tree-codes), or via target builtins.
14268 Output:
14269 - CODE1 and CODE2 are codes of vector operations to be used when
14270 vectorizing the operation, if available.
14271 - MULTI_STEP_CVT determines the number of required intermediate steps in
14272 case of multi-step conversion (like char->short->int - in that case
14273 MULTI_STEP_CVT will be 1).
14274 - INTERM_TYPES contains the intermediate type required to perform the
14275 widening operation (short in the above example). */
14277 bool
14279 code_helper code,
14280 stmt_vec_info stmt_info,
14286 {
14289 machine_mode vec_mode;
14305 {
14307 /* Don't set c1 and c2 if code is not a tree_code. */
14311 /* The result of a vectorized widening operation usually requires
14312 two vectors (because the widened results do not fit into one vector).
14313 The generated vector results would normally be expected to be
14314 generated in the same order as in the original scalar computation,
14315 i.e. if 8 results are generated in each vector iteration, they are
14316 to be organized as follows:
14317 vect1: [res1,res2,res3,res4],
14318 vect2: [res5,res6,res7,res8].
14320 However, in the special case that the result of the widening
14321 operation is used in a reduction computation only, the order doesn't
14322 matter (because when vectorizing a reduction we change the order of
14323 the computation). Some targets can take advantage of this and
14324 generate more efficient code. For example, targets like Altivec,
14325 that support widen_mult using a sequence of {mult_even,mult_odd}
14326 generate the following vectors:
14327 vect1: [res1,res3,res5,res7],
14328 vect2: [res2,res4,res6,res8].
14330 When vectorizing outer-loops, we execute the inner-loop sequentially
14331 (each vectorized inner-loop iteration contributes to VF outer-loop
14332 iterations in parallel). We therefore don't allow to change the
14333 order of the computation in the inner-loop during outer-loop
14334 vectorization. */
14335 /* TODO: Another case in which order doesn't *really* matter is when we
14336 widen and then contract again, e.g. (short)((int)x * y >> 8).
14337 Normally, pack_trunc performs an even/odd permute, whereas the
14338 repack from an even/odd expansion would be an interleave, which
14339 would be significantly simpler for e.g. AVX2. */
14340 /* In any case, in order to avoid duplicating the code below, recurse
14341 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
14342 are properly set up for the caller. If we fail, we'll continue with
14343 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
14350 interm_types))
14351 {
14352 /* Elements in a vector with vect_used_by_reduction property cannot
14353 be reordered if the use chain with this property does not have the
14354 same operation. One such an example is s += a * b, where elements
14355 in a and b cannot be reordered. Here we check if the vector defined
14356 by STMT is only directly used in the reduction statement. */
14361 }
14377 /* Support the recursion induced just above. */
14387 CASE_CONVERT:
14404 }
14410 {
14411 /* The signedness is determined from output operand. */
14414 }
14420 {
14421 /* If the input and result modes are the same, a different optab
14422 is needed where we pass in the number of units in vectype. */
14425 }
14429 {
14430 /* If this is an internal fn then we must check whether the target
14431 supports either a low-high split or an even-odd split. */
14441 /* If we don't support low-high, then check for even-odd. */
14444 || !optab2
14446 {
14452 }
14453 }
14455 {
14457 {
14458 /* The signedness is determined from output operand. */
14461 }
14467 {
14468 /* If the input and result modes are the same, a different optab
14469 is needed where we pass in the number of units in vectype. */
14472 }
14473 else
14474 {
14477 }
14480 }
14492 {
14495 /* For scalar masks we may have different boolean
14496 vector types having the same QImode. Thus we
14497 add additional check for elements number. */
14501 }
14503 /* Check if it's a multi-step conversion that can be done using intermediate
14504 types. */
14512 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14513 intermediate steps in promotion sequence. We try
14514 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14515 not. */
14518 {
14521 intermediate_type
14524 {
14525 tree intermediate_element_type
14528 intermediate_type
14530 intermediate_mode);
14531 }
14532 else
14533 intermediate_type
14541 {
14542 /* If the input and result modes are the same, a different optab
14543 is needed where we pass in the number of units in vectype. */
14546 }
14547 else
14548 {
14551 }
14569 {
14575 }
14579 }
14583 }
14586 /* Function supportable_narrowing_operation
14588 Check whether an operation represented by the code CODE is a
14589 narrowing operation that is supported by the target platform in
14590 vector form (i.e., when operating on arguments of type VECTYPE_IN
14591 and producing a result of type VECTYPE_OUT).
14593 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14594 and FLOAT. This function checks if these operations are supported by
14595 the target platform directly via vector tree-codes.
14597 Output:
14598 - CODE1 is the code of a vector operation to be used when
14599 vectorizing the operation, if available.
14600 - MULTI_STEP_CVT determines the number of required intermediate steps in
14601 case of multi-step conversion (like int->short->char - in that case
14602 MULTI_STEP_CVT will be 1).
14603 - INTERM_TYPES contains the intermediate type required to perform the
14604 narrowing operation (short in the above example). */
14606 bool
14611 {
14612 machine_mode vec_mode;
14629 {
14630 CASE_CONVERT:
14638 else
14644 /* The signedness is determined from output operand. */
14655 }
14667 {
14670 /* For scalar masks we may have different boolean
14671 vector types having the same QImode. Thus we
14672 add additional check for elements number. */
14676 }
14681 /* Check if it's a multi-step conversion that can be done using intermediate
14682 types. */
14687 else
14690 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14691 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14692 costly than signed. */
14694 {
14697 intermediate_type
14699 interm_optab
14705 {
14709 }
14710 }
14712 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14713 intermediate steps in promotion sequence. We try
14714 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14717 {
14720 intermediate_type
14722 else
14723 intermediate_type
14731 else
14732 interm_optab
14734 optab_default);
14746 {
14752 }
14757 }
14761 }
14763 /* Function supportable_indirect_convert_operation
14765 Check whether an operation represented by the code CODE is single or multi
14766 operations that are supported by the target platform in
14767 vector form (i.e., when operating on arguments of type VECTYPE_IN
14768 producing a result of type VECTYPE_OUT).
14770 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14771 This function checks if these operations are supported
14772 by the target platform directly (via vector tree-codes).
14774 Output:
14775 - converts contains some pairs to perform the convert operation,
14776 the pair's first is the intermediate type, and its second is the code of
14777 a vector operation to be used when converting the operation from the
14778 previous type to the intermediate type. */
14779 bool
14781 tree vectype_out,
14782 tree vectype_in,
14784 tree op0)
14785 {
14789 opt_scalar_mode mode_iter;
14796 vectype_out,
14797 vectype_in,
14799 {
14802 }
14804 /* For conversions between float and integer types try whether
14805 we can use intermediate signed integer types to support the
14806 conversion. */
14810 {
14814 scalar_mode intermediate_mode;
14816 {
14819 }
14820 else
14821 {
14826 }
14829 opt_scalar_mode mode_iter;
14831 {
14837 scalar_mode cvt_mode;
14842 cvt_type = build_nonstandard_integer_type
14845 /* Check if the intermediate type can hold OP0's range.
14846 When converting from float to integer this is not necessary
14847 because values that do not fit the (smaller) target type are
14848 unspecified anyway. */
14850 {
14852 /* For vector form, it looks like op0 doesn't have RANGE_INFO.
14853 In the future, if it is supported, changes may need to be made
14854 to this part, such as checking the RANGE of each element
14855 in the vector. */
14868 }
14871 cvt_type,
14872 nelts);
14873 /* This should only happened for SLP as long as loop vectorizer
14874 only supports same-sized vector. */
14878 vectype_out,
14881 cvt_type,
14887 }
14890 {
14895 }
14896 }
14898 }
14900 /* Generate and return a vector mask of MASK_TYPE such that
14901 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14902 Add the statements to SEQ. */
14904 tree
14907 {
14911 OPTIMIZE_FOR_SPEED));
14915 tree tmp;
14918 else
14923 }
14925 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14926 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14928 tree
14930 tree end_index)
14931 {
14934 }
14936 /* Try to compute the vector types required to vectorize STMT_INFO,
14937 returning true on success and false if vectorization isn't possible.
14938 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14939 take sure that the number of elements in the vectors is no bigger
14940 than GROUP_SIZE.
14942 On success:
14944 - Set *STMT_VECTYPE_OUT to:
14945 - NULL_TREE if the statement doesn't need to be vectorized;
14946 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14948 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14949 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14950 statement does not help to determine the overall number of units. */
14952 opt_result
14957 {
14960 /* For BB vectorization, we should always have a group size once we've
14961 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
14962 are tentative requests during things like early data reference
14963 analysis and pattern recognition. */
14966 else
14973 /* Allow vector conditionals through here. */
14975 /* MASK_STORE and friends have no lhs, but are ok. */
14979 {
14981 {
14982 /* Ignore calls with no lhs. These must be calls to
14983 #pragma omp simd functions, and what vectorization factor
14984 it really needs can't be determined until
14985 vectorizable_simd_clone_call. */
14990 }
14994 }
14996 tree vectype;
14999 {
15004 }
15006 {
15015 }
15016 else
15017 {
15018 /* If we got here with a gcond it means that the target had no available vector
15019 mode for the scalar type. We can't vectorize so abort. */
15022 "not vectorized:"
15024 scalar_type);
15028 else
15032 {
15035 "get vectype for scalar type (group size %d):"
15037 else
15040 }
15044 "not vectorized:"
15046 scalar_type);
15050 }
15055 stmt);
15059 /* Don't try to compute scalar types if the stmt produces a boolean
15060 vector; use the existing vector type instead. */
15063 {
15064 /* The number of units is set according to the smallest scalar
15065 type (or the largest vector size, but we only support one
15066 vector size per vectorization). */
15070 {
15074 scalar_type);
15076 group_size);
15080 scalar_type);
15083 nunits_vectype);
15084 }
15085 }
15090 "Not vectorized: Incompatible number "
15095 {
15099 }
15103 }
15105 /* Generate and return statement sequence that sets vector length LEN that is:
15107 min_of_start_and_end = min (START_INDEX, END_INDEX);
15108 left_len = END_INDEX - min_of_start_and_end;
15109 rhs = min (left_len, LEN_LIMIT);
15110 LEN = rhs;
15112 Note: the cost of the code generated by this function is modeled
15113 by vect_estimate_min_profitable_iters, so changes here may need
15114 corresponding changes there. */
15116 gimple_seq
15118 {
15130 }