| blob | 9b14b96cb5a6ae0aa38ae3bc4ed68efce936d650 |
1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
23 #define INCLUDE_MEMORY
62 /* For lang_hooks.types.type_for_mode. */
65 /* Return the vectorized type for the given statement. */
67 tree
69 {
71 }
73 /* Return TRUE iff the given statement is in an inner loop relative to
74 the loop being vectorized. */
75 bool
77 {
89 }
91 /* Record the cost of a statement, either by directly informing the
92 target model or by saving it in a vector for later processing.
93 Return a preliminary estimate of the statement's cost. */
95 static unsigned
101 {
109 stmt_info_for_cost si
115 }
117 unsigned
122 {
125 }
127 unsigned
132 {
135 }
137 unsigned
141 {
146 }
148 /* Return a variable of type ELEM_TYPE[NELEMS]. */
150 static tree
152 {
155 }
157 /* ARRAY is an array of vectors created by create_vector_array.
158 Return an SSA_NAME for the vector in index N. The reference
159 is part of the vectorization of STMT_INFO and the vector is associated
160 with scalar destination SCALAR_DEST. */
162 static tree
166 {
183 }
185 /* ARRAY is an array of vectors created by create_vector_array.
186 Emit code to store SSA_NAME VECT in index N of the array.
187 The store is part of the vectorization of STMT_INFO. */
189 static void
193 {
194 tree array_ref;
203 }
205 /* PTR is a pointer to an array of type TYPE. Return a representation
206 of *PTR. The memory reference replaces those in FIRST_DR
207 (and its group). */
209 static tree
211 {
212 tree mem_ref;
215 /* Arrays have the same alignment as their type. */
218 }
220 /* Add a clobber of variable VAR to the vectorization of STMT_INFO.
221 Emit the clobber before *GSI. */
223 static void
226 {
230 }
232 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
234 /* Function vect_mark_relevant.
236 Mark STMT_INFO as "relevant for vectorization" and add it to WORKLIST. */
238 static void
241 {
250 /* If this stmt is an original stmt in a pattern, we might need to mark its
251 related pattern stmt instead of the original stmt. However, such stmts
252 may have their own uses that are not in any pattern, in such cases the
253 stmt itself should be marked. */
255 {
256 /* This is the last stmt in a sequence that was detected as a
257 pattern that can potentially be vectorized. Don't mark the stmt
258 as relevant/live because it's not going to be vectorized.
259 Instead mark the pattern-stmt that replaces it. */
263 "last stmt in pattern. don't mark"
273 {
276 "vec_stmt_relevant_p: forcing live pattern stmt "
279 }
285 }
293 {
298 }
301 }
304 /* Function is_simple_and_all_uses_invariant
306 Return true if STMT_INFO is simple and all uses of it are invariant. */
308 bool
310 loop_vec_info loop_vinfo)
311 {
312 tree op;
313 ssa_op_iter iter;
320 {
324 {
329 }
333 }
335 }
337 /* Function vect_stmt_relevant_p.
339 Return true if STMT_INFO, in the loop that is represented by LOOP_VINFO,
340 is "relevant for vectorization".
342 A stmt is considered "relevant for vectorization" if:
343 - it has uses outside the loop.
344 - it has vdefs (it alters memory).
345 - control stmts in the loop (except for the exit condition).
346 - it is an induction and we have multiple exits.
348 CHECKME: what other side effects would the vectorizer allow? */
350 static bool
353 {
355 ssa_op_iter op_iter;
356 imm_use_iterator imm_iter;
357 use_operand_p use_p;
358 def_operand_p def_p;
363 /* cond stmt other than loop exit cond. */
370 /* changing memory. */
374 {
379 }
381 /* uses outside the loop. */
383 {
385 {
388 {
396 /* We expect all such uses to be in the loop exit phis
397 (because of loop closed form) */
401 }
402 }
403 }
405 /* Check if it's an induction and multiple exits. In this case there will be
406 a usage later on after peeling which is needed for the alternate exit. */
409 {
412 "vec_stmt_relevant_p: induction forced for "
417 }
421 {
426 }
429 }
432 /* Function exist_non_indexing_operands_for_use_p
434 USE is one of the uses attached to STMT_INFO. Check if USE is
435 used in STMT_INFO for anything other than indexing an array. */
437 static bool
439 {
440 tree operand;
442 /* USE corresponds to some operand in STMT. If there is no data
443 reference in STMT, then any operand that corresponds to USE
444 is not indexing an array. */
448 /* STMT has a data_ref. FORNOW this means that its of one of
449 the following forms:
450 -1- ARRAY_REF = var
451 -2- var = ARRAY_REF
452 (This should have been verified in analyze_data_refs).
454 'var' in the second case corresponds to a def, not a use,
455 so USE cannot correspond to any operands that are not used
456 for array indexing.
458 Therefore, all we need to check is if STMT falls into the
459 first case, and whether var corresponds to USE. */
463 {
466 {
479 }
481 }
493 }
496 /*
497 Function process_use.
499 Inputs:
500 - a USE in STMT_VINFO in a loop represented by LOOP_VINFO
501 - RELEVANT - enum value to be set in the STMT_VINFO of the stmt
502 that defined USE. This is done by calling mark_relevant and passing it
503 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
504 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
505 be performed.
507 Outputs:
508 Generally, LIVE_P and RELEVANT are used to define the liveness and
509 relevance info of the DEF_STMT of this USE:
510 STMT_VINFO_LIVE_P (DEF_stmt_vinfo) <-- live_p
511 STMT_VINFO_RELEVANT (DEF_stmt_vinfo) <-- relevant
512 Exceptions:
513 - case 1: If USE is used only for address computations (e.g. array indexing),
514 which does not need to be directly vectorized, then the liveness/relevance
515 of the respective DEF_STMT is left unchanged.
516 - case 2: If STMT_VINFO is a reduction phi and DEF_STMT is a reduction stmt,
517 we skip DEF_STMT cause it had already been processed.
518 - case 3: If DEF_STMT and STMT_VINFO are in different nests, then
519 "relevant" will be modified accordingly.
521 Return true if everything is as expected. Return false otherwise. */
523 static opt_result
527 {
528 stmt_vec_info dstmt_vinfo;
531 /* case 1: we are only interested in uses that need to be vectorized. Uses
532 that are used for address computation are not considered relevant. */
538 "not vectorized:"
547 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DSTMT_VINFO).
548 We have to force the stmt live since the epilogue loop needs it to
549 continue computing the reduction. */
555 {
561 }
563 /* case 3a: outer-loop stmt defining an inner-loop stmt:
564 outer-loop-header-bb:
565 d = dstmt_vinfo
566 inner-loop:
567 stmt # use (d)
568 outer-loop-tail-bb:
569 ... */
571 {
577 {
598 }
599 }
601 /* case 3b: inner-loop stmt defining an outer-loop stmt:
602 outer-loop-header-bb:
603 ...
604 inner-loop:
605 d = dstmt_vinfo
606 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
607 stmt # use (d) */
609 {
615 {
633 }
634 }
635 /* We are also not interested in uses on loop PHI backedges that are
636 inductions. Otherwise we'll needlessly vectorize the IV increment
637 and cause hybrid SLP for SLP inductions. Unless the PHI is live
638 of course. */
645 {
650 }
655 }
658 /* Function vect_mark_stmts_to_be_vectorized.
660 Not all stmts in the loop need to be vectorized. For example:
662 for i...
663 for j...
664 1. T0 = i + j
665 2. T1 = a[T0]
667 3. j = j + 1
669 Stmt 1 and 3 do not need to be vectorized, because loop control and
670 addressing of vectorized data-refs are handled differently.
672 This pass detects such stmts. */
674 opt_result
676 {
680 gimple_stmt_iterator si;
682 basic_block bb;
690 /* 1. Init worklist. */
692 {
695 {
703 }
705 {
715 }
716 }
718 /* 2. Process_worklist */
720 {
721 use_operand_p use_p;
722 ssa_op_iter iter;
729 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
730 (DEF_STMT) as relevant/irrelevant according to the relevance property
731 of STMT. */
734 /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is
735 propagated as is to the DEF_STMTs of its USEs.
737 One exception is when STMT has been identified as defining a reduction
738 variable; in this case we set the relevance to vect_used_by_reduction.
739 This is because we distinguish between two kinds of relevant stmts -
740 those that are used by a reduction computation, and those that are
741 (also) used by a regular computation. This allows us later on to
742 identify stmts that are used solely by a reduction, and therefore the
743 order of the results that they produce does not have to be kept. */
746 {
775 }
778 {
779 /* Pattern statements are not inserted into the code, so
780 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
781 have to scan the RHS or function arguments instead. */
783 {
789 {
790 opt_result res
800 }
802 {
805 {
806 opt_result res
811 }
812 }
813 }
815 {
818 opt_result res
827 }
829 {
831 {
833 opt_result res
838 }
839 }
840 else
842 }
843 else
845 {
847 opt_result res
852 }
855 {
856 gather_scatter_info gs_info;
859 opt_result res
863 {
867 }
868 }
872 }
874 /* Function vect_model_simple_cost.
876 Models cost for simple operations, i.e. those that only emit ncopies of a
877 single op. Right now, this does not account for multiple insns that could
878 be generated for the single vector op. We will handle that shortly. */
880 static void
885 slp_tree node,
888 {
893 /* ??? Somehow we need to fix this at the callers. */
898 /* Cost the "broadcast" of a scalar operand in to a vector operand.
899 Use scalar_to_vec to cost the broadcast, as elsewhere in the vector
900 cost model. */
906 /* Pass the inside-of-loop statements to the target-specific cost model. */
912 "vect_model_simple_cost: inside_cost = %d, "
914 }
917 /* Model cost for type demotion and promotion operations. PWR is
918 normally zero for single-step promotions and demotions. It will be
919 one if two-step promotion/demotion is required, and so on. NCOPIES
920 is the number of vector results (and thus number of instructions)
921 for the narrowest end of the operation chain. Each additional
922 step doubles the number of instructions required. If WIDEN_ARITH
923 is true the stmt is doing widening arithmetic. */
925 static void
931 {
936 {
938 widen_arith
942 }
944 /* FORNOW: Assuming maximum 2 args per stmts. */
952 "vect_model_promotion_demotion_cost: inside_cost = %d, "
954 }
956 /* Returns true if the current function returns DECL. */
958 static bool
960 {
961 edge_iterator ei;
962 edge e;
964 {
970 /* We often end up with an aggregate copy to the result decl,
971 handle that case as well. First skip intermediate clobbers
972 though. */
974 do
975 {
977 }
983 }
985 }
987 /* Calculate cost of DR's memory access. */
988 void
990 dr_alignment_support alignment_support_scheme,
994 {
996 {
998 {
1001 vect_body);
1007 }
1010 {
1011 /* Here, we assign an additional cost for the unaligned store. */
1017 "vect_model_store_cost: unaligned supported by "
1020 }
1023 {
1030 }
1034 }
1035 }
1037 /* Calculate cost of DR's memory access. */
1038 void
1040 dr_alignment_support alignment_support_scheme,
1047 {
1049 {
1051 {
1060 }
1062 {
1063 /* Here, we assign an additional cost for the unaligned load. */
1070 "vect_model_load_cost: unaligned supported by "
1074 }
1076 {
1082 /* FIXME: If the misalignment remains fixed across the iterations of
1083 the containing loop, the following cost should be added to the
1084 prologue costs. */
1094 }
1096 {
1099 "vect_model_load_cost: unaligned software "
1102 /* Unaligned software pipeline has a load of an address, an initial
1103 load, and possibly a mask operation to "prime" the loop. However,
1104 if this is an access in a group of loads, which provide grouped
1105 access, then the above cost should only be considered for one
1106 access in the group. Inside the loop, there is a load op
1107 and a realignment op. */
1110 {
1118 }
1127 "vect_model_load_cost: explicit realign optimized"
1131 }
1134 {
1141 }
1145 }
1146 }
1148 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1149 the loop preheader for the vectorized stmt STMT_VINFO. */
1151 static void
1154 {
1157 else
1163 }
1165 /* Function vect_init_vector.
1167 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1168 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1169 vector type a vector with all elements equal to VAL is created first.
1170 Place the initialization at GSI if it is not NULL. Otherwise, place the
1171 initialization at the loop preheader.
1172 Return the DEF of INIT_STMT.
1173 It will be used in the vectorization of STMT_INFO. */
1175 tree
1178 {
1180 tree new_temp;
1182 /* We abuse this function to push sth to a SSA name with initial 'val'. */
1184 {
1187 {
1188 /* Scalar boolean value should be transformed into
1189 all zeros or all ones value before building a vector. */
1191 {
1197 else
1198 {
1204 }
1205 }
1206 else
1207 {
1212 else
1213 /* ??? Condition vectorization expects us to do
1214 promotion of invariant/external defs. */
1218 {
1222 }
1223 }
1224 }
1226 }
1232 }
1235 /* Function vect_get_vec_defs_for_operand.
1237 OP is an operand in STMT_VINFO. This function returns a vector of
1238 NCOPIES defs that will be used in the vectorized stmts for STMT_VINFO.
1240 In the case that OP is an SSA_NAME which is defined in the loop, then
1241 STMT_VINFO_VEC_STMTS of the defining stmt holds the relevant defs.
1243 In case OP is an invariant or constant, a new stmt that creates a vector def
1244 needs to be introduced. VECTYPE may be used to specify a required type for
1245 vector invariant. */
1247 void
1251 {
1261 stmt_vec_info def_stmt_info;
1270 {
1272 tree vector_type;
1279 else
1286 }
1287 else
1288 {
1294 }
1295 }
1298 /* Get vectorized definitions for OP0 and OP1. */
1300 void
1307 {
1309 {
1318 }
1319 else
1320 {
1333 }
1334 }
1336 void
1343 {
1349 }
1351 /* Helper function called by vect_finish_replace_stmt and
1352 vect_finish_stmt_generation. Set the location of the new
1353 statement and create and return a stmt_vec_info for it. */
1355 static void
1358 {
1363 {
1366 /* While EH edges will generally prevent vectorization, stmt might
1367 e.g. be in a must-not-throw region. Ensure newly created stmts
1368 that could throw are part of the same region. */
1372 }
1373 else
1375 }
1377 /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT,
1378 which sets the same scalar result as STMT_INFO did. Create and return a
1379 stmt_vec_info for VEC_STMT. */
1381 void
1384 {
1392 }
1394 /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it
1395 before *GSI. Create and return a stmt_vec_info for VEC_STMT. */
1397 void
1401 {
1406 {
1410 {
1414 /* If we have an SSA vuse and insert a store, update virtual
1415 SSA form to avoid triggering the renamer. Do so only
1416 if we can easily see all uses - which is what almost always
1417 happens with the way vectorized stmts are inserted. */
1426 {
1430 }
1431 }
1432 }
1435 }
1437 /* We want to vectorize a call to combined function CFN with function
1438 decl FNDECL, using VECTYPE_OUT as the type of the output and VECTYPE_IN
1439 as the types of all inputs. Check whether this is possible using
1440 an internal function, returning its code if so or IFN_LAST if not. */
1442 static internal_fn
1445 {
1446 internal_fn ifn;
1449 else
1452 {
1455 {
1460 /* The type size of both the vectype_in and vectype_out should be
1461 exactly the same when vectype_out isn't participating the optab.
1462 While there is no restriction for type size when vectype_out
1463 is part of the optab query. */
1468 OPTIMIZE_FOR_SPEED))
1470 }
1471 }
1473 }
1477 gimple_stmt_iterator *);
1479 /* Check whether a load or store statement in the loop described by
1480 LOOP_VINFO is possible in a loop using partial vectors. This is
1481 testing whether the vectorizer pass has the appropriate support,
1482 as well as whether the target does.
1484 VLS_TYPE says whether the statement is a load or store and VECTYPE
1485 is the type of the vector being loaded or stored. SLP_NODE is the SLP
1486 node that contains the statement, or null if none. MEMORY_ACCESS_TYPE
1487 says how the load or store is going to be implemented and GROUP_SIZE
1488 is the number of load or store statements in the containing group.
1489 If the access is a gather load or scatter store, GS_INFO describes
1490 its arguments. If the load or store is conditional, SCALAR_MASK is the
1491 condition under which it occurs.
1493 Clear LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P if a loop using partial
1494 vectors is not supported, otherwise record the required rgroup control
1495 types. */
1497 static void
1499 slp_tree slp_node,
1500 vec_load_store_type vls_type,
1502 vect_memory_access_type
1503 memory_access_type,
1505 tree scalar_mask)
1506 {
1507 /* Invariant loads need no special support. */
1517 {
1520 internal_fn ifn
1527 scalar_mask);
1528 else
1529 {
1532 "can't operate on partial vectors because"
1533 " the target doesn't have an appropriate"
1536 }
1538 }
1541 {
1542 internal_fn ifn = (is_load
1543 ? IFN_MASK_GATHER_LOAD
1545 internal_fn len_ifn = (is_load
1546 ? IFN_MASK_LEN_GATHER_LOAD
1558 scalar_mask);
1559 else
1560 {
1563 "can't operate on partial vectors because"
1564 " the target doesn't have an appropriate"
1567 }
1569 }
1573 {
1574 /* Element X of the data must come from iteration i * VF + X of the
1575 scalar loop. We need more work to support other mappings. */
1578 "can't operate on partial vectors because an"
1582 }
1585 {
1588 "can't operate on partial vectors when emulating"
1592 }
1594 /* We might load more scalars than we need for permuting SLP loads.
1595 We checked in get_group_load_store_type that the extra elements
1596 don't leak into a new vector. */
1598 {
1603 };
1607 machine_mode mask_mode;
1608 machine_mode vmode;
1611 {
1616 }
1619 {
1623 }
1626 {
1629 "can't operate on partial vectors because the"
1630 " target doesn't have the appropriate partial"
1633 }
1634 }
1636 /* Return the mask input to a masked load or store. VEC_MASK is the vectorized
1637 form of the scalar mask condition and LOOP_MASK, if nonnull, is the mask
1638 that needs to be applied to all loads and stores in a vectorized loop.
1639 Return VEC_MASK if LOOP_MASK is null or if VEC_MASK is already masked,
1640 otherwise return VEC_MASK & LOOP_MASK.
1642 MASK_TYPE is the type of both masks. If new statements are needed,
1643 insert them before GSI. */
1645 tree
1648 {
1664 }
1666 /* Determine whether we can use a gather load or scatter store to vectorize
1667 strided load or store STMT_INFO by truncating the current offset to a
1668 smaller width. We need to be able to construct an offset vector:
1670 { 0, X, X*2, X*3, ... }
1672 without loss of precision, where X is STMT_INFO's DR_STEP.
1674 Return true if this is possible, describing the gather load or scatter
1675 store in GS_INFO. MASKED_P is true if the load or store is conditional. */
1677 static bool
1681 {
1686 {
1687 /* ??? Perhaps we could use range information here? */
1692 }
1694 /* Get the number of bits in an element. */
1699 /* Set COUNT to the upper limit on the number of elements - 1.
1700 Start with the maximum vectorization factor. */
1703 /* Try lowering COUNT to the number of scalar latch iterations. */
1705 widest_int max_iters;
1710 /* Try scales of 1 and the element size. */
1714 {
1716 widest_int factor;
1720 /* Determine the minimum precision of (COUNT - 1) * STEP / SCALE. */
1727 /* Find the narrowest viable offset type. */
1732 /* See whether the target supports the operation with an offset
1733 no narrower than OFFSET_TYPE. */
1742 /* Logically the sum of DR_BASE_ADDRESS, DR_INIT and DR_OFFSET,
1743 but we don't need to store that here. */
1751 }
1755 "truncating gather/scatter offset to %d bits"
1759 }
1761 /* Return true if we can use gather/scatter internal functions to
1762 vectorize STMT_INFO, which is a grouped or strided load or store.
1763 MASKED_P is true if load or store is conditional. When returning
1764 true, fill in GS_INFO with the information required to perform the
1765 operation. */
1767 static bool
1771 {
1786 "using gather/scatter for strided/grouped access,"
1790 }
1792 /* STMT_INFO is a non-strided load or store, meaning that it accesses
1793 elements with a known constant step. Return -1 if that step
1794 is negative, 0 if it is zero, and 1 if it is greater than zero. */
1796 int
1798 {
1801 size_zero_node);
1802 }
1804 /* If the target supports a permute mask that reverses the elements in
1805 a vector of type VECTYPE, return that mask, otherwise return null. */
1807 tree
1809 {
1812 /* The encoding has a single stepped pattern. */
1819 indices))
1822 }
1824 /* A subroutine of get_load_store_type, with a subset of the same
1825 arguments. Handle the case where STMT_INFO is a load or store that
1826 accesses consecutive elements with a negative step. Sets *POFFSET
1827 to the offset to be applied to the DR for the first access. */
1829 static vect_memory_access_type
1832 vec_load_store_type vls_type,
1834 {
1836 dr_alignment_support alignment_support_scheme;
1839 {
1844 }
1846 /* For backward running DRs the first access in vectype actually is
1847 N-1 elements before the address of the DR. */
1852 alignment_support_scheme
1856 {
1862 }
1865 {
1868 "negative step with invariant source;"
1871 }
1874 {
1880 }
1883 }
1885 /* STMT_INFO is either a masked or unconditional store. Return the value
1886 being stored. */
1888 tree
1890 {
1892 {
1895 }
1897 {
1902 }
1904 }
1906 /* Function VECTOR_VECTOR_COMPOSITION_TYPE
1908 This function returns a vector type which can be composed with NETLS pieces,
1909 whose type is recorded in PTYPE. VTYPE should be a vector type, and has the
1910 same vector size as the return vector. It checks target whether supports
1911 pieces-size vector mode for construction firstly, if target fails to, check
1912 pieces-size scalar mode for construction further. It returns NULL_TREE if
1913 fails to find the available composition.
1915 For example, for (vtype=V16QI, nelts=4), we can probably get:
1916 - V16QI with PTYPE V4QI.
1917 - V4SI with PTYPE SI.
1918 - NULL_TREE. */
1920 static tree
1922 {
1930 /* When we are asked to compose the vector from its components let
1931 that happen directly. */
1933 {
1936 }
1941 {
1942 /* First check if vec_init optab supports construction from
1943 vector pieces directly. */
1946 machine_mode rmode;
1950 {
1953 }
1955 /* Otherwise check if exists an integer type of the same piece size and
1956 if vec_init optab supports construction from it directly. */
1961 {
1964 }
1965 }
1968 }
1970 /* A subroutine of get_load_store_type, with a subset of the same
1971 arguments. Handle the case where STMT_INFO is part of a grouped load
1972 or store.
1974 For stores, the statements in the group are all consecutive
1975 and there is no gap at the end. For loads, the statements in the
1976 group might not be consecutive; there can be gaps between statements
1977 as well as at the end. */
1979 static bool
1989 {
1992 stmt_vec_info first_stmt_info;
1997 {
2003 }
2004 else
2005 {
2010 }
2014 /* True if the vectorized statements would access beyond the last
2015 statement in the group. */
2018 /* True if we can cope with such overrun by peeling for gaps, so that
2019 there is at least one final scalar iteration after the vector loop. */
2022 && loop_vinfo
2025 /* There can only be a gap at the end of the group if the stride is
2026 known at compile time. */
2029 /* Stores can't yet have gaps. */
2033 {
2034 /* For SLP vectorization we directly vectorize a subchain
2035 without permutation. */
2037 first_dr_info
2040 /* Try to use consecutive accesses of as many elements as possible,
2041 separated by the stride, until we have a complete vector.
2042 Fall back to scalar accesses if that isn't possible. */
2044 else
2045 {
2048 {
2050 /* ??? The VMAT_CONTIGUOUS_REVERSE code generation is
2051 only correct for single element "interleaving" SLP. */
2054 else
2055 {
2056 /* Try to use consecutive accesses of DR_GROUP_SIZE elements,
2057 separated by the stride, until we have a complete vector.
2058 Fall back to scalar accesses if that isn't possible. */
2061 else
2063 }
2064 }
2066 {
2069 /* Invariant accesses perform only component accesses, alignment
2070 is irrelevant for them. */
2072 }
2073 /* Try using LOAD/STORE_LANES. */
2075 && (*lanes_ifn
2081 else
2086 {
2088 "Grouped store with gaps requires"
2091 }
2092 /* An overrun is fine if the trailing elements are smaller
2093 than the alignment boundary B. Every vector access will
2094 be a multiple of B and so we are guaranteed to access a
2095 non-gap element in the same B-sized block. */
2098 vectype)
2102 /* When we have a contiguous access across loop iterations
2103 but the access in the loop doesn't cover the full vector
2104 we can end up with no gap recorded but still excess
2105 elements accessed, see PR103116. Make sure we peel for
2106 gaps if necessary and sufficient and give up if not.
2108 If there is a combination of the access not covering the full
2109 vector and a gap recorded then we may need to peel twice. */
2115 nunits))
2118 /* If the gap splits the vector in half and the target
2119 can do half-vector operations avoid the epilogue peeling
2120 by simply loading half of the vector only. Usually
2121 the construction with an upper zero half will be elided. */
2122 dr_alignment_support alss;
2124 tree half_vtype;
2125 poly_uint64 remain;
2128 && !masked_p
2132 == dr_aligned
2146 {
2151 }
2152 /* Peeling for gaps assumes that a single scalar iteration
2153 is enough to make sure the last vector iteration doesn't
2154 access excess elements. */
2160 {
2161 /* But peeling a single scalar iteration is enough if
2162 we can use the next power-of-two sized partial
2163 access and that is sufficiently small to be covered
2164 by the single scalar iteration. */
2171 || vector_vector_composition_type
2174 {
2177 "peeling for gaps insufficient for "
2180 }
2181 }
2183 /* If this is single-element interleaving with an element
2184 distance that leaves unused vector loads around punt - we
2185 at least create very sub-optimal code in that case (and
2186 blow up memory, see PR65518). */
2189 && single_element_p
2191 {
2193 {
2197 "single-element interleaving not supported "
2198 "for not adjacent vector loads, using "
2200 }
2201 else
2202 {
2205 "single-element interleaving not supported "
2208 }
2209 }
2210 }
2211 }
2212 else
2213 {
2214 /* We can always handle this case using elementwise accesses,
2215 but see if something more efficient is available. */
2218 /* If there is a gap at the end of the group then these optimizations
2219 would access excess elements in the last iteration. */
2221 /* An overrun is fine if the trailing elements are smaller than the
2222 alignment boundary B. Every vector access will be a multiple of B
2223 and so we are guaranteed to access a non-gap element in the
2224 same B-sized block. */
2226 && !masked_p
2234 {
2235 /* First cope with the degenerate case of a single-element
2236 vector. */
2238 ;
2240 else
2241 {
2242 /* Otherwise try using LOAD/STORE_LANES. */
2243 *lanes_ifn
2247 masked_p);
2249 {
2252 }
2254 /* If that fails, try using permuting loads. */
2257 single_element_p,
2258 group_size)
2260 {
2263 }
2264 }
2265 }
2266 }
2268 /* As a last resort, trying using a gather load or scatter store.
2270 ??? Although the code can handle all group sizes correctly,
2271 it probably isn't a win to use separate strided accesses based
2272 on nearby locations. Or, even if it's a win over scalar code,
2273 it might not be a win over vectorizing at a lower VF, if that
2274 allows us to use contiguous accesses. */
2276 && single_element_p
2277 && loop_vinfo
2284 {
2287 }
2288 else
2289 {
2291 *alignment_support_scheme
2294 }
2297 {
2298 /* STMT is the leader of the group. Check the operands of all the
2299 stmts of the group. */
2302 {
2306 {
2311 }
2313 }
2314 }
2317 {
2321 "Data access with gaps requires scalar "
2324 }
2327 }
2329 /* Analyze load or store statement STMT_INFO of type VLS_TYPE. Return true
2330 if there is a memory access type that the vectorized form can use,
2331 storing it in *MEMORY_ACCESS_TYPE if so. If we decide to use gathers
2332 or scatters, fill in GS_INFO accordingly. In addition
2333 *ALIGNMENT_SUPPORT_SCHEME is filled out and false is returned if
2334 the target does not support the alignment scheme. *MISALIGNMENT
2335 is set according to the alignment of the access (including
2336 DR_MISALIGNMENT_UNKNOWN when it is unknown).
2338 SLP says whether we're performing SLP rather than loop vectorization.
2339 MASKED_P is true if the statement is conditional on a vectorized mask.
2340 VECTYPE is the vector type that the vectorized statements will use.
2341 NCOPIES is the number of vector statements that will be needed. */
2343 static bool
2354 {
2360 {
2364 /* When using internal functions, we rely on pattern recognition
2365 to convert the type of the offset to the type that the target
2366 requires, with the result being a call to an internal function.
2367 If that failed for some reason (e.g. because another pattern
2368 took priority), just handle cases in which the offset already
2369 has the right type. */
2374 {
2380 }
2384 {
2390 }
2392 {
2398 {
2401 "unsupported vector types for emulated "
2404 }
2405 }
2406 /* Gather-scatter accesses perform only component accesses, alignment
2407 is irrelevant for them. */
2409 }
2411 {
2413 masked_p,
2415 alignment_support_scheme,
2418 }
2420 {
2426 else
2428 /* Alignment is irrelevant here. */
2430 }
2431 else
2432 {
2435 {
2438 /* Invariant accesses perform only component accesses, alignment
2439 is irrelevant for them. */
2441 }
2442 else
2443 {
2447 else
2451 *alignment_support_scheme
2455 }
2456 }
2461 {
2464 "Not using elementwise accesses due to variable "
2467 }
2470 {
2475 }
2477 /* FIXME: At the moment the cost model seems to underestimate the
2478 cost of using elementwise accesses. This check preserves the
2479 traditional behavior until that can be fixed. */
2488 {
2493 }
2495 }
2497 /* Return true if boolean argument at MASK_INDEX is suitable for vectorizing
2498 conditional operation STMT_INFO. When returning true, store the mask
2499 in *MASK, the type of its definition in *MASK_DT_OUT, the type of the
2500 vectorized mask in *MASK_VECTYPE_OUT and the SLP node corresponding
2501 to the mask in *MASK_NODE if MASK_NODE is not NULL. */
2503 static bool
2508 {
2510 tree mask_vectype;
2511 slp_tree mask_node_1;
2514 {
2519 }
2522 {
2527 }
2529 /* If the caller is not prepared for adjusting an external/constant
2530 SLP mask vector type fail. */
2532 && !mask_node
2534 {
2539 }
2544 mask_node_1);
2547 {
2552 }
2556 {
2559 "vector mask type %T"
2564 }
2571 }
2573 /* Return true if stored value is suitable for vectorizing store
2574 statement STMT_INFO. When returning true, store the scalar stored
2575 in *RHS and *RHS_NODE, the type of the definition in *RHS_DT_OUT,
2576 the type of the vectorized store value in
2577 *RHS_VECTYPE_OUT and the type of the store in *VLS_TYPE_OUT. */
2579 static bool
2584 {
2587 {
2591 }
2593 op_no = vect_slp_child_index_for_operand
2597 tree rhs_vectype;
2600 {
2605 }
2607 /* In the case this is a store from a constant make sure
2608 native_encode_expr can handle it. */
2611 {
2616 }
2620 {
2625 }
2631 else
2634 }
2636 /* Build an all-ones vector mask of type MASKTYPE while vectorizing STMT_INFO.
2637 Note that we support masks with floating-point type, in which case the
2638 floats are interpreted as a bitmask. */
2640 static tree
2643 {
2648 {
2652 }
2654 {
2655 REAL_VALUE_TYPE r;
2663 }
2665 }
2667 /* Build an all-zero merge value of type VECTYPE while vectorizing
2668 STMT_INFO as a gather load. */
2670 static tree
2673 {
2674 tree merge;
2678 {
2679 REAL_VALUE_TYPE r;
2685 }
2686 else
2690 }
2692 /* Build a gather load call while vectorizing STMT_INFO. Insert new
2693 instructions before GSI and add them to VEC_STMT. GS_INFO describes
2694 the gather load operation. If the load is conditional, MASK is the
2695 vectorized condition, otherwise MASK is null. PTR is the base
2696 pointer and OFFSET is the vectorized offset. */
2703 {
2712 tree var;
2714 && (!mask
2720 {
2728 }
2733 {
2735 {
2740 else
2744 gassign *new_stmt
2749 {
2756 }
2759 }
2760 else
2761 {
2764 }
2765 }
2766 else
2767 {
2770 }
2777 {
2785 }
2788 }
2790 /* Build a scatter store call while vectorizing STMT_INFO. Insert new
2791 instructions before GSI. GS_INFO describes the scatter store operation.
2792 PTR is the base pointer, OFFSET the vectorized offsets and OPRND the
2793 vectorized data to store.
2794 If the store is conditional, MASK is the vectorized condition, otherwise
2795 MASK is null. */
2802 {
2815 {
2818 tree utype;
2821 else
2825 gassign *new_stmt
2830 {
2836 }
2837 }
2838 else
2839 {
2842 }
2846 {
2854 }
2858 {
2866 }
2869 gcall *new_stmt
2872 }
2874 /* Prepare the base and offset in GS_INFO for vectorization.
2875 Set *DATAREF_PTR to the loop-invariant base address and *VEC_OFFSET
2876 to the vectorized offset argument for the first copy of STMT_INFO.
2877 STMT_INFO is the statement described by GS_INFO and LOOP is the
2878 containing loop. */
2880 static void
2885 {
2889 {
2890 basic_block new_bb;
2894 }
2897 else
2898 {
2899 unsigned ncopies
2904 }
2905 }
2907 /* Prepare to implement a grouped or strided load or store using
2908 the gather load or scatter store operation described by GS_INFO.
2909 STMT_INFO is the load or store statement.
2911 Set *DATAREF_BUMP to the amount that should be added to the base
2912 address after each copy of the vectorized statement. Set *VEC_OFFSET
2913 to an invariant offset vector in which element I has the value
2914 I * DR_STEP / SCALE. */
2916 static void
2918 loop_vec_info loop_vinfo,
2923 {
2928 {
2929 /* _31 = .SELECT_VL (ivtmp_29, POLY_INT_CST [4, 4]);
2930 ivtmp_8 = _31 * 16 (step in bytes);
2931 .MASK_LEN_SCATTER_STORE (vectp_a.9_7, ... );
2932 vectp_a.9_26 = vectp_a.9_7 + ivtmp_8; */
2933 tree loop_len
2935 tree tmp
2938 loop_len);
2940 GSI_SAME_STMT);
2941 }
2942 else
2943 {
2944 tree bump
2949 }
2951 /* The offset given in GS_INFO can have pointer type, so use the element
2952 type of the vector instead. */
2955 /* Calculate X = DR_STEP / SCALE and convert it to the appropriate type. */
2960 /* Create {0, X, X*2, X*3, ...}. */
2964 }
2966 /* Prepare the pointer IVs which needs to be updated by a variable amount.
2967 Such variable amount is the outcome of .SELECT_VL. In this case, we can
2968 allow each iteration process the flexible number of elements as long as
2969 the number <= vf elments.
2971 Return data reference according to SELECT_VL.
2972 If new statements are needed, insert them before GSI. */
2974 static tree
2978 vect_memory_access_type memory_access_type)
2979 {
2983 /* gather/scatter never reach here. */
2986 /* When we support SELECT_VL pattern, we dynamic adjust
2987 the memory address by .SELECT_VL result.
2989 The result of .SELECT_VL is the number of elements to
2990 be processed of each iteration. So the memory address
2991 adjustment operation should be:
2993 addr = addr + .SELECT_VL (ARG..) * step;
2994 */
2995 tree loop_len
2998 /* Since the outcome of .SELECT_VL is element size, we should adjust
2999 it into bytesize so that it can be used in address pointer variable
3000 amount IVs adjustment. */
3007 }
3009 /* Return the amount that should be added to a vector pointer to move
3010 to the next or previous copy of AGGR_TYPE. DR_INFO is the data reference
3011 being vectorized and MEMORY_ACCESS_TYPE describes the type of
3012 vectorization. */
3014 static tree
3017 vect_memory_access_type memory_access_type,
3019 {
3027 memory_access_type);
3034 }
3036 /* Check and perform vectorization of BUILT_IN_BSWAP{16,32,64,128}. */
3038 static bool
3044 {
3054 /* Multiple types in SLP are handled by creating the appropriate number of
3055 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3056 case of SLP. */
3059 else
3065 {
3071 }
3082 /* The encoding uses one stepped pattern for each byte in the word. */
3094 {
3097 {
3102 }
3109 slp_node
3113 }
3117 /* Transform. */
3121 /* Arguments are ready. create the new vector stmt. */
3123 tree vop;
3125 {
3141 else
3143 }
3150 }
3152 /* Return true if vector types VECTYPE_IN and VECTYPE_OUT have
3153 integer elements and if we can narrow VECTYPE_IN to VECTYPE_OUT
3154 in a single step. On success, store the binary pack code in
3155 *CONVERT_CODE. */
3157 static bool
3160 {
3165 code_helper code;
3175 }
3177 /* Function vectorizable_call.
3179 Check if STMT_INFO performs a function call that can be vectorized.
3180 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3181 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3182 Return true if STMT_INFO is vectorizable in this way. */
3184 static bool
3189 {
3191 tree vec_dest;
3192 tree scalar_dest;
3193 tree op;
3196 poly_uint64 nunits_in;
3197 poly_uint64 nunits_out;
3203 vect_unknown_def_type };
3211 tree lhs;
3221 /* Is STMT_INFO a vectorizable call? */
3229 /* Handled by vectorizable_load and vectorizable_store. */
3240 /* Process function arguments. */
3245 /* Bail out if the function has more than four arguments, we do not have
3246 interesting builtin functions to vectorize with more than two arguments
3247 except for fma. No arguments is also not good. */
3251 /* Ignore the arguments of IFN_GOMP_SIMD_LANE, they are magic. */
3254 {
3257 }
3258 /* Similarly pretend IFN_CLZ and IFN_CTZ only has one argument, the second
3259 argument just says whether it is well-defined at zero or not and what
3260 value should be returned for it. */
3262 {
3265 }
3272 {
3274 {
3279 }
3283 {
3288 }
3290 /* We can only handle calls with arguments of the same type. */
3293 {
3298 }
3306 {
3311 }
3312 }
3313 /* If all arguments are external or constant defs, infer the vector type
3314 from the scalar type. */
3320 {
3326 }
3330 {
3335 }
3338 {
3343 }
3345 /* FORNOW */
3354 else
3357 /* We only handle functions that do not read or clobber memory. */
3359 {
3364 }
3366 /* For now, we only vectorize functions if a target specific builtin
3367 is available. TODO -- in some cases, it might be profitable to
3368 insert the calls for pieces of the vector, in order to be able
3369 to vectorize other operations in the loop. */
3374 /* First try using an internal function. */
3382 vectype_in);
3384 /* If that fails, try asking for a target-specific built-in function. */
3386 {
3393 }
3396 {
3399 && loop_vinfo
3404 {
3405 /* We can handle IFN_GOMP_SIMD_LANE by returning a
3406 { 0, 1, 2, ... vf - 1 } vector. */
3408 }
3416 else
3417 {
3422 }
3423 }
3429 else
3432 /* Sanity check: make sure that at least one copy of the vectorized stmt
3433 needs to be generated. */
3443 {
3449 {
3454 }
3466 {
3470 OPTIMIZE_FOR_SPEED))
3473 OPTIMIZE_FOR_SPEED)))
3474 {
3477 "can't use a fully-masked loop because no"
3480 }
3481 else
3482 {
3483 unsigned int nvectors
3484 = (slp_node
3492 OPTIMIZE_FOR_SPEED))
3495 else
3497 scalar_mask);
3498 }
3499 }
3501 }
3503 /* Transform. */
3508 /* Handle def. */
3516 {
3518 {
3520 /* COND_* -> COND_LEN_* takes 2 extra arguments:LEN,BIAS. */
3522 }
3525 }
3527 {
3530 }
3535 {
3540 {
3541 /* Build argument list for the vectorized call. */
3543 {
3545 {
3547 {
3548 /* ??? For multi-lane SLP we'd need to build
3549 { 0, 0, .., 1, 1, ... }. */
3552 tree new_var
3558 gimple *new_stmt
3561 gsi);
3563 }
3565 }
3571 /* Arguments are ready. Create the new vector stmt. */
3573 {
3576 {
3578 /* Always true for SLP. */
3583 }
3586 {
3589 }
3597 {
3598 /* We don't define any narrowing conditional functions
3599 at present. */
3602 gcall *call
3608 {
3611 }
3617 }
3618 else
3619 {
3621 {
3623 /* Always true for SLP. */
3625 tree len
3628 signed char biasval
3633 }
3635 {
3637 /* Always true for SLP. */
3645 }
3650 else
3657 }
3659 }
3661 }
3668 {
3671 {
3676 }
3678 }
3685 {
3688 signed char biasval
3693 }
3695 {
3701 }
3705 {
3707 tree new_var
3714 }
3716 {
3717 /* We don't define any narrowing conditional functions at
3718 present. */
3726 {
3729 }
3732 half_res);
3734 }
3735 else
3736 {
3740 else
3747 }
3752 }
3754 {
3757 }
3758 }
3760 {
3762 /* We don't define any narrowing conditional functions at present. */
3765 {
3766 /* Build argument list for the vectorized call. */
3769 else
3773 {
3779 /* Arguments are ready. Create the new vector stmt. */
3781 {
3785 {
3789 }
3793 else
3800 }
3802 }
3805 {
3808 {
3812 }
3818 }
3826 }
3832 {
3835 }
3836 }
3837 else
3838 /* No current target implements this case. */
3843 /* The call in STMT might prevent it from being removed in dce.
3844 We however cannot remove it here, due to the way the ssa name
3845 it defines is mapped to the new definition. So just replace
3846 rhs of the statement with something harmless. */
3854 gassign *new_stmt
3859 }
3862 struct simd_call_arg_info
3863 {
3864 tree vectype;
3865 tree op;
3866 HOST_WIDE_INT linear_step;
3870 };
3872 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
3873 is linear within simd lane (but not within whole loop), note it in
3874 *ARGINFO. */
3876 static void
3879 {
3891 {
3892 tree t;
3896 {
3911 CASE_CONVERT:
3923 }
3929 {
3936 }
3937 }
3938 }
3940 /* Function vectorizable_simd_clone_call.
3942 Check if STMT_INFO performs a function call that can be vectorized
3943 by calling a simd clone of the function.
3944 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
3945 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3946 Return true if STMT_INFO is vectorizable in this way. */
3948 static bool
3952 stmt_vector_for_cost *)
3953 {
3954 tree vec_dest;
3955 tree scalar_dest;
3958 tree vectype;
3959 poly_uint64 nunits;
3972 /* Is STMT a vectorizable call? */
3980 {
3986 }
4012 /* Process function arguments. */
4015 /* Bail out if the function has zero arguments. */
4028 {
4029 simd_call_arg_info thisarginfo;
4030 affine_iv iv;
4044 {
4049 }
4053 {
4054 /* With SLP we determine the vector type of constants/externals
4055 at analysis time, handling conflicts via
4056 vect_maybe_update_slp_op_vectype. At transform time
4057 we have a vector type recorded for SLP. */
4059 || !slp_node
4064 slp_node);
4065 }
4066 else
4069 /* For linear arguments, the analyze phase should have saved
4070 the base and step in {STMT_VINFO,SLP_TREE}_SIMD_CLONE_INFO. */
4074 {
4077 thisarginfo.simd_lane_linear
4079 /* If loop has been peeled for alignment, we need to adjust it. */
4083 {
4089 thisarginfo.op
4093 }
4094 }
4098 && loop_vinfo
4103 {
4106 }
4111 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
4112 linear too. */
4115 && !vec_stmt
4118 && loop_vinfo
4123 }
4131 else
4134 {
4137 /* The number of arguments in the call and the number of parameters in
4138 the simdclone should match. However, when the simdclone is
4139 'inbranch', it could have one more paramater than nargs when using
4140 an inbranch simdclone to call a non-inbranch call, either in a
4141 non-masked loop using a all true constant mask, or inside a masked
4142 loop using it's mask. */
4145 simd_nargs--;
4160 {
4162 {
4193 /* FORNOW */
4197 /* While we can create a traditional data vector from
4198 an incoming integer mode mask we have no good way to
4199 force generate an integer mode mask from a traditional
4200 boolean vector input. */
4208 }
4212 {
4215 }
4219 }
4225 {
4227 SIMD_CLONE_ARG_TYPE_MASK);
4228 /* Penalize using a masked SIMD clone in a non-masked loop, that is
4229 not in a branch, as we'd have to construct an all-true mask. */
4232 }
4234 {
4237 }
4238 }
4247 num_mask_args++;
4250 {
4254 {
4258 slp_node);
4263 }
4267 {
4272 }
4275 {
4278 {
4281 {
4282 /* FORNOW we only have partial support for vector-type masks
4283 that can't hold all of simdlen. */
4286 vect_location,
4287 "in-branch vector clones are not yet"
4290 }
4293 {
4296 vect_location,
4297 "cannot compute mask argument for"
4300 }
4301 }
4303 {
4306 num_mask_args),
4308 {
4309 /* FORNOW we only have partial support for integer-type masks
4310 that represent the same number of lanes as the
4311 vectorized mask inputs. */
4314 vect_location,
4315 "in-branch vector clones are not yet "
4318 }
4319 }
4320 else
4321 {
4324 vect_location,
4325 "in-branch vector clones not supported"
4328 }
4329 }
4330 }
4336 else
4339 /* If the function isn't const, only allow it in simd loops where user
4340 has asserted that at least nunits consecutive iterations can be
4341 performed using SIMD instructions. */
4346 /* Sanity check: make sure that at least one copy of the vectorized stmt
4347 needs to be generated. */
4351 {
4355 {
4360 }
4361 /* When the original call is pure or const but the SIMD ABI dictates
4362 an aggregate return we will have to use a virtual definition and
4363 in a loop eventually even need to add a virtual PHI. That's
4364 not straight-forward so allow to fix this up via renaming. */
4369 /* ??? For SLP code-gen we end up inserting after the last
4370 vector argument def rather than at the original call position
4371 so automagic virtual operand updating doesn't work. */
4376 {
4378 {
4383 {
4393 }
4398 {
4399 unsigned nmasks
4405 }
4408 }
4409 }
4412 {
4416 "can't use a fully-masked loop because a"
4419 }
4423 /* vect_model_simple_cost (vinfo, stmt_info, ncopies,
4424 dt, slp_node, cost_vec); */
4426 }
4428 /* Transform. */
4433 /* Handle def. */
4439 {
4443 {
4446 }
4447 }
4453 {
4456 }
4457 else
4460 {
4461 poly_uint64 callee_nelements;
4462 poly_uint64 caller_nelements;
4463 /* Build argument list for the vectorized call. */
4466 else
4470 {
4472 tree atype;
4475 {
4482 {
4484 {
4492 {
4499 }
4500 else
4501 {
4505 }
4507 vec_oprnd0
4511 gassign *new_stmt
4513 vec_oprnd0);
4517 }
4518 else
4519 {
4527 else
4530 {
4532 {
4536 op,
4540 }
4541 else
4547 vec_oprnd0);
4548 }
4551 atype))
4552 {
4554 vec_oprnd0);
4555 gassign *new_stmt
4557 vec_oprnd0);
4561 }
4562 else
4564 else
4565 {
4567 gassign *new_stmt
4569 vec_oprnd0);
4573 }
4574 }
4575 }
4579 {
4588 {
4590 {
4591 /* The mask type has fewer elements than simdlen. */
4593 /* FORNOW */
4595 }
4597 {
4598 /* The SIMD clone function has the same number of
4599 elements as the current function. */
4601 {
4605 op,
4608 }
4612 {
4613 vec_loop_masks *loop_masks
4615 tree loop_mask
4619 vec_oprnd0
4623 gsi);
4625 loop_mask });
4627 }
4628 vec_oprnd0
4632 gassign *new_stmt
4634 vec_oprnd0);
4638 }
4639 else
4640 {
4641 /* The mask type has more elements than simdlen. */
4643 /* FORNOW */
4645 }
4646 }
4647 }
4649 {
4651 /* Guess the number of lanes represented by atype. */
4652 poly_uint64 atype_subparts
4654 num_mask_args);
4657 {
4659 {
4663 op,
4666 }
4669 {
4670 /* The mask argument has fewer elements than the
4671 input vector. */
4672 /* FORNOW */
4674 }
4677 {
4678 /* The vector mask argument matches the input
4679 in the number of lanes, but not necessarily
4680 in the mode. */
4685 vec_oprnd0);
4686 gassign *new_stmt
4688 vec_oprnd0);
4692 {
4693 new_stmt
4695 NOP_EXPR,
4696 gimple_assign_lhs
4700 }
4702 }
4703 else
4704 {
4705 /* The mask argument has more elements than the
4706 input vector. */
4707 /* FORNOW */
4709 }
4710 }
4711 }
4712 else
4721 {
4722 gimple_seq stmts;
4727 {
4728 basic_block new_bb;
4732 }
4734 {
4737 }
4742 enum tree_code code
4747 poly_widest_int cst
4752 gassign *new_stmt
4757 UNKNOWN_LOCATION);
4760 }
4761 else
4762 {
4763 enum tree_code code
4768 poly_widest_int cst
4773 gassign *new_stmt
4778 }
4788 }
4789 }
4794 {
4797 tree mask;
4799 SIMD_CLONE_ARG_TYPE_MASK);
4803 /* Guess the number of lanes represented by masktype. */
4806 else
4810 {
4812 {
4816 }
4817 else
4822 {
4823 /* This means we are dealing with integer mask modes.
4824 First convert to an integer type with the same size as
4825 the current vector type. */
4826 unsigned HOST_WIDE_INT intermediate_size
4828 tree mid_int_type =
4831 new_stmt
4833 mask);
4835 /* Then zero-extend to the mask mode. */
4838 }
4840 {
4842 integer_one_node);
4844 integer_zero_node);
4848 }
4849 else
4857 }
4858 }
4862 {
4869 else
4872 }
4877 {
4879 {
4887 {
4888 tree t;
4890 {
4894 }
4895 else
4906 else
4908 }
4913 }
4915 {
4924 {
4929 {
4933 tem);
4938 }
4940 }
4941 else
4946 new_stmt
4955 else
4958 }
4960 {
4967 }
4969 {
4971 new_stmt
4974 }
4975 }
4981 else
4983 }
4986 {
4989 }
4992 /* Mark the clone as no longer being a candidate for GC. */
4995 /* The call in STMT might prevent it from being removed in dce.
4996 We however cannot remove it here, due to the way the ssa name
4997 it defines is mapped to the new definition. So just replace
4998 rhs of the statement with something harmless. */
5005 {
5009 }
5010 else
5016 }
5019 /* Function vect_gen_widened_results_half
5021 Create a vector stmt whose code, type, number of arguments, and result
5022 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
5023 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at GSI.
5024 In the case that CODE is a CALL_EXPR, this means that a call to DECL
5025 needs to be created (DECL is a function-decl of a target-builtin).
5026 STMT_INFO is the original scalar stmt that we are vectorizing. */
5032 stmt_vec_info stmt_info)
5033 {
5035 tree new_temp;
5037 /* Generate half of the widened result: */
5046 }
5049 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
5050 For multi-step conversions store the resulting vectors and call the function
5051 recursively. When NARROW_SRC_P is true, there's still a conversion after
5052 narrowing, don't store the vectors in the SLP_NODE or in vector info of
5053 the scalar statement(or in STMT_VINFO_RELATED_STMT chain). */
5055 static void
5058 stmt_vec_info stmt_info,
5063 {
5070 {
5071 /* Create demotion operation. */
5079 /* Store the resulting vector for next recursive call,
5080 or return the resulting vector_tmp for NARROW FLOAT_EXPR. */
5082 else
5083 {
5084 /* This is the last step of the conversion sequence. Store the
5085 vectors in SLP_NODE or in vector info of the scalar statement
5086 (or in STMT_VINFO_RELATED_STMT chain). */
5089 else
5091 }
5092 }
5094 /* For multi-step demotion operations we first generate demotion operations
5095 from the source type to the intermediate types, and then combine the
5096 results (stored in VEC_OPRNDS) in demotion operation to the destination
5097 type. */
5099 {
5100 /* At each level of recursion we have half of the operands we had at the
5101 previous level. */
5107 narrow_src_p);
5108 }
5111 }
5114 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
5115 and VEC_OPRNDS1, for a binary operation associated with scalar statement
5116 STMT_INFO. For multi-step conversions store the resulting vectors and
5117 call the function recursively. */
5119 static void
5125 code_helper ch1,
5127 {
5135 {
5138 else
5141 /* Generate the two halves of promotion operation. */
5144 stmt_info);
5147 stmt_info);
5149 {
5152 }
5153 else
5154 {
5157 }
5159 /* Store the results for the next step. */
5162 }
5166 }
5168 /* Create vectorized promotion stmts for widening stmts using only half the
5169 potential vector size for input. */
5170 static void
5176 code_helper code1,
5178 {
5188 {
5194 /* Widen the first vector input. */
5200 {
5201 /* Widen the second vector input. */
5205 /* Perform the operation. With both vector inputs widened. */
5207 }
5208 else
5209 {
5210 /* Perform the operation. With the single vector input widened. */
5212 }
5218 /* Store the results for the next step. */
5220 }
5224 }
5227 /* Check if STMT_INFO performs a conversion operation that can be vectorized.
5228 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5229 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5230 Return true if STMT_INFO is vectorizable in this way. */
5232 static bool
5237 {
5239 tree scalar_dest;
5242 tree_code tc1;
5245 tree new_temp;
5248 poly_uint64 nunits_in;
5249 poly_uint64 nunits_out;
5253 /* For conversions between floating point and integer, there're 2 NARROW
5254 cases. NARROW_SRC is for FLOAT_EXPR, means
5255 integer --DEMOTION--> integer --FLOAT_EXPR--> floating point.
5256 This is safe when the range of the source integer can fit into the lower
5257 precision. NARROW_DST is for FIX_TRUNC_EXPR, means
5258 floating point --FIX_TRUNC_EXPR--> integer --DEMOTION--> INTEGER.
5259 For other conversions, when there's narrowing, NARROW_DST is used as
5260 default. */
5264 tree vop0;
5272 /* Is STMT a vectorizable conversion? */
5293 {
5296 }
5298 {
5301 }
5302 else
5315 /* Check types of lhs and rhs. */
5320 /* Check the operands of the operation. */
5324 {
5329 }
5344 {
5347 "type conversion to/from bit-precision unsupported."
5350 }
5353 {
5360 tree vectype1_in;
5363 {
5368 }
5369 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
5370 OP1. */
5373 }
5375 /* If op0 is an external or constant def, infer the vector type
5376 from the scalar type. */
5382 {
5388 }
5392 {
5395 "can't convert between boolean and non "
5399 }
5406 else
5410 else
5411 {
5414 }
5416 /* Multiple types in SLP are handled by creating the appropriate number of
5417 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5418 case of SLP. */
5423 else
5426 /* Sanity check: make sure that at least one copy of the vectorized stmt
5427 needs to be generated. */
5433 opt_scalar_mode rhs_mode_iter;
5436 /* Supportable by target? */
5438 {
5446 vectype_out,
5447 vectype_in,
5449 op0))
5450 {
5454 else
5455 {
5461 }
5463 }
5465 /* FALLTHRU */
5466 unsupported:
5474 {
5483 }
5488 {
5489 /* Binary widening operation can only be supported directly by the
5490 architecture. */
5493 }
5501 {
5506 cvt_type
5513 {
5520 }
5527 else
5531 cvt_type,
5535 {
5538 }
5539 }
5546 else
5547 {
5548 multi_step_cvt++;
5551 }
5565 {
5566 cvt_type
5574 else
5580 }
5581 /* If op0 can be represented with low precision integer,
5582 truncate it to cvt_type and the do FLOAT_EXPR. */
5584 {
5589 cvt_type
5607 {
5611 }
5612 }
5618 }
5621 {
5625 {
5630 }
5633 {
5638 }
5640 {
5642 /* The final packing step produces one vector result per copy. */
5643 unsigned int nvectors
5647 widen_arith);
5648 }
5649 else
5650 {
5652 /* The initial unpacking step produces two vector results
5653 per copy. MULTI_STEP_CVT is 0 for a single conversion,
5654 so >> MULTI_STEP_CVT divides by 2^(number of steps - 1). */
5655 unsigned int nvectors
5656 = (slp_node
5661 widen_arith);
5662 }
5665 }
5667 /* Transform. */
5673 {
5678 }
5680 /* In case of multi-step conversion, we first generate conversion operations
5681 to the intermediate types, and then from that types to the final one.
5682 We create vector destinations for the intermediate type (TYPES) received
5683 from supportable_*_operation, and store them in the correct order
5684 for future use in vect_create_vectorized_*_stmts (). */
5686 bool widen_or_narrow_float_p
5689 widen_or_narrow_float_p
5694 {
5697 {
5699 intermediate_type);
5701 }
5702 }
5706 widen_or_narrow_float_p
5711 {
5713 ;
5715 {
5719 }
5720 }
5723 {
5727 /* vec_dest is intermediate type operand when multi_step_cvt. */
5729 {
5732 }
5735 {
5736 /* Arguments are ready, create the new vector stmt. */
5739 {
5746 }
5754 else
5756 }
5760 /* In case the vectorization factor (VF) is bigger than the number
5761 of elements that we can fit in a vectype (nunits), we have to
5762 generate more than one vector stmt - i.e - we need to "unroll"
5763 the vector stmt by a factor VF/nunits. */
5769 {
5774 }
5775 /* Arguments are ready. Create the new vector stmts. */
5777 {
5781 {
5784 }
5788 op_type);
5789 else
5794 }
5797 {
5800 {
5804 }
5805 else
5810 else
5812 }
5817 /* In case the vectorization factor (VF) is bigger than the number
5818 of elements that we can fit in a vectype (nunits), we have to
5819 generate more than one vector stmt - i.e - we need to "unroll"
5820 the vector stmt by a factor VF/nunits. */
5823 /* Arguments are ready. Create the new vector stmts. */
5826 {
5831 }
5834 multi_step_cvt,
5838 /* After demoting op0 to cvt_type, convert it to dest. */
5840 {
5842 {
5843 /* Arguments are ready, create the new vector stmt. */
5845 gimple *new_stmt
5851 /* This is the last step of the conversion sequence. Store the
5852 vectors in SLP_NODE or in vector info of the scalar statement
5853 (or in STMT_VINFO_RELATED_STMT chain). */
5856 else
5858 }
5859 }
5861 }
5870 }
5872 /* Return true if we can assume from the scalar form of STMT_INFO that
5873 neither the scalar nor the vector forms will generate code. STMT_INFO
5874 is known not to involve a data reference. */
5876 bool
5878 {
5894 }
5896 /* Function vectorizable_assignment.
5898 Check if STMT_INFO performs an assignment (copy) that can be vectorized.
5899 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
5900 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5901 Return true if STMT_INFO is vectorizable in this way. */
5903 static bool
5908 {
5909 tree vec_dest;
5910 tree scalar_dest;
5911 tree op;
5913 tree new_temp;
5919 tree vop;
5922 tree vectype_in;
5931 /* Is vectorizable assignment? */
5952 /* Multiple types in SLP are handled by creating the appropriate number of
5953 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5954 case of SLP. */
5957 else
5962 slp_tree slp_op;
5965 {
5970 }
5974 /* We can handle VIEW_CONVERT conversions that do not change the number
5975 of elements or the vector size or other conversions when the component
5976 types are nop-convertible. */
5988 {
5991 "can't convert between boolean and non "
5995 }
5997 /* We do not handle bit-precision changes. */
6004 /* But a conversion that does not change the bit-pattern is ok. */
6012 {
6015 "type conversion to/from bit-precision "
6018 }
6021 {
6024 {
6029 }
6034 cost_vec);
6036 }
6038 /* Transform. */
6042 /* Handle def. */
6045 /* Handle use. */
6048 /* Arguments are ready. create the new vector stmt. */
6050 {
6060 else
6062 }
6068 }
6071 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
6072 either as shift by a scalar or by a vector. */
6074 bool
6076 {
6078 machine_mode vec_mode;
6079 optab optab;
6081 tree vectype;
6090 {
6096 }
6104 }
6107 /* Function vectorizable_shift.
6109 Check if STMT_INFO performs a shift operation that can be vectorized.
6110 If VEC_STMT is also passed, vectorize the STMT_INFO: create a vectorized
6111 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6112 Return true if STMT_INFO is vectorizable in this way. */
6114 static bool
6119 {
6120 tree vec_dest;
6121 tree scalar_dest;
6124 tree vectype;
6127 machine_mode vec_mode;
6128 tree new_temp;
6129 optab optab;
6131 machine_mode optab_op2_mode;
6134 poly_uint64 nunits_in;
6135 poly_uint64 nunits_out;
6136 tree vectype_out;
6137 tree op1_vectype;
6156 /* Is STMT a vectorizable binary/unary operation? */
6173 {
6178 }
6180 slp_tree slp_op0;
6183 {
6188 }
6189 /* If op0 is an external or constant def, infer the vector type
6190 from the scalar type. */
6196 {
6201 }
6208 stmt_vec_info op1_def_stmt_info;
6209 slp_tree slp_op1;
6212 {
6217 }
6219 /* Multiple types in SLP are handled by creating the appropriate number of
6220 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6221 case of SLP. */
6224 else
6229 /* Determine whether the shift amount is a vector, or scalar. If the
6230 shift/rotate amount is a vector, use the vector/vector shift optabs. */
6240 {
6241 /* In SLP, need to check whether the shift count is the same,
6242 in loops if it is a constant or invariant, it is always
6243 a scalar shift. */
6245 {
6247 stmt_vec_info slpstmt_info;
6251 {
6255 }
6257 /* For internal SLP defs we have to make sure we see scalar stmts
6258 for all vector elements.
6259 ??? For different vectors we could resort to a different
6260 scalar shift operand but code-generation below simply always
6261 takes the first. */
6266 }
6268 /* If the shift amount is computed by a pattern stmt we cannot
6269 use the scalar amount directly thus give up and use a vector
6270 shift. */
6273 }
6274 else
6275 {
6280 }
6282 /* Vector shifted by vector. */
6285 {
6293 slp_op1);
6294 incompatible_op1_vectype_p
6300 && (!slp_node
6303 {
6306 "unusable type for last operand in"
6309 }
6310 }
6311 /* See if the machine has a vector shifted by scalar insn and if not
6312 then see if it has a vector shifted by vector insn. */
6313 else
6314 {
6318 {
6322 }
6323 else
6324 {
6329 {
6339 slp_op1);
6341 /* Unlike the other binary operators, shifts/rotates have
6342 the rhs being int, instead of the same type as the lhs,
6343 so make sure the scalar is the right type if we are
6344 dealing with vectors of long long/long/short/char. */
6345 incompatible_op1_vectype_p
6346 = (!op1_vectype
6351 {
6354 "unusable type for last operand in"
6357 }
6358 }
6359 }
6360 }
6362 /* Supportable by target? */
6364 {
6369 }
6373 {
6378 }
6379 /* vector lowering cannot optimize vector shifts using word arithmetic. */
6384 {
6388 && (!incompatible_op1_vectype_p
6390 && !vect_maybe_update_slp_op_vectype
6393 {
6398 }
6399 /* Now adjust the constant shift amount in place. */
6401 && incompatible_op1_vectype_p
6403 {
6406 {
6412 }
6413 }
6419 }
6421 /* Transform. */
6428 {
6434 }
6436 /* Handle def. */
6440 {
6441 /* Vector shl and shr insn patterns can be defined with scalar
6442 operand 2 (shift operand). In this case, use constant or loop
6443 invariant op1 directly, without extending it to vector mode
6444 first. */
6447 {
6454 /* Store vec_oprnd1 for every vector stmt to be created.
6455 We check during the analysis that all the shift arguments
6456 are the same.
6457 TODO: Allow different constants for different vector
6458 stmts generated for an SLP instance. */
6462 }
6463 }
6465 {
6467 {
6468 /* If the argument was the same in all lanes create
6469 the correctly typed vector shift amount directly. */
6478 }
6480 /* The constant shift amount has been adjusted in place. */
6481 ;
6482 else
6484 }
6486 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
6487 (a special case for certain kind of vector shifts); otherwise,
6488 operand 1 should be of a vector type (the usual case). */
6493 /* Arguments are ready. Create the new vector stmt. */
6495 {
6496 /* For internal defs where we need to use a scalar shift arg
6497 extract the first lane. */
6499 {
6502 gassign *new_stmt
6505 vop1,
6507 bitsize_zero_node));
6510 }
6511 else
6519 else
6521 }
6530 }
6532 /* Function vectorizable_operation.
6534 Check if STMT_INFO performs a binary, unary or ternary operation that can
6535 be vectorized.
6536 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
6537 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
6538 Return true if STMT_INFO is vectorizable in this way. */
6540 static bool
6545 {
6546 tree vec_dest;
6547 tree scalar_dest;
6549 tree vectype;
6552 machine_mode vec_mode;
6553 tree new_temp;
6555 optab optab;
6560 poly_uint64 nunits_in;
6561 poly_uint64 nunits_out;
6562 tree vectype_out;
6579 /* Is STMT a vectorizable binary/unary operation? */
6584 /* Loads and stores are handled in vectorizable_{load,store}. */
6590 /* Shifts are handled in vectorizable_shift. */
6597 /* Comparisons are handled in vectorizable_comparison. */
6601 /* Conditions are handled in vectorizable_condition. */
6605 /* For pointer addition and subtraction, we should use the normal
6606 plus and minus for the vector operation. */
6612 /* Support only unary or binary operations. */
6615 {
6619 op_type);
6621 }
6626 /* Most operations cannot handle bit-precision types without extra
6627 truncations. */
6631 /* Exception are bitwise binary operations. */
6635 {
6640 }
6642 slp_tree slp_op0;
6645 {
6650 }
6653 /* If op0 is an external or constant def, infer the vector type
6654 from the scalar type. */
6656 {
6657 /* For boolean type we cannot determine vectype by
6658 invariant value (don't know whether it is a vector
6659 of booleans or vector of integers). We use output
6660 vectype because operations on boolean don't change
6661 type. */
6663 {
6665 {
6670 }
6672 }
6673 else
6675 slp_node);
6676 }
6680 {
6687 }
6698 {
6701 {
6706 }
6714 }
6716 {
6719 {
6724 }
6732 }
6734 /* Multiple types in SLP are handled by creating the appropriate number of
6735 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6736 case of SLP. */
6738 {
6741 }
6742 else
6743 {
6746 }
6750 /* Reject attempts to combine mask types with nonmask types, e.g. if
6751 we have an AND between a (nonmask) boolean loaded from memory and
6752 a (mask) boolean result of a comparison.
6754 TODO: We could easily fix these cases up using pattern statements. */
6758 {
6763 }
6765 /* Supportable by target? */
6770 else
6771 {
6774 {
6779 }
6782 }
6786 {
6790 /* When vec_mode is not a vector mode and we verified ops we
6791 do not have to lower like AND are natively supported let
6792 those through even when the mode isn't word_mode. For
6793 ops we have to lower the lowering code assumes we are
6794 dealing with word_mode. */
6799 /* Check only during analysis. */
6801 {
6805 }
6810 }
6818 /* If operating on inactive elements could generate spurious traps,
6819 we need to restrict the operation to active lanes. Note that this
6820 specifically doesn't apply to unhoisted invariants, since they
6821 operate on the same value for every lane.
6823 Similarly, if this operation is part of a reduction, a fully-masked
6824 loop should only change the active lanes of the reduction chain,
6825 keeping the inactive lanes as-is. */
6830 {
6834 {
6837 OPTIMIZE_FOR_SPEED))
6842 OPTIMIZE_FOR_SPEED))
6845 else
6846 {
6849 "can't use a fully-masked loop because no"
6852 }
6853 }
6855 /* Put types on constant and invariant SLP children. */
6860 {
6865 }
6872 {
6873 /* The above vect_model_simple_cost call handles constants
6874 in the prologue and (mis-)costs one of the stmts as
6875 vector stmt. See below for the actual lowering that will
6876 be applied. */
6877 unsigned n
6880 {
6891 /* Bit operations do not have extra cost and are accounted
6892 as vector stmt by vect_model_simple_cost. */
6895 }
6897 {
6898 /* We also need to materialize two large constants. */
6903 }
6904 }
6906 }
6908 /* Transform. */
6917 /* POINTER_DIFF_EXPR has pointer arguments which are vectorized as
6918 vectors with unsigned elements, but the result is signed. So, we
6919 need to compute the MINUS_EXPR into vectype temporary and
6920 VIEW_CONVERT_EXPR it into the final vectype_out result. */
6923 {
6926 }
6927 /* Handle def. */
6928 else
6931 /* In case the vectorization factor (VF) is bigger than the number
6932 of elements that we can fit in a vectype (nunits), we have to generate
6933 more than one vector stmt - i.e - we need to "unroll" the
6934 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6935 from one copy of the vector stmt to the next, in the field
6936 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6937 stages to find the correct vector defs to be used when vectorizing
6938 stmts that use the defs of the current stmt. The example below
6939 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
6940 we need to create 4 vectorized stmts):
6942 before vectorization:
6943 RELATED_STMT VEC_STMT
6944 S1: x = memref - -
6945 S2: z = x + 1 - -
6947 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
6948 there):
6949 RELATED_STMT VEC_STMT
6950 VS1_0: vx0 = memref0 VS1_1 -
6951 VS1_1: vx1 = memref1 VS1_2 -
6952 VS1_2: vx2 = memref2 VS1_3 -
6953 VS1_3: vx3 = memref3 - -
6954 S1: x = load - VS1_0
6955 S2: z = x + 1 - -
6957 step2: vectorize stmt S2 (done here):
6958 To vectorize stmt S2 we first need to find the relevant vector
6959 def for the first operand 'x'. This is, as usual, obtained from
6960 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
6961 that defines 'x' (S1). This way we find the stmt VS1_0, and the
6962 relevant vector def 'vx0'. Having found 'vx0' we can generate
6963 the vector stmt VS2_0, and as usual, record it in the
6964 STMT_VINFO_VEC_STMT of stmt S2.
6965 When creating the second copy (VS2_1), we obtain the relevant vector
6966 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
6967 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
6968 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
6969 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
6970 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
6971 chain of stmts and pointers:
6972 RELATED_STMT VEC_STMT
6973 VS1_0: vx0 = memref0 VS1_1 -
6974 VS1_1: vx1 = memref1 VS1_2 -
6975 VS1_2: vx2 = memref2 VS1_3 -
6976 VS1_3: vx3 = memref3 - -
6977 S1: x = load - VS1_0
6978 VS2_0: vz0 = vx0 + v1 VS2_1 -
6979 VS2_1: vz1 = vx1 + v1 VS2_2 -
6980 VS2_2: vz2 = vx2 + v1 VS2_3 -
6981 VS2_3: vz3 = vx3 + v1 - -
6982 S2: z = x + 1 - VS2_0 */
6986 /* Arguments are ready. Create the new vector stmt. */
6988 {
6995 {
6996 /* Lower the operation. This follows vector lowering. */
6999 tree word_type
7003 tree high_bits
7012 {
7030 else
7035 {
7041 }
7050 }
7051 else
7052 {
7069 }
7071 signs);
7081 }
7083 {
7084 tree mask;
7088 else
7089 /* Dummy mask. */
7099 {
7100 /* Perform the operation on active elements only and take
7101 inactive elements from the reduction chain input. */
7104 }
7105 else
7106 {
7110 }
7112 {
7115 signed char biasval
7120 }
7121 gcall *call
7124 vops);
7130 }
7131 else
7132 {
7134 /* When combining two masks check if either of them is elsewhere
7135 combined with a loop mask, if that's the case we can mark that the
7136 new combined mask doesn't need to be combined with a loop mask. */
7140 {
7142 ncopies}))
7143 {
7149 }
7152 ncopies }))
7153 {
7159 }
7160 }
7169 /* Enter the combined value into the vector cond hash so we don't
7170 AND it with a loop mask again. */
7173 }
7176 {
7179 new_temp);
7184 }
7188 else
7190 }
7200 }
7202 /* A helper function to ensure data reference DR_INFO's base alignment. */
7204 static void
7206 {
7207 /* Alignment is only analyzed for the first element of a DR group,
7208 use that to look at base alignment we need to enforce. */
7215 {
7218 // We should only be able to increase the alignment of a base object if
7219 // we know what its new alignment should be at compile time.
7226 {
7229 }
7231 }
7232 }
7235 /* Function get_group_alias_ptr_type.
7237 Return the alias type for the group starting at FIRST_STMT_INFO. */
7239 static tree
7241 {
7247 {
7251 {
7256 }
7258 }
7260 }
7263 /* Function scan_operand_equal_p.
7265 Helper function for check_scan_store. Compare two references
7266 with .GOMP_SIMD_LANE bases. */
7268 static bool
7270 {
7275 {
7276 machine_mode mode;
7286 {
7292 {
7297 }
7298 }
7299 }
7306 {
7310 {
7313 {
7316 {
7322 {
7325 }
7326 }
7328 {
7331 }
7334 {
7338 }
7347 }
7351 }
7352 }
7354 }
7358 /* Normal permutation. */
7359 scan_store_kind_perm,
7361 /* Whole vector left shift permutation with zero init. */
7362 scan_store_kind_lshift_zero,
7364 /* Whole vector left shift permutation and VEC_COND_EXPR. */
7365 scan_store_kind_lshift_cond
7366 };
7368 /* Function check_scan_store.
7370 Verify if we can perform the needed permutations or whole vector shifts.
7371 Return -1 on failure, otherwise exact log2 of vectype's nunits.
7372 USE_WHOLE_VECTOR is a vector of enum scan_store_kind which operation
7373 to do at each step. */
7375 static int
7378 {
7390 {
7396 {
7399 }
7400 else
7401 {
7406 }
7409 {
7414 {
7418 /* Whole vector shifts shift in zeros, so if init is all zero
7419 constant, there is no need to do anything further. */
7423 {
7428 }
7429 }
7431 }
7433 {
7438 }
7439 }
7442 }
7445 /* Function check_scan_store.
7447 Check magic stores for #pragma omp scan {in,ex}clusive reductions. */
7449 static bool
7452 vect_memory_access_type memory_access_type)
7453 {
7456 tree ref_type;
7460 || mask
7472 {
7477 }
7479 /* We need to pattern match code built by OpenMP lowering and simplified
7480 by following optimizations into something we can handle.
7481 #pragma omp simd reduction(inscan,+:r)
7482 for (...)
7483 {
7484 r += something ();
7485 #pragma omp scan inclusive (r)
7486 use (r);
7487 }
7488 shall have body with:
7489 // Initialization for input phase, store the reduction initializer:
7490 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7491 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7492 D.2042[_21] = 0;
7493 // Actual input phase:
7494 ...
7495 r.0_5 = D.2042[_20];
7496 _6 = _4 + r.0_5;
7497 D.2042[_20] = _6;
7498 // Initialization for scan phase:
7499 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 2);
7500 _26 = D.2043[_25];
7501 _27 = D.2042[_25];
7502 _28 = _26 + _27;
7503 D.2043[_25] = _28;
7504 D.2042[_25] = _28;
7505 // Actual scan phase:
7506 ...
7507 r.1_8 = D.2042[_20];
7508 ...
7509 The "omp simd array" variable D.2042 holds the privatized copy used
7510 inside of the loop and D.2043 is another one that holds copies of
7511 the current original list item. The separate GOMP_SIMD_LANE ifn
7512 kinds are there in order to allow optimizing the initializer store
7513 and combiner sequence, e.g. if it is originally some C++ish user
7514 defined reduction, but allow the vectorizer to pattern recognize it
7515 and turn into the appropriate vectorized scan.
7517 For exclusive scan, this is slightly different:
7518 #pragma omp simd reduction(inscan,+:r)
7519 for (...)
7520 {
7521 use (r);
7522 #pragma omp scan exclusive (r)
7523 r += something ();
7524 }
7525 shall have body with:
7526 // Initialization for input phase, store the reduction initializer:
7527 _20 = .GOMP_SIMD_LANE (simduid.3_14(D), 0);
7528 _21 = .GOMP_SIMD_LANE (simduid.3_14(D), 1);
7529 D.2042[_21] = 0;
7530 // Actual input phase:
7531 ...
7532 r.0_5 = D.2042[_20];
7533 _6 = _4 + r.0_5;
7534 D.2042[_20] = _6;
7535 // Initialization for scan phase:
7536 _25 = .GOMP_SIMD_LANE (simduid.3_14(D), 3);
7537 _26 = D.2043[_25];
7538 D.2044[_25] = _26;
7539 _27 = D.2042[_25];
7540 _28 = _26 + _27;
7541 D.2043[_25] = _28;
7542 // Actual scan phase:
7543 ...
7544 r.1_8 = D.2044[_20];
7545 ... */
7548 {
7549 /* Match the D.2042[_21] = 0; store above. Just require that
7550 it is a constant or external definition store. */
7552 {
7553 fail_init:
7558 }
7568 /* These stores can be vectorized normally. */
7570 }
7573 {
7574 fail:
7579 }
7588 bool inscan_var_store
7592 {
7594 {
7595 use_operand_p use_p;
7596 imm_use_iterator iter;
7598 {
7605 || other_store_stmt
7609 }
7615 }
7616 }
7618 {
7619 use_operand_p use_p;
7620 imm_use_iterator iter;
7622 {
7629 }
7630 }
7631 else
7641 /* For pointer addition, we should use the normal plus for the vector
7642 operation. */
7644 {
7652 }
7680 {
7686 tree lrhs;
7689 else
7691 use_operand_p use_p;
7692 imm_use_iterator iter;
7694 {
7701 }
7702 }
7710 stmt_vec_info other_store_stmt_info
7723 {
7727 }
7738 {
7749 }
7768 {
7773 }
7781 /* The IL is as expected, now check if we can actually vectorize it.
7782 Inclusive scan:
7783 _26 = D.2043[_25];
7784 _27 = D.2042[_25];
7785 _28 = _26 + _27;
7786 D.2043[_25] = _28;
7787 D.2042[_25] = _28;
7788 should be vectorized as (where _40 is the vectorized rhs
7789 from the D.2042[_21] = 0; store):
7790 _30 = MEM <vector(8) int> [(int *)&D.2043];
7791 _31 = MEM <vector(8) int> [(int *)&D.2042];
7792 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7793 _33 = _31 + _32;
7794 // _33 = { _31[0], _31[0]+_31[1], _31[1]+_31[2], ..., _31[6]+_31[7] };
7795 _34 = VEC_PERM_EXPR <_40, _33, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7796 _35 = _33 + _34;
7797 // _35 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7798 // _31[1]+.._31[4], ... _31[4]+.._31[7] };
7799 _36 = VEC_PERM_EXPR <_40, _35, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7800 _37 = _35 + _36;
7801 // _37 = { _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7802 // _31[0]+.._31[4], ... _31[0]+.._31[7] };
7803 _38 = _30 + _37;
7804 _39 = VEC_PERM_EXPR <_38, _38, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7805 MEM <vector(8) int> [(int *)&D.2043] = _39;
7806 MEM <vector(8) int> [(int *)&D.2042] = _38;
7807 Exclusive scan:
7808 _26 = D.2043[_25];
7809 D.2044[_25] = _26;
7810 _27 = D.2042[_25];
7811 _28 = _26 + _27;
7812 D.2043[_25] = _28;
7813 should be vectorized as (where _40 is the vectorized rhs
7814 from the D.2042[_21] = 0; store):
7815 _30 = MEM <vector(8) int> [(int *)&D.2043];
7816 _31 = MEM <vector(8) int> [(int *)&D.2042];
7817 _32 = VEC_PERM_EXPR <_40, _31, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7818 _33 = VEC_PERM_EXPR <_40, _32, { 0, 8, 9, 10, 11, 12, 13, 14 }>;
7819 _34 = _32 + _33;
7820 // _34 = { 0, _31[0], _31[0]+_31[1], _31[1]+_31[2], _31[2]+_31[3],
7821 // _31[3]+_31[4], ... _31[5]+.._31[6] };
7822 _35 = VEC_PERM_EXPR <_40, _34, { 0, 1, 8, 9, 10, 11, 12, 13 }>;
7823 _36 = _34 + _35;
7824 // _36 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7825 // _31[1]+.._31[4], ... _31[3]+.._31[6] };
7826 _37 = VEC_PERM_EXPR <_40, _36, { 0, 1, 2, 3, 8, 9, 10, 11 }>;
7827 _38 = _36 + _37;
7828 // _38 = { 0, _31[0], _31[0]+_31[1], _31[0]+.._31[2], _31[0]+.._31[3],
7829 // _31[0]+.._31[4], ... _31[0]+.._31[6] };
7830 _39 = _30 + _38;
7831 _50 = _31 + _39;
7832 _51 = VEC_PERM_EXPR <_50, _50, { 7, 7, 7, 7, 7, 7, 7, 7 }>;
7833 MEM <vector(8) int> [(int *)&D.2044] = _39;
7834 MEM <vector(8) int> [(int *)&D.2042] = _51; */
7845 }
7848 /* Function vectorizable_scan_store.
7850 Helper of vectorizable_score, arguments like on vectorizable_store.
7851 Handle only the transformation, checking is done in check_scan_store. */
7853 static bool
7857 {
7872 bool inscan_var_store
7876 {
7877 use_operand_p use_p;
7878 imm_use_iterator iter;
7880 {
7886 }
7887 }
7908 {
7912 }
7927 {
7934 else
7935 {
7940 }
7944 {
7951 }
7952 else
7954 }
7967 /* The initialization is invariant. */
7972 {
7973 /* We want to lookup the vector operands of the reduction, not those
7974 of the store - for SLP we have to use the proper SLP node for the
7975 lookup, which should be the single child of the scan store. */
7978 /* ??? For SLP we do not key the def on 'rhs1' or 'rhs2' but get
7979 them in SLP child order. So we have to swap here with logic
7980 similar to above. */
7981 stmt_vec_info load
7989 }
7990 else
7994 {
8004 {
8008 dataref_offset);
8013 {
8016 }
8017 }
8021 {
8024 (zero_vec
8031 {
8034 }
8037 {
8038 /* Whole vector shift shifted in zero bits, but if *init
8039 is not initializer_zerop, we need to replace those elements
8040 with elements from vec_oprnd1. */
8054 }
8056 /* For exclusive scan, perform the perms[i] permutation once
8057 more. */
8061 {
8065 }
8074 }
8083 /* For exclusive scan, new_temp computed above is the exclusive scan
8084 prefix sum. Turn it into inclusive prefix sum for the broadcast
8085 of the last element into orig. */
8087 {
8093 }
8103 {
8106 dataref_offset);
8112 }
8113 }
8117 {
8123 dataref_offset);
8129 }
8131 }
8134 /* Function vectorizable_store.
8136 Check if STMT_INFO defines a non scalar data-ref (array/pointer/structure)
8137 that can be vectorized.
8138 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
8139 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
8140 Return true if STMT_INFO is vectorizable in this way. */
8142 static bool
8147 {
8148 tree data_ref;
8150 tree elem_type;
8153 machine_mode vec_mode;
8154 tree dummy;
8162 stmt_vec_info first_stmt_info;
8168 tree aggr_type;
8169 gather_scatter_info gs_info;
8170 poly_uint64 vf;
8171 vec_load_store_type vls_type;
8172 tree ref_type;
8181 /* Is vectorizable store? */
8186 {
8199 }
8200 else
8201 {
8212 mask_index = vect_slp_child_index_for_operand
8219 }
8221 /* Cannot have hybrid store SLP -- that would mean storing to the
8222 same location twice. */
8229 {
8232 }
8233 else
8236 /* Multiple types in SLP are handled by creating the appropriate number of
8237 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
8238 case of SLP. */
8241 else
8246 /* FORNOW. This restriction should be relaxed. */
8250 {
8255 }
8257 tree op;
8258 slp_tree op_node;
8269 vect_memory_access_type memory_access_type;
8272 poly_int64 poffset;
8273 internal_fn lanes_ifn;
8283 {
8288 }
8291 {
8293 {
8298 }
8302 {
8307 }
8311 {
8316 }
8317 }
8318 else
8319 {
8320 /* FORNOW. In some cases can vectorize even if data-type not supported
8321 (e.g. - array initialization with 0). */
8324 }
8331 {
8335 }
8336 else
8337 {
8341 }
8344 {
8346 memory_access_type))
8348 }
8352 {
8362 mask);
8366 || (mask
8368 mask_vectype))))
8369 {
8374 }
8385 /* As function vect_transform_stmt shows, for interleaving stores
8386 the whole chain is vectorized when the last store in the chain
8387 is reached, the other stores in the group are skipped. So we
8388 want to only cost the last one here, but it's not trivial to
8389 get the last, as it's equivalent to use the first one for
8390 costing, use the first one instead. */
8392 && !slp
8395 }
8398 else
8401 /* Transform. */
8406 {
8410 {
8421 "vect_model_store_cost: inside_cost = %d, "
8426 }
8429 }
8432 {
8433 /* FORNOW */
8435 || !loop
8439 {
8441 /* VEC_NUM is the number of vect stmts to be created for this
8442 group. */
8450 }
8451 else
8452 /* VEC_NUM is the number of vect stmts to be created for this
8453 group. */
8457 }
8458 else
8463 ncopies);
8465 /* Check if we need to update prologue cost for invariant,
8466 and update it accordingly if so. If it's not for
8467 interleaving store, we can just check vls_type; but if
8468 it's for interleaving store, need to check the def_type
8469 of the stored value since the current vls_type is just
8470 for first_stmt_info. */
8472 {
8477 {
8483 }
8488 };
8492 {
8494 gimple_stmt_iterator incr_gsi;
8497 tree offvar;
8498 tree ivstep;
8499 tree running_off;
8502 tree dr_offset;
8504 /* Checked by get_load_store_type. */
8511 stride_base
8512 = fold_build_pointer_plus
8519 /* For a store with loop-invariant (but other than power-of-2)
8520 stride (i.e. not a grouped access) like so:
8522 for (i = 0; i < n; i += stride)
8523 array[i] = ...;
8525 we generate a new induction variable and new stores from
8526 the components of the (vectorized) rhs:
8528 for (j = 0; ; j += VF*stride)
8529 vectemp = ...;
8530 tmp1 = vectemp[0];
8531 array[j] = tmp1;
8532 tmp2 = vectemp[1];
8533 array[j + stride] = tmp2;
8534 ...
8535 */
8542 {
8545 {
8547 dr_alignment_support dr_align
8549 mis_align);
8552 {
8559 }
8560 }
8562 {
8568 /* First check if vec_extract optab doesn't support extraction
8569 of vector elts directly. */
8571 machine_mode vmode;
8578 {
8579 /* Try to avoid emitting an extract of vector elements
8580 by performing the extracts using an integer type of the
8581 same size, extracting from a vector of those and then
8582 re-interpreting it as the original vector type if
8583 supported. */
8584 unsigned lsize
8587 /* If we can't construct such a vector fall back to
8588 element extracts from the original vector type and
8589 element size stores. */
8597 {
8602 }
8603 /* Else fall back to vector extraction anyway.
8604 Fewer stores are more important than avoiding spilling
8605 of the vector we extract from. Compared to the
8606 construction case in vectorizable_load no store-forwarding
8607 issue exists here for reasonable archs. */
8608 }
8609 }
8612 }
8615 {
8629 }
8634 /* For costing some adjacent vector stores, we'd like to cost with
8635 the total number of them once instead of cost each one by one. */
8638 {
8641 {
8643 {
8645 tree pos
8652 }
8653 }
8659 else
8662 unsigned HOST_WIDE_INT
8665 {
8667 {
8669 /* Pun the vector to extract from if necessary. */
8671 {
8673 tree cvt
8678 }
8679 }
8681 {
8683 {
8684 /* Only need vector extracting when there are more
8685 than one stores. */
8687 inside_cost
8690 /* Take a single lane vector type store as scalar
8691 store to avoid ICE like 110776. */
8694 n_adjacent_stores++;
8695 else
8696 inside_cost
8700 }
8704 /* Extract the i'th component. */
8712 GSI_SAME_STMT);
8720 /* And store it to *running_off. */
8727 {
8735 }
8738 {
8742 }
8743 }
8744 }
8749 }
8752 {
8759 "vect_model_store_cost: inside_cost = %d, "
8762 }
8765 }
8768 vec_loop_masks *loop_masks
8772 vec_loop_lens *loop_lens
8777 /* The vect_transform_stmt and vect_analyze_stmt will go here but there
8778 are some difference here. We cannot enable both the lens and masks
8779 during transform but it is allowed during analysis.
8780 Shouldn't go with length-based approach if fully masked. */
8782 /* The cost_vec is NULL during transfrom. */
8785 /* Targets with store-lane instructions must not require explicit
8786 realignment. vect_supportable_dr_alignment always returns either
8787 dr_aligned or dr_unaligned_supported for masked operations. */
8789 && !mask
8798 tree bump;
8801 {
8804 }
8806 {
8811 }
8812 else
8813 {
8816 else
8821 }
8826 /* In case the vectorization factor (VF) is bigger than the number
8827 of elements that we can fit in a vectype (nunits), we have to generate
8828 more than one vector stmt - i.e - we need to "unroll" the
8829 vector stmt by a factor VF/nunits. */
8831 /* In case of interleaving (non-unit grouped access):
8833 S1: &base + 2 = x2
8834 S2: &base = x0
8835 S3: &base + 1 = x1
8836 S4: &base + 3 = x3
8838 We create vectorized stores starting from base address (the access of the
8839 first stmt in the chain (S2 in the above example), when the last store stmt
8840 of the chain (S4) is reached:
8842 VS1: &base = vx2
8843 VS2: &base + vec_size*1 = vx0
8844 VS3: &base + vec_size*2 = vx1
8845 VS4: &base + vec_size*3 = vx3
8847 Then permutation statements are generated:
8849 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
8850 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
8851 ...
8853 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
8854 (the order of the data-refs in the output of vect_permute_store_chain
8855 corresponds to the order of scalar stmts in the interleaving chain - see
8856 the documentation of vect_permute_store_chain()).
8858 In case of both multiple types and interleaving, above vector stores and
8859 permutation stmts are created for every copy. The result vector stmts are
8860 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
8861 STMT_VINFO_RELATED_STMT for the next copies.
8862 */
8872 {
8874 /* Update all incoming store operand nodes, the general handling
8875 above only handles the mask and the first store operand node. */
8879 {
8884 }
8886 /* For costing some adjacent vector stores, we'd like to cost with
8887 the total number of them once instead of cost each one by one. */
8892 {
8895 {
8896 /* For interleaved stores we collect vectorized defs for all
8897 the stores in the group in DR_CHAIN. DR_CHAIN is then used
8898 as an input to vect_permute_store_chain(). */
8901 {
8902 /* Since gaps are not supported for interleaved stores,
8903 DR_GROUP_SIZE is the exact number of stmts in the
8904 chain. Therefore, NEXT_STMT_INFO can't be NULL_TREE. */
8909 {
8915 }
8917 }
8920 {
8922 {
8925 else
8928 mask_vectype);
8930 }
8932 dataref_ptr
8936 }
8937 }
8939 {
8941 /* DR_CHAIN is then used as an input to
8942 vect_permute_store_chain(). */
8944 {
8945 /* We should have caught mismatched types earlier. */
8949 {
8952 }
8953 }
8958 }
8961 {
8964 }
8966 /* Get an array into which we can store the individual vectors. */
8969 /* Invalidate the current contents of VEC_ARRAY. This should
8970 become an RTL clobber too, which prevents the vector registers
8971 from being upward-exposed. */
8974 /* Store the individual vectors into the array. */
8976 {
8978 {
8979 slp_tree child;
8982 else
8985 }
8986 else
8989 i);
8990 }
9003 {
9007 else
9009 signed char biasval
9013 {
9016 }
9017 }
9021 {
9022 /* Emit:
9023 MASK_LEN_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
9024 LEN, BIAS, VEC_ARRAY). */
9030 vec_array);
9031 }
9033 {
9034 /* Emit:
9035 MASK_STORE_LANES (DATAREF_PTR, ALIAS_PTR, VEC_MASK,
9036 VEC_ARRAY). */
9042 }
9043 else
9044 {
9045 /* Emit:
9046 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
9050 }
9055 /* Record that VEC_ARRAY is now dead. */
9061 }
9064 {
9071 "vect_model_store_cost: inside_cost = %d, "
9074 }
9077 }
9080 {
9085 {
9088 {
9093 {
9094 /* Since the store is not grouped, DR_GROUP_SIZE is 1, and
9095 DR_CHAIN is of size 1. */
9099 else
9103 {
9106 else
9108 ncopies,
9110 mask_vectype);
9111 }
9117 else
9118 dataref_ptr
9122 bump);
9123 }
9124 }
9126 {
9131 }
9135 {
9137 {
9141 /* We should have catched mismatched types earlier. */
9144 }
9150 {
9158 }
9161 {
9163 {
9165 inside_cost
9169 }
9176 {
9181 else
9183 signed char biasval
9187 {
9190 }
9191 }
9195 call = gimple_build_call_internal
9200 call = gimple_build_call_internal
9203 else
9210 }
9212 {
9213 /* The builtin decls path for scatter is legacy, x86 only. */
9215 && (!final_mask
9216 || SCALAR_INT_MODE_P
9219 {
9221 inside_cost
9225 }
9226 poly_uint64 offset_nunits
9229 {
9230 new_stmt = vect_build_one_scatter_store_call
9236 }
9238 {
9239 /* We have a offset vector with half the number of
9240 lanes but the builtins will store full vectype
9241 data from the lower lanes. */
9242 new_stmt = vect_build_one_scatter_store_call
9244 dataref_ptr,
9255 tree perm_mask
9259 perm_mask);
9265 {
9267 VEC_UNPACK_HI_EXPR,
9268 final_mask);
9269 final_mask = make_ssa_name
9274 }
9275 new_stmt = vect_build_one_scatter_store_call
9277 dataref_ptr,
9282 }
9284 {
9285 /* We have a offset vector with double the number of
9286 lanes. Select the low/high part accordingly. */
9289 {
9296 tree perm_mask = vect_gen_perm_mask_checked
9299 VEC_PERM_EXPR,
9300 vec_offset,
9301 vec_offset,
9302 perm_mask);
9307 }
9308 new_stmt = vect_build_one_scatter_store_call
9314 }
9315 else
9317 }
9318 else
9319 {
9320 /* Emulated scatter. */
9323 {
9325 /* For emulated scatter N offset vector element extracts
9326 (we assume the scalar scaling and ptr + offset add is
9327 consumed by the load). */
9328 inside_cost
9331 /* N scalar stores plus extracting the elements. */
9332 inside_cost
9335 inside_cost
9339 }
9342 unsigned HOST_WIDE_INT const_offset_nunits
9348 unsigned HOST_WIDE_INT elt_size
9350 /* We support offset vectors with more elements
9351 than the data vector for now. */
9352 unsigned HOST_WIDE_INT factor
9355 unsigned elt_offset
9362 {
9363 /* Compute the offsetted pointer. */
9366 tree idx
9372 tree ptr
9377 /* Extract the element to be stored. */
9378 tree elt
9385 tree ref
9391 }
9394 }
9395 }
9398 }
9405 "vect_model_store_cost: inside_cost = %d, "
9410 }
9418 /* For costing some adjacent vector stores, we'd like to cost with
9419 the total number of them once instead of cost each one by one. */
9424 {
9427 {
9429 {
9430 /* Get vectorized arguments for SLP_NODE. */
9436 }
9437 else
9438 {
9439 /* For interleaved stores we collect vectorized defs for all the
9440 stores in the group in DR_CHAIN. DR_CHAIN is then used as an
9441 input to vect_permute_store_chain().
9443 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN
9444 is of size 1. */
9447 {
9448 /* Since gaps are not supported for interleaved stores,
9449 DR_GROUP_SIZE is the exact number of stmts in the chain.
9450 Therefore, NEXT_STMT_INFO can't be NULL_TREE. In case
9451 that there is no interleaving, DR_GROUP_SIZE is 1,
9452 and only one iteration of the loop will be executed. */
9456 else
9457 {
9463 }
9465 }
9467 {
9470 mask_vectype);
9472 }
9473 }
9475 /* We should have catched mismatched types earlier. */
9479 bool simd_lane_access_p
9482 && simd_lane_access_p
9483 && !loop_masks
9490 {
9493 }
9495 dataref_ptr
9500 }
9502 {
9504 /* DR_CHAIN is then used as an input to vect_permute_store_chain().
9505 If the store is not grouped, DR_GROUP_SIZE is 1, and DR_CHAIN is
9506 of size 1. */
9508 {
9511 }
9516 else
9519 }
9523 {
9524 /* Permute. */
9527 {
9534 "vect_model_store_cost: "
9536 group_size);
9537 }
9538 else
9541 }
9545 {
9547 {
9551 /* For grouped stores vectorized defs are interleaved in
9552 vect_permute_store_chain(). */
9554 }
9557 {
9561 else
9562 {
9568 /* Generate the permute statement. */
9569 gimple *perm_stmt
9573 gsi);
9577 }
9578 }
9581 {
9582 n_adjacent_stores++;
9585 {
9589 }
9592 }
9608 /* Bump the vector pointer. */
9618 {
9621 }
9622 else
9627 misalign);
9630 /* Compute IFN when LOOP_LENS or final_mask valid. */
9635 {
9636 opt_machine_mode new_ovmode
9639 unsigned factor
9644 }
9646 {
9651 }
9654 {
9656 {
9657 /* Pass VF value to 'len' argument of
9658 MASK_LEN_STORE if LOOP_LENS is invalid. */
9660 }
9662 {
9663 /* Pass all ones value to 'mask' argument of
9664 MASK_LEN_STORE if final_mask is invalid. */
9667 }
9668 }
9670 {
9671 signed char biasval
9675 }
9677 /* Arguments are ready. Create the new vector stmt. */
9679 {
9682 /* Need conversion if it's wrapped with VnQI. */
9684 {
9685 tree new_vtype
9687 new_vmode);
9690 gassign *new_stmt
9694 }
9700 else
9707 }
9709 {
9711 gcall *call
9717 }
9718 else
9719 {
9720 data_ref
9722 dataref_offset ? dataref_offset
9725 ;
9726 else
9733 }
9741 }
9743 {
9747 }
9748 }
9751 {
9757 /* When vectorizing a store into the function result assign
9758 a penalty if the function returns in a multi-register location.
9759 In this case we assume we'll end up with having to spill the
9760 vector result and do piecewise loads as a conservative estimate. */
9766 {
9768 /* ??? Handle PARALLEL in some way. */
9770 {
9772 /* Assume that a single reg-reg move is possible and cheap,
9773 do not account for vector to gp register move cost. */
9775 {
9776 /* Spill. */
9777 prologue_cost
9780 /* Loads. */
9781 prologue_cost
9784 }
9785 }
9786 }
9789 "vect_model_store_cost: inside_cost = %d, "
9792 }
9795 }
9797 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
9798 VECTOR_CST mask. No checks are made that the target platform supports the
9799 mask, so callers may wish to test can_vec_perm_const_p separately, or use
9800 vect_gen_perm_mask_checked. */
9802 tree
9804 {
9805 tree mask_type;
9812 }
9814 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_const_p,
9815 i.e. that the target supports the pattern _for arbitrary input vectors_. */
9817 tree
9819 {
9823 }
9825 /* Given a vector variable X and Y, that was generated for the scalar
9826 STMT_INFO, generate instructions to permute the vector elements of X and Y
9827 using permutation mask MASK_VEC, insert them at *GSI and return the
9828 permuted vector variable. */
9830 static tree
9834 {
9842 else
9846 /* Generate the permute statement. */
9851 }
9853 /* Hoist the definitions of all SSA uses on STMT_INFO out of the loop LOOP,
9854 inserting them on the loops preheader edge. Returns true if we
9855 were successful in doing so (and thus STMT_INFO can be moved then),
9856 otherwise returns false. HOIST_P indicates if we want to hoist the
9857 definitions of all SSA uses, it would be false when we are costing. */
9859 static bool
9861 {
9862 ssa_op_iter i;
9863 use_operand_p use_p;
9867 {
9871 {
9872 /* Make sure we don't need to recurse. While we could do
9873 so in simple cases when there are more complex use webs
9874 we don't have an easy way to preserve stmt order to fulfil
9875 dependencies within them. */
9876 tree op2;
9877 ssa_op_iter i2;
9883 {
9888 }
9890 }
9891 }
9899 /* Instead of moving defs we copy them so we can zero their UID to not
9900 confuse dominance queries in the preheader. */
9903 {
9914 }
9917 }
9919 /* vectorizable_load.
9921 Check if STMT_INFO reads a non scalar data-ref (array/pointer/structure)
9922 that can be vectorized.
9923 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
9924 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
9925 Return true if STMT_INFO is vectorizable in this way. */
9927 static bool
9932 {
9933 tree scalar_dest;
9940 tree elem_type;
9941 /* Avoid false positive uninitialized warning, see PR110652. */
9943 machine_mode mode;
9944 tree dummy;
9951 poly_uint64 group_gap_adj;
9957 stmt_vec_info first_stmt_info;
9965 poly_uint64 vf;
9966 tree aggr_type;
9967 gather_scatter_info gs_info;
9968 tree ref_type;
9985 {
10000 }
10001 else
10002 {
10017 mask_index = vect_slp_child_index_for_operand
10023 }
10029 {
10033 }
10034 else
10037 /* Multiple types in SLP are handled by creating the appropriate number of
10038 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
10039 case of SLP. */
10042 else
10047 /* FORNOW. This restriction should be relaxed. */
10050 {
10055 }
10057 /* Invalidate assumptions made by dependence analysis when vectorization
10058 on the unrolled body effectively re-orders stmts. */
10063 {
10066 "cannot perform implicit CSE when unrolling "
10069 }
10074 /* FORNOW. In some cases can vectorize even if data-type not supported
10075 (e.g. - data copies). */
10077 {
10082 }
10084 /* Check if the load is a part of an interleaving chain. */
10086 {
10088 /* FORNOW */
10095 /* Refuse non-SLP vectorization of SLP-only groups. */
10097 {
10102 }
10104 /* Invalidate assumptions made by dependence analysis when vectorization
10105 on the unrolled body effectively re-orders stmts. */
10109 {
10112 "cannot perform implicit CSE when performing "
10115 }
10116 }
10117 else
10120 vect_memory_access_type memory_access_type;
10123 poly_int64 poffset;
10124 internal_fn lanes_ifn;
10131 /* ??? The following checks should really be part of
10132 get_group_load_store_type. */
10138 {
10142 {
10143 /* In BB vectorization we may not actually use a loaded vector
10144 accessing elements in excess of DR_GROUP_SIZE. */
10155 {
10158 "BB vectorization with gaps at the end of "
10161 }
10162 }
10168 {
10171 vect_location,
10174 }
10175 }
10180 {
10185 }
10188 {
10190 {
10196 }
10199 {
10204 }
10208 {
10213 }
10216 {
10221 }
10222 }
10227 {
10229 && mask
10231 mask_vectype))
10232 {
10237 }
10241 else
10249 mask);
10262 }
10267 else
10275 /* Transform. */
10281 {
10283 /* If we have versioned for aliasing or the loop doesn't
10284 have any data dependencies that would preclude this,
10285 then we are sure this is a loop invariant load and
10286 thus we can insert it on the preheader edge.
10287 TODO: hoist_defs_of_uses should ideally be computed
10288 once at analysis time, remembered and used in the
10289 transform time. */
10291 && !nested_in_vect_loop
10294 {
10295 enum vect_cost_model_location cost_loc
10300 cost_loc);
10305 "vect_model_load_cost: inside_cost = %d, "
10309 }
10311 {
10321 tree vuse;
10324 else
10330 }
10331 /* These copies are all equivalent. */
10335 else
10336 {
10341 }
10346 else
10347 {
10351 }
10353 }
10357 {
10358 gimple_stmt_iterator incr_gsi;
10360 tree offvar;
10361 tree ivstep;
10362 tree running_off;
10365 /* Checked by get_load_store_type. */
10368 tree dr_offset;
10375 {
10378 }
10379 else
10380 {
10383 }
10387 {
10390 }
10391 else
10392 {
10394 cst_offset
10397 first_stmt_info));
10400 }
10403 {
10411 /* For a load with loop-invariant (but other than power-of-2)
10412 stride (i.e. not a grouped access) like so:
10414 for (i = 0; i < n; i += stride)
10415 ... = array[i];
10417 we generate a new induction variable and new accesses to
10418 form a new vector (or vectors, depending on ncopies):
10420 for (j = 0; ; j += VF*stride)
10421 tmp1 = array[j];
10422 tmp2 = array[j + stride];
10423 ...
10424 vectemp = {tmp1, tmp2, ...}
10425 */
10439 }
10449 {
10451 /* Use the target vector type if the group size is a multiple
10452 of it. */
10454 {
10458 }
10459 /* Else use the biggest vector we can load the group without
10460 accessing excess elements. */
10462 {
10463 tree ptype;
10464 tree vtype
10468 {
10473 }
10474 }
10475 /* Else fall back to the default element-wise access. */
10477 }
10478 /* Load vector(1) scalar_type if it's 1 element-wise vectype. */
10483 {
10484 /* For SLP permutation support we need to load the whole group,
10485 not only the number of vector stmts the permutation result
10486 fits in. */
10488 {
10489 /* We don't yet generate SLP_TREE_LOAD_PERMUTATIONs for
10490 variable VF. */
10494 }
10495 else
10497 }
10499 unsigned HOST_WIDE_INT
10502 /* For costing some adjacent vector loads, we'd like to cost with
10503 the total number of them once instead of cost each one by one. */
10506 {
10511 {
10513 {
10514 /* For VMAT_ELEMENTWISE, just cost it as scalar_load to
10515 avoid ICE, see PR110776. */
10518 n_adjacent_loads++;
10519 else
10523 }
10537 {
10538 n_groups++;
10539 /* When doing SLP make sure to not load elements from
10540 the next vector iteration, those will not be accessed
10541 so just use the last element again. See PR107451. */
10543 {
10545 gimple *incr
10550 }
10552 }
10553 }
10556 {
10560 else
10561 {
10567 {
10568 new_stmt
10570 VIEW_CONVERT_EXPR,
10574 gsi);
10575 }
10576 }
10577 }
10580 {
10582 {
10585 else
10587 }
10588 else
10589 {
10593 }
10594 }
10595 }
10597 {
10600 {
10606 }
10607 else
10610 }
10613 {
10621 "vect_model_load_cost: inside_cost = %u, "
10623 inside_cost);
10624 }
10627 }
10635 {
10637 {
10640 }
10641 else
10642 {
10645 }
10646 /* For SLP vectorization we directly vectorize a subchain
10647 without permutation. */
10650 /* For BB vectorization always use the first stmt to base
10651 the data ref pointer on. */
10653 first_stmt_info_for_drptr
10656 /* Check if the chain of loads is already vectorized. */
10658 /* For SLP we would need to copy over SLP_TREE_VEC_DEFS.
10659 ??? But we can only do so if there is exactly one
10660 as we have no way to get at the rest. Leave the CSE
10661 opportunity alone.
10662 ??? With the group load eventually participating
10663 in multiple different permutations (having multiple
10664 slp nodes which refer to the same group) the CSE
10665 is even wrong code. See PR56270. */
10667 {
10670 }
10674 /* VEC_NUM is the number of vect stmts to be created for this group. */
10676 {
10678 /* If an SLP permutation is from N elements to N elements,
10679 and if one vector holds a whole number of N, we can load
10680 the inputs to the permutation in the same way as an
10681 unpermuted sequence. In other cases we need to load the
10682 whole group, not only the number of vector stmts the
10683 permutation result fits in. */
10686 /* We do not support grouped accesses in a nested loop,
10687 instead the access is contiguous but it might be
10688 permuted. No gap adjustment is needed though. */
10693 {
10694 /* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
10695 variable VF; see vect_transform_slp_perm_load. */
10700 }
10701 else
10702 {
10704 group_gap_adj
10706 }
10707 }
10708 else
10712 }
10713 else
10714 {
10722 }
10725 vec_loop_masks *loop_masks
10729 vec_loop_lens *loop_lens
10734 /* The vect_transform_stmt and vect_analyze_stmt will go here but there
10735 are some difference here. We cannot enable both the lens and masks
10736 during transform but it is allowed during analysis.
10737 Shouldn't go with length-based approach if fully masked. */
10739 /* The cost_vec is NULL during transfrom. */
10742 /* Targets with store-lane instructions must not require explicit
10743 realignment. vect_supportable_dr_alignment always returns either
10744 dr_aligned or dr_unaligned_supported for masked operations. */
10746 && !mask
10751 /* In case the vectorization factor (VF) is bigger than the number
10752 of elements that we can fit in a vectype (nunits), we have to generate
10753 more than one vector stmt - i.e - we need to "unroll" the
10754 vector stmt by a factor VF/nunits. In doing so, we record a pointer
10755 from one copy of the vector stmt to the next, in the field
10756 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
10757 stages to find the correct vector defs to be used when vectorizing
10758 stmts that use the defs of the current stmt. The example below
10759 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
10760 need to create 4 vectorized stmts):
10762 before vectorization:
10763 RELATED_STMT VEC_STMT
10764 S1: x = memref - -
10765 S2: z = x + 1 - -
10767 step 1: vectorize stmt S1:
10768 We first create the vector stmt VS1_0, and, as usual, record a
10769 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
10770 Next, we create the vector stmt VS1_1, and record a pointer to
10771 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
10772 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
10773 stmts and pointers:
10774 RELATED_STMT VEC_STMT
10775 VS1_0: vx0 = memref0 VS1_1 -
10776 VS1_1: vx1 = memref1 VS1_2 -
10777 VS1_2: vx2 = memref2 VS1_3 -
10778 VS1_3: vx3 = memref3 - -
10779 S1: x = load - VS1_0
10780 S2: z = x + 1 - -
10781 */
10783 /* In case of interleaving (non-unit grouped access):
10785 S1: x2 = &base + 2
10786 S2: x0 = &base
10787 S3: x1 = &base + 1
10788 S4: x3 = &base + 3
10790 Vectorized loads are created in the order of memory accesses
10791 starting from the access of the first stmt of the chain:
10793 VS1: vx0 = &base
10794 VS2: vx1 = &base + vec_size*1
10795 VS3: vx3 = &base + vec_size*2
10796 VS4: vx4 = &base + vec_size*3
10798 Then permutation statements are generated:
10800 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
10801 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
10802 ...
10804 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
10805 (the order of the data-refs in the output of vect_permute_load_chain
10806 corresponds to the order of scalar stmts in the interleaving chain - see
10807 the documentation of vect_permute_load_chain()).
10808 The generation of permutation stmts and recording them in
10809 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
10811 In case of both multiple types and interleaving, the vector loads and
10812 permutation stmts above are created for every copy. The result vector
10813 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
10814 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
10816 /* If the data reference is aligned (dr_aligned) or potentially unaligned
10817 on a target that supports unaligned accesses (dr_unaligned_supported)
10818 we generate the following code:
10819 p = initial_addr;
10820 indx = 0;
10821 loop {
10822 p = p + indx * vectype_size;
10823 vec_dest = *(p);
10824 indx = indx + 1;
10825 }
10827 Otherwise, the data reference is potentially unaligned on a target that
10828 does not support unaligned accesses (dr_explicit_realign_optimized) -
10829 then generate the following code, in which the data in each iteration is
10830 obtained by two vector loads, one from the previous iteration, and one
10831 from the current iteration:
10832 p1 = initial_addr;
10833 msq_init = *(floor(p1))
10834 p2 = initial_addr + VS - 1;
10835 realignment_token = call target_builtin;
10836 indx = 0;
10837 loop {
10838 p2 = p2 + indx * vectype_size
10839 lsq = *(floor(p2))
10840 vec_dest = realign_load (msq, lsq, realignment_token)
10841 indx = indx + 1;
10842 msq = lsq;
10843 } */
10845 /* If the misalignment remains the same throughout the execution of the
10846 loop, we can create the init_addr and permutation mask at the loop
10847 preheader. Otherwise, it needs to be created inside the loop.
10848 This can only occur when vectorizing memory accesses in the inner-loop
10849 nested within an outer-loop that is being vectorized. */
10854 {
10857 }
10859 bool diff_first_stmt_info
10866 {
10867 /* If we have different first_stmt_info, we can't set up realignment
10868 here, since we can't guarantee first_stmt_info DR has been
10869 initialized yet, use first_stmt_info_for_drptr DR by bumping the
10870 distance from first_stmt_info DR instead as below. */
10872 {
10879 {
10882 size_one_node);
10884 }
10885 }
10886 }
10887 else
10891 offset = (offset
10895 tree bump;
10898 {
10901 }
10903 {
10908 }
10909 else
10910 {
10913 else
10918 }
10923 {
10927 else
10930 }
10934 {
10939 /* For costing some adjacent vector loads, we'd like to cost with
10940 the total number of them once instead of cost each one by one. */
10945 {
10947 {
10948 /* An IFN_LOAD_LANES will load all its vector results,
10949 regardless of which ones we actually need. Account
10950 for the cost of unused results. */
10952 {
10955 do
10956 {
10959 }
10962 {
10965 "vect_model_load_cost: %d "
10967 gaps);
10969 alignment_support_scheme,
10973 }
10974 }
10975 n_adjacent_loads++;
10977 }
10979 /* 1. Create the vector or array pointer update chain. */
10981 dataref_ptr
10985 else
10986 {
10990 }
11007 {
11011 else
11013 signed char biasval
11017 {
11020 }
11021 }
11025 {
11026 /* Emit:
11027 VEC_ARRAY = MASK_LEN_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
11028 VEC_MASK, LEN, BIAS). */
11034 }
11036 {
11037 /* Emit:
11038 VEC_ARRAY = MASK_LOAD_LANES (DATAREF_PTR, ALIAS_PTR,
11039 VEC_MASK). */
11044 final_mask);
11045 }
11046 else
11047 {
11048 /* Emit:
11049 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
11052 }
11059 /* Extract each vector into an SSA_NAME. */
11061 {
11066 else
11068 }
11071 /* Record the mapping between SSA_NAMEs and statements. */
11074 /* Record that VEC_ARRAY is now dead. */
11082 }
11085 {
11093 "vect_model_load_cost: inside_cost = %u, "
11096 }
11099 }
11102 {
11109 {
11110 /* 1. Create the vector or array pointer update chain. */
11112 {
11117 else
11118 dataref_ptr
11122 }
11124 {
11129 }
11133 {
11138 {
11142 final_mask
11153 }
11155 /* 2. Create the vector-load in the loop. */
11158 {
11160 {
11162 inside_cost
11166 }
11173 {
11175 final_len
11179 else
11180 final_len
11183 signed char biasval
11187 {
11190 }
11191 }
11195 call
11204 else
11211 }
11213 {
11214 /* The builtin decls path for gather is legacy, x86 only. */
11217 {
11219 inside_cost
11223 }
11224 poly_uint64 offset_nunits
11227 {
11228 new_stmt = vect_build_one_gather_load_call
11231 final_mask);
11233 }
11235 {
11236 /* We have a offset vector with half the number of
11237 lanes but the builtins will produce full vectype
11238 data with just the lower lanes filled. */
11239 new_stmt = vect_build_one_gather_load_call
11242 final_mask);
11248 /* now put upper half of final_mask in final_mask low. */
11250 && !SCALAR_INT_MODE_P
11252 {
11259 tree perm_mask = vect_gen_perm_mask_checked
11262 VEC_PERM_EXPR,
11263 final_mask,
11264 final_mask,
11265 perm_mask);
11270 }
11272 {
11274 VEC_UNPACK_HI_EXPR,
11275 final_mask);
11276 final_mask = make_ssa_name
11281 }
11283 new_stmt = vect_build_one_gather_load_call
11285 dataref_ptr,
11287 final_mask);
11293 /* compose low + high. */
11300 tree perm_mask
11303 VEC_PERM_EXPR,
11306 }
11308 {
11309 /* We have a offset vector with double the number of
11310 lanes. Select the low/high part accordingly. */
11313 {
11320 tree perm_mask = vect_gen_perm_mask_checked
11323 VEC_PERM_EXPR,
11324 vec_offset,
11325 vec_offset,
11326 perm_mask);
11331 }
11332 new_stmt = vect_build_one_gather_load_call
11336 }
11337 else
11339 }
11340 else
11341 {
11342 /* Emulated gather-scatter. */
11346 {
11347 /* For emulated gathers N offset vector element
11348 offset add is consumed by the load). */
11352 /* N scalar loads plus gathering them into a
11353 vector. */
11354 inside_cost
11357 inside_cost
11361 }
11362 unsigned HOST_WIDE_INT const_offset_nunits
11368 /* We support offset vectors with more elements
11369 than the data vector for now. */
11370 unsigned HOST_WIDE_INT factor
11373 unsigned elt_offset
11380 {
11383 tree idx
11388 scale);
11400 }
11405 }
11408 /* DATA_REF is null if we've already built the statement. */
11410 {
11413 }
11418 /* Store vector loads in the corresponding SLP_NODE. */
11421 }
11425 }
11432 "vect_model_load_cost: inside_cost = %u, "
11436 }
11440 /* For costing some adjacent vector loads, we'd like to cost with
11441 the total number of them once instead of cost each one by one. */
11444 {
11445 /* 1. Create the vector or array pointer update chain. */
11447 {
11448 bool simd_lane_access_p
11459 {
11462 }
11464 {
11465 dataref_ptr
11469 bump);
11470 /* Adjust the pointer by the difference to first_stmt. */
11471 data_reference_p ptrdr
11473 tree diff
11481 {
11485 alignment_support_scheme,
11488 }
11489 }
11490 else
11491 dataref_ptr
11493 at_loop,
11496 }
11498 {
11502 bump);
11503 else
11506 }
11513 {
11518 {
11532 }
11534 /* 2. Create the vector-load in the loop. */
11536 {
11539 {
11549 {
11550 align
11553 }
11554 else
11559 misalign);
11562 /* Compute IFN when LOOP_LENS or final_mask valid. */
11567 {
11568 opt_machine_mode new_ovmode
11571 unsigned factor
11576 }
11578 {
11583 }
11586 {
11588 {
11589 /* Pass VF value to 'len' argument of
11590 MASK_LEN_LOAD if LOOP_LENS is invalid. */
11592 }
11594 {
11595 /* Pass all ones value to 'mask' argument of
11596 MASK_LEN_LOAD if final_mask is invalid. */
11599 }
11600 }
11602 {
11603 signed char biasval
11607 }
11610 {
11617 bias);
11618 else
11626 /* Need conversion if it's wrapped with VnQI. */
11628 {
11631 tree var
11635 gsi);
11639 }
11640 }
11642 {
11646 final_mask);
11650 }
11651 else
11652 {
11656 unsigned int vect_align
11658 /* Try to use a single smaller load when we are about
11659 to load excess elements compared to the unrolled
11660 scalar loop. */
11663 {
11668 /* DR will be unused. */
11671 tree_to_poly_uint64
11673 /* Aligned access to excess elements is OK if
11674 at least one element is accessed in the
11675 scalar loop. */
11676 ;
11679 * vect_get_scalar_dr_size
11681 /* Aligned access to the gap area when there's
11682 at least one element in it is OK. */
11683 ;
11684 else
11685 {
11686 /* remain should now be > 0 and < nunits. */
11690 {
11691 tree ptype;
11692 new_vtype
11694 num,
11698 }
11699 /* Else use multiple loads or a masked load? */
11700 /* For loop vectorization we now should have
11701 an alternate type or LOOP_VINFO_PEELING_FOR_GAPS
11702 set. */
11705 || LOOP_VINFO_PEELING_FOR_GAPS
11707 /* But still reduce the access size to the next
11708 required power-of-two so peeling a single
11709 scalar iteration is sufficient. */
11712 {
11713 unsigned HOST_WIDE_INT cpart_size
11718 {
11719 tree ptype;
11720 new_vtype
11722 num,
11726 }
11727 }
11728 }
11729 }
11730 tree offset
11734 ;
11737 {
11738 poly_uint64 gap_offset
11743 }
11745 {
11746 data_ref
11749 ;
11750 else
11754 }
11758 {
11764 gsi);
11767 /* We've computed 'num' above to statically two
11768 or via constant_multiple_p. */
11769 unsigned num
11772 tree_to_poly_uint64
11777 {
11782 }
11783 else
11784 {
11789 }
11794 else
11795 {
11802 vec_dest,
11804 }
11805 }
11806 }
11808 }
11810 {
11820 dr_explicit_realign,
11825 else
11827 // For explicit realign the target alignment should be
11828 // known at compile time.
11829 unsigned HOST_WIDE_INT align
11836 data_ref
11850 bump);
11856 else
11860 data_ref
11863 }
11865 {
11870 else
11872 // We should only be doing this if we know the target
11873 // alignment at compile time.
11874 unsigned HOST_WIDE_INT align
11884 }
11887 }
11889 /* One common place to cost the above vect load for different
11890 alignment support schemes. */
11892 {
11893 /* For VMAT_CONTIGUOUS_PERMUTE if it's grouped load, we
11894 only need to take care of the first stmt, whose
11895 stmt_info is first_stmt_info, vec_num iterating on it
11896 will cover the cost for the remaining, it's consistent
11897 with transforming. For the prologue cost for realign,
11898 we only need to count it once for the whole group. */
11905 {
11906 /* Leave realign cases alone to keep them simple. */
11914 else
11915 n_adjacent_loads++;
11916 }
11917 }
11918 else
11919 {
11921 /* DATA_REF is null if we've already built the statement. */
11923 {
11926 }
11930 }
11932 /* 3. Handle explicit realignment if necessary/supported.
11933 Create in loop:
11934 vec_dest = realign_load (msq, lsq, realignment_token) */
11938 {
11950 {
11954 UNKNOWN_LOCATION);
11956 }
11957 }
11960 {
11964 else
11965 {
11970 }
11971 }
11973 /* Collect vector loads and later create their permutation in
11974 vect_transform_grouped_load (). */
11978 /* Store vector loads in the corresponding SLP_NODE. */
11982 /* With SLP permutation we load the gaps as well, without
11983 we need to skip the gaps after we manage to fully load
11984 all elements. group_gap_adj is DR_GROUP_SIZE here. */
11988 && !slp_perm
11990 {
11991 poly_wide_int bump_val
12000 }
12001 }
12002 /* Bump the vector pointer to account for a gap or for excess
12003 elements loaded for a permuted SLP load. */
12007 {
12008 poly_wide_int bump_val
12015 }
12021 {
12023 /* For SLP we know we've seen all possible uses of dr_chain so
12024 direct vect_transform_slp_perm_load to DCE the unused parts.
12025 ??? This is a hack to prevent compile-time issues as seen
12026 in PR101120 and friends. */
12028 {
12033 }
12034 else
12035 {
12040 }
12041 }
12042 else
12043 {
12045 {
12047 /* We assume that the cost of a single load-lanes instruction
12048 is equivalent to the cost of DR_GROUP_SIZE separate loads.
12049 If a grouped access is instead being provided by a
12050 load-and-permute operation, include the cost of the
12051 permutes. */
12053 {
12054 /* Uses an even and odd extract operations or shuffle
12055 operations for each needed permute. */
12063 "vect_model_load_cost:"
12065 group_size);
12066 }
12068 {
12072 }
12073 }
12076 }
12078 }
12083 {
12094 "vect_model_load_cost: inside_cost = %u, "
12097 }
12100 }
12102 /* Function vect_is_simple_cond.
12104 Input:
12105 LOOP - the loop that is being vectorized.
12106 COND - Condition that is checked for simple use.
12108 Output:
12109 *COMP_VECTYPE - the vector type for the comparison.
12110 *DTS - The def types for the arguments of the comparison
12112 Returns whether a COND can be vectorized. Checks whether
12113 condition operands are supportable using vec_is_simple_use. */
12115 static bool
12119 {
12122 slp_tree slp_op;
12124 /* Mask case. */
12127 {
12130 || !*comp_vectype
12134 }
12143 {
12147 }
12151 else
12155 {
12159 }
12163 else
12172 /* Invariant comparison. */
12174 {
12178 else
12179 {
12180 /* If we can widen the comparison to match vectype do so. */
12182 && !slp_node
12185 scalar_type = build_nonstandard_integer_type
12188 slp_node);
12189 }
12190 }
12193 }
12195 /* vectorizable_condition.
12197 Check if STMT_INFO is conditional modify expression that can be vectorized.
12198 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12199 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
12200 at GSI.
12202 When STMT_INFO is vectorized as a nested cycle, for_reduction is true.
12204 Return true if STMT_INFO is vectorizable in this way. */
12206 static bool
12211 {
12219 tree vec_compare;
12220 tree new_temp;
12235 tree vec_cmp_type;
12241 /* Is vectorizable conditional operation? */
12253 bool for_reduction
12256 {
12264 }
12265 else
12266 {
12269 }
12275 {
12278 }
12279 else
12280 {
12283 }
12319 {
12322 }
12324 /* For conditional reductions, the "then" value needs to be the candidate
12325 value calculated by this iteration while the "else" value needs to be
12326 the result carried over from previous iterations. If the COND_EXPR
12327 is the other way around, we need to swap it. */
12330 {
12333 else
12334 {
12339 else
12340 {
12342 /* Make sure we don't accidentally use the old condition. */
12344 }
12345 }
12346 /* ??? The vectorized operand query below doesn't allow swapping
12347 this way for SLP. */
12351 }
12354 {
12355 /* Boolean values may have another representation in vectors
12356 and therefore we prefer bit operations over comparison for
12357 them (which also works for scalar masks). We store opcodes
12358 to use in bitop1 and bitop2. Statement is vectorized as
12359 BITOP2 (rhs1 BITOP1 rhs2) or rhs1 BITOP2 (BITOP1 rhs2)
12360 depending on bitop1 and bitop2 arity. */
12362 {
12390 }
12392 }
12397 {
12402 }
12405 {
12407 {
12409 optab optab;
12416 {
12418 optab_default);
12421 }
12422 }
12426 /* Count one reduction-like operation per vector. */
12429 && (masked
12431 cond_code)
12433 ERROR_MARK))))
12437 && (!vect_maybe_update_slp_op_vectype
12440 && !vect_maybe_update_slp_op_vectype
12444 {
12449 }
12453 {
12455 {
12461 else
12465 }
12466 /* Extra inactive lanes should be safe for vect_nested_cycle. */
12468 {
12471 "conditional reduction prevents the use"
12474 }
12475 }
12481 }
12483 /* Transform. */
12485 /* Handle def. */
12492 /* See whether another part of the vectorized code applies a loop
12493 mask to the condition, or to its inverse. */
12498 {
12501 }
12503 {
12506 else
12507 {
12511 else
12512 {
12517 {
12521 }
12522 else
12523 {
12524 /* Try the inverse of the current mask. We check if the
12525 inverse mask is live and if so we generate a negate of
12526 the current mask such that we still honor NaNs. */
12530 {
12535 }
12536 }
12537 }
12538 }
12539 }
12541 /* Handle cond expr. */
12546 reduction_type != EXTRACT_LAST_REDUCTION
12548 else
12553 reduction_type != EXTRACT_LAST_REDUCTION
12559 /* Arguments are ready. Create the new vector stmt. */
12561 {
12571 else
12572 {
12575 {
12580 }
12581 else
12582 {
12587 vec_cond_rhs);
12588 else
12589 new_stmt
12591 vec_cond_rhs);
12597 {
12598 /* Instead of doing ~x ? y : z do x ? z : y. */
12601 }
12602 else
12603 {
12606 new_stmt
12608 else
12609 new_stmt
12614 }
12615 }
12616 }
12618 /* If we decided to apply a loop mask to the result of the vector
12619 comparison, AND the comparison with the mask now. Later passes
12620 should then be able to reuse the AND results between mulitple
12621 vector statements.
12623 For example:
12624 for (int i = 0; i < 100; ++i)
12625 x[i] = y[i] ? z[i] : 10;
12627 results in following optimized GIMPLE:
12629 mask__35.8_43 = vect__4.7_41 != { 0, ... };
12630 vec_mask_and_46 = loop_mask_40 & mask__35.8_43;
12631 _19 = &MEM[base: z_12(D), index: ivtmp_56, step: 4, offset: 0B];
12632 vect_iftmp.11_47 = .MASK_LOAD (_19, 4B, vec_mask_and_46);
12633 vect_iftmp.12_52 = VEC_COND_EXPR <vec_mask_and_46,
12634 vect_iftmp.11_47, { 10, ... }>;
12636 instead of using a masked and unmasked forms of
12637 vec != { 0, ... } (masked in the MASK_LOAD,
12638 unmasked in the VEC_COND_EXPR). */
12640 /* Force vec_compare to be an SSA_NAME rather than a comparison,
12641 in cases where that's necessary. */
12645 {
12647 {
12650 vec_compare);
12653 }
12656 {
12659 BIT_NOT_EXPR,
12660 vec_compare);
12663 }
12667 {
12669 {
12672 signed char biasval
12675 }
12676 else
12677 {
12680 }
12681 }
12683 {
12684 tree loop_mask
12688 gassign *g
12690 loop_mask);
12693 }
12694 }
12698 {
12704 new_stmt = gimple_build_call_internal
12707 else
12708 new_stmt = gimple_build_call_internal
12710 vec_then_clause);
12714 {
12717 }
12720 else
12721 {
12722 /* In this case we're moving the definition to later in the
12723 block. That doesn't matter because the only uses of the
12724 lhs are in phi statements. */
12728 }
12729 }
12730 else
12731 {
12736 }
12739 else
12741 }
12752 }
12754 /* Helper of vectorizable_comparison.
12756 Check if STMT_INFO is comparison expression CODE that can be vectorized.
12757 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12758 comparison, put it in VEC_STMT, and insert it at GSI.
12760 Return true if STMT_INFO is vectorizable in this way. */
12762 static bool
12767 {
12771 tree new_temp;
12775 poly_uint64 nunits;
12782 tree mask_type;
12796 else
12820 /* Invariant comparison. */
12822 {
12826 }
12830 /* Can't compare mask and non-mask types. */
12835 /* Boolean values may have another representation in vectors
12836 and therefore we prefer bit operations over comparison for
12837 them (which also works for scalar masks). We store opcodes
12838 to use in bitop1 and bitop2. Statement is vectorized as
12839 BITOP2 (rhs1 BITOP1 rhs2) or
12840 rhs1 BITOP2 (BITOP1 rhs2)
12841 depending on bitop1 and bitop2 arity. */
12844 {
12846 {
12849 }
12851 {
12854 }
12856 {
12860 }
12862 {
12866 }
12867 else
12868 {
12872 }
12873 }
12876 {
12878 {
12881 }
12882 else
12883 {
12885 optab optab;
12892 {
12896 }
12897 }
12899 /* Put types on constant and invariant SLP children. */
12903 {
12908 }
12914 }
12916 /* Transform. */
12918 /* Handle def. */
12929 /* Arguments are ready. Create the new vector stmt. */
12931 {
12937 else
12940 {
12944 }
12945 else
12946 {
12949 else
12951 vec_rhs2);
12954 {
12958 else
12960 new_temp);
12962 }
12963 }
12966 else
12968 }
12977 }
12979 /* vectorizable_comparison.
12981 Check if STMT_INFO is comparison expression that can be vectorized.
12982 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
12983 comparison, put it in VEC_STMT, and insert it at GSI.
12985 Return true if STMT_INFO is vectorizable in this way. */
12987 static bool
12992 {
13015 }
13017 /* Check to see if the current early break given in STMT_INFO is valid for
13018 vectorization. */
13020 bool
13024 {
13041 slp_tree slp_op0;
13042 tree op0;
13045 /* Early break gcond kind SLP trees can be root only and have no children,
13046 for instance in the case where the argument is an external. If that's
13047 the case there is no operand to analyse use of. */
13051 {
13056 }
13058 /* For SLP we don't want to use the type of the operands of the SLP node, when
13059 vectorizing using SLP slp_node will be the children of the gcond and we
13060 want to use the type of the direct children which since the gcond is root
13061 will be the current node, rather than a child node as vect_is_simple_use
13062 assumes. */
13073 {
13076 }
13077 else
13078 {
13081 }
13088 /* Now build the new conditional. Pattern gimple_conds get dropped during
13089 codegen so we must replace the original insn. */
13092 /* When vectorizing we assume that if the branch edge is taken that we're
13093 exiting the loop. This is not however always the case as the compiler will
13094 rewrite conditions to always be a comparison against 0. To do this it
13095 sometimes flips the edges. This is fine for scalar, but for vector we
13096 then have to flip the test, as we're still assuming that if you take the
13097 branch edge that we found the exit condition. i.e. we need to know whether
13098 we are generating a `forall` or an `exist` condition. */
13109 {
13114 }
13116 /* Analyze only. */
13118 {
13120 {
13123 "can't vectorize early exit because the "
13124 "target doesn't support flag setting vector "
13127 }
13131 {
13134 "can't vectorize early exit because the "
13135 "target does not support boolean vector %s "
13138 vectype);
13140 }
13147 {
13149 OPTIMIZE_FOR_SPEED))
13152 else
13155 }
13158 }
13160 /* Tranform. */
13168 /* For SLP we don't do codegen of the body starting from the gcond, the gconds are
13169 roots and so by the time we get to them we have already codegened the SLP tree
13170 and so we shouldn't try to do so again. The arguments have already been
13171 vectorized. It's not very clean to do this here, But the masking code below is
13172 complex and this keeps it all in one place to ease fixes and backports. Once we
13173 drop the non-SLP loop vect or split vectorizable_* this can be simplified. */
13175 {
13179 }
13189 else
13190 {
13195 }
13197 /* Determine if we need to reduce the final value. */
13199 {
13200 /* We build the reductions in a way to maintain as much parallelism as
13201 possible. */
13204 /* Mask the statements as we queue them up. Normally we loop over
13205 vec_num, but since we inspect the exact results of vectorization
13206 we don't need to and instead can just use the stmts themselves. */
13209 {
13210 tree stmt_mask
13213 stmt_mask
13217 }
13220 {
13226 }
13227 else
13231 {
13239 }
13240 }
13241 else
13242 {
13245 {
13246 tree mask
13250 }
13254 }
13263 else
13270 }
13272 /* If SLP_NODE is nonnull, return true if vectorizable_live_operation
13273 can handle all live statements in the node. Otherwise return true
13274 if STMT_INFO is not live or if vectorizable_live_operation can handle it.
13275 VEC_STMT_P is as for vectorizable_live_operation. */
13277 static bool
13282 {
13285 {
13286 stmt_vec_info slp_stmt_info;
13289 {
13292 || (loop_vinfo
13300 }
13301 }
13311 }
13313 /* Make sure the statement is vectorizable. */
13315 opt_result
13320 {
13324 gimple_seq pattern_def_seq;
13332 "not vectorized:"
13339 {
13340 gimple_stmt_iterator si;
13343 {
13344 stmt_vec_info pattern_def_stmt_info
13348 {
13349 /* Analyze def stmt of STMT if it's a pattern stmt. */
13355 opt_result res
13358 cost_vec);
13361 }
13362 }
13363 }
13365 /* Skip stmts that do not need to be vectorized. In loops this is expected
13366 to include:
13367 - the COND_EXPR which is the loop exit condition
13368 - any LABEL_EXPRs in the loop
13369 - computations that are used only for array indexing or loop control.
13370 In basic blocks we only analyze statements that are a part of some SLP
13371 instance, therefore, all the statements are relevant.
13373 Pattern statement needs to be analyzed instead of the original statement
13374 if the original statement is not relevant. Otherwise, we analyze both
13375 statements. In basic blocks we are called from some SLP instance
13376 traversal, don't analyze pattern stmts instead, the pattern stmts
13377 already will be part of SLP instance. */
13382 {
13384 && pattern_stmt_info
13387 {
13388 /* Analyze PATTERN_STMT instead of the original stmt. */
13394 }
13395 else
13396 {
13402 "not vectorized:"
13403 " irrelevant stmt as SLP node %p "
13407 }
13408 }
13411 && pattern_stmt_info
13414 {
13415 /* Analyze PATTERN_STMT too. */
13421 opt_result res
13426 }
13429 {
13458 }
13465 {
13471 }
13474 {
13479 }
13481 /* When we arrive here with a non-SLP statement and we are supposed
13482 to use SLP for everything fail vectorization. */
13491 /* Prefer vectorizable_call over vectorizable_simd_clone_call so
13492 -mveclibabi= takes preference over library functions with
13493 the simd attribute. */
13496 cost_vec)
13515 cost_vec)
13521 cost_vec));
13522 else
13523 {
13529 cost_vec)
13535 cost_vec)
13543 cost_vec)
13546 cost_vec));
13548 }
13555 "not vectorized:"
13559 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
13560 need extra handling, except for vectorizable reductions. */
13568 "not vectorized:"
13573 }
13576 /* Function vect_transform_stmt.
13578 Create a vectorized stmt to replace STMT_INFO, and insert it at GSI. */
13580 bool
13584 {
13596 {
13608 NULL);
13620 NULL);
13632 NULL);
13638 && !slp_node
13641 /* In case of interleaving, the whole chain is vectorized when the
13642 last store in the chain is reached. Store stmts before the last
13643 one are skipped, and there vec_stmt_info shouldn't be freed
13644 meanwhile. */
13645 ;
13646 else
13647 {
13652 }
13714 {
13719 }
13721 }
13727 {
13728 /* Handle stmts whose DEF is used outside the loop-nest that is
13729 being vectorized. */
13733 }
13739 }
13742 /* Remove a group of stores (for SLP or interleaving), free their
13743 stmt_vec_info. */
13745 void
13747 {
13751 {
13754 /* Free the attached stmt_vec_info and remove the stmt. */
13757 }
13758 }
13760 /* If NUNITS is nonzero, return a vector type that contains NUNITS
13761 elements of type SCALAR_TYPE, or null if the target doesn't support
13762 such a type.
13764 If NUNITS is zero, return a vector type that contains elements of
13765 type SCALAR_TYPE, choosing whichever vector size the target prefers.
13767 If PREVAILING_MODE is VOIDmode, we have not yet chosen a vector mode
13768 for this vectorization region and want to "autodetect" the best choice.
13769 Otherwise, PREVAILING_MODE is a previously-chosen vector TYPE_MODE
13770 and we want the new type to be interoperable with it. PREVAILING_MODE
13771 in this case can be a scalar integer mode or a vector mode; when it
13772 is a vector mode, the function acts like a tree-level version of
13773 related_vector_mode. */
13775 tree
13778 {
13780 scalar_mode inner_mode;
13781 machine_mode simd_mode;
13782 tree vectype;
13793 /* Interoperability between modes requires one to be a constant multiple
13794 of the other, so that the number of vectors required for each operation
13795 is a compile-time constant. */
13803 /* For vector types of elements whose mode precision doesn't
13804 match their types precision we use a element type of mode
13805 precision. The vectorization routines will have to make sure
13806 they support the proper result truncation/extension.
13807 We also make sure to build vector types with INTEGER_TYPE
13808 component type only. */
13815 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
13816 When the component mode passes the above test simply use a type
13817 corresponding to that mode. The theory is that any use that
13818 would cause problems with this will disable vectorization anyway. */
13823 /* We can't build a vector type of elements with alignment bigger than
13824 their size. */
13829 /* If we felt back to using the mode fail if there was
13830 no scalar type for it. */
13834 /* If no prevailing mode was supplied, use the mode the target prefers.
13835 Otherwise lookup a vector mode based on the prevailing mode. */
13837 {
13841 {
13842 /* Traditional behavior is not to take the integer mode
13843 literally, but simply to use it as a way of determining
13844 the vector size. It is up to mode_for_vector to decide
13845 what the TYPE_MODE should be.
13847 Note that nunits == 1 is allowed in order to support single
13848 element vector types. */
13852 }
13853 }
13857 {
13858 /* Fall back to using mode_for_vector, mostly in the hope of being
13859 able to use an integer mode. */
13866 }
13870 /* In cases where the mode was chosen by mode_for_vector, check that
13871 the target actually supports the chosen mode, or that it at least
13872 allows the vector mode to be replaced by a like-sized integer. */
13877 /* Re-attach the address-space qualifier if we canonicalized the scalar
13878 type. */
13880 return build_qualified_type
13884 }
13886 /* Function get_vectype_for_scalar_type.
13888 Returns the vector type corresponding to SCALAR_TYPE as supported
13889 by the target. If GROUP_SIZE is nonzero and we're performing BB
13890 vectorization, make sure that the number of elements in the vector
13891 is no bigger than GROUP_SIZE. */
13893 tree
13896 {
13897 /* For BB vectorization, we should always have a group size once we've
13898 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
13899 are tentative requests during things like early data reference
13900 analysis and pattern recognition. */
13903 else
13907 scalar_type);
13911 /* Register the natural choice of vector type, before the group size
13912 has been applied. */
13916 /* If the natural choice of vector type doesn't satisfy GROUP_SIZE,
13917 try again with an explicit number of elements. */
13919 && group_size
13921 {
13922 /* Start with the biggest number of units that fits within
13923 GROUP_SIZE and halve it until we find a valid vector type.
13924 Usually either the first attempt will succeed or all will
13925 fail (in the latter case because GROUP_SIZE is too small
13926 for the target), but it's possible that a target could have
13927 a hole between supported vector types.
13929 If GROUP_SIZE is not a power of 2, this has the effect of
13930 trying the largest power of 2 that fits within the group,
13931 even though the group is not a multiple of that vector size.
13932 The BB vectorizer will then try to carve up the group into
13933 smaller pieces. */
13935 do
13936 {
13940 }
13942 }
13945 }
13947 /* Return the vector type corresponding to SCALAR_TYPE as supported
13948 by the target. NODE, if nonnull, is the SLP tree node that will
13949 use the returned vector type. */
13951 tree
13953 {
13958 }
13960 /* Function get_mask_type_for_scalar_type.
13962 Returns the mask type corresponding to a result of comparison
13963 of vectors of specified SCALAR_TYPE as supported by target.
13964 If GROUP_SIZE is nonzero and we're performing BB vectorization,
13965 make sure that the number of elements in the vector is no bigger
13966 than GROUP_SIZE. */
13968 tree
13971 {
13978 }
13980 /* Function get_mask_type_for_scalar_type.
13982 Returns the mask type corresponding to a result of comparison
13983 of vectors of specified SCALAR_TYPE as supported by target.
13984 NODE, if nonnull, is the SLP tree node that will use the returned
13985 vector type. */
13987 tree
13989 slp_tree node)
13990 {
13997 }
13999 /* Function get_same_sized_vectype
14001 Returns a vector type corresponding to SCALAR_TYPE of size
14002 VECTOR_TYPE if supported by the target. */
14004 tree
14006 {
14010 poly_uint64 nunits;
14017 }
14019 /* Return true if replacing LOOP_VINFO->vector_mode with VECTOR_MODE
14020 would not change the chosen vector modes. */
14022 bool
14024 {
14031 }
14033 /* Function vect_is_simple_use.
14035 Input:
14036 VINFO - the vect info of the loop or basic block that is being vectorized.
14037 OPERAND - operand in the loop or bb.
14038 Output:
14039 DEF_STMT_INFO_OUT (optional) - information about the defining stmt in
14040 case OPERAND is an SSA_NAME that is defined in the vectorizable region
14041 DEF_STMT_OUT (optional) - the defining stmt in case OPERAND is an SSA_NAME;
14042 the definition could be anywhere in the function
14043 DT - the type of definition
14045 Returns whether a stmt with OPERAND can be vectorized.
14046 For loops, supportable operands are constants, loop invariants, and operands
14047 that are defined by the current iteration of the loop. Unsupportable
14048 operands are those that are defined by a previous iteration of the loop (as
14049 is the case in reduction/induction computations).
14050 For basic blocks, supportable operands are constants and bb invariants.
14051 For now, operands defined outside the basic block are not supported. */
14053 bool
14056 {
14064 {
14070 else
14072 }
14082 else
14083 {
14088 else
14089 {
14095 }
14098 }
14101 {
14104 {
14138 }
14139 }
14142 {
14147 }
14150 }
14152 /* Function vect_is_simple_use.
14154 Same as vect_is_simple_use but also determines the vector operand
14155 type of OPERAND and stores it to *VECTYPE. If the definition of
14156 OPERAND is vect_uninitialized_def, vect_constant_def or
14157 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
14158 is responsible to compute the best suited vector type for the
14159 scalar operand. */
14161 bool
14165 {
14166 stmt_vec_info def_stmt_info;
14176 /* Now get a vector type if the def is internal, otherwise supply
14177 NULL_TREE and leave it up to the caller to figure out a proper
14178 type for the use stmt. */
14185 {
14191 }
14196 else
14200 }
14202 /* Function vect_is_simple_use.
14204 Same as vect_is_simple_use but determines the operand by operand
14205 position OPERAND from either STMT or SLP_NODE, filling in *OP
14206 and *SLP_DEF (when SLP_NODE is not NULL). */
14208 bool
14213 {
14215 {
14220 {
14221 /* ??? VEC_PERM nodes might be intermediate and their lane value
14222 have no representative (nor do we build a VEC_PERM stmt for
14223 the actual operation). Note for two-operator nodes we set
14224 a representative but leave scalar stmts empty as we'd only
14225 have one for a subset of lanes. Ideally no caller would
14226 require *op for internal defs. */
14228 {
14231 }
14232 else
14233 {
14240 }
14241 }
14242 else
14243 {
14249 }
14250 }
14251 else
14252 {
14255 {
14258 {
14261 else
14263 }
14266 else
14268 }
14273 else
14276 }
14277 }
14279 /* If OP is not NULL and is external or constant update its vector
14280 type with VECTYPE. Returns true if successful or false if not,
14281 for example when conflicting vector types are present. */
14283 bool
14285 {
14290 /* For external defs refuse to produce VECTOR_BOOLEAN_TYPE_P, those
14291 should be handled by patters. Allow vect_constant_def for now. */
14297 }
14299 /* Function supportable_widening_operation
14301 Check whether an operation represented by the code CODE is a
14302 widening operation that is supported by the target platform in
14303 vector form (i.e., when operating on arguments of type VECTYPE_IN
14304 producing a result of type VECTYPE_OUT).
14306 Widening operations we currently support are NOP (CONVERT), FLOAT,
14307 FIX_TRUNC and WIDEN_MULT. This function checks if these operations
14308 are supported by the target platform either directly (via vector
14309 tree-codes), or via target builtins.
14311 Output:
14312 - CODE1 and CODE2 are codes of vector operations to be used when
14313 vectorizing the operation, if available.
14314 - MULTI_STEP_CVT determines the number of required intermediate steps in
14315 case of multi-step conversion (like char->short->int - in that case
14316 MULTI_STEP_CVT will be 1).
14317 - INTERM_TYPES contains the intermediate type required to perform the
14318 widening operation (short in the above example). */
14320 bool
14322 code_helper code,
14323 stmt_vec_info stmt_info,
14329 {
14332 machine_mode vec_mode;
14348 {
14350 /* Don't set c1 and c2 if code is not a tree_code. */
14354 /* The result of a vectorized widening operation usually requires
14355 two vectors (because the widened results do not fit into one vector).
14356 The generated vector results would normally be expected to be
14357 generated in the same order as in the original scalar computation,
14358 i.e. if 8 results are generated in each vector iteration, they are
14359 to be organized as follows:
14360 vect1: [res1,res2,res3,res4],
14361 vect2: [res5,res6,res7,res8].
14363 However, in the special case that the result of the widening
14364 operation is used in a reduction computation only, the order doesn't
14365 matter (because when vectorizing a reduction we change the order of
14366 the computation). Some targets can take advantage of this and
14367 generate more efficient code. For example, targets like Altivec,
14368 that support widen_mult using a sequence of {mult_even,mult_odd}
14369 generate the following vectors:
14370 vect1: [res1,res3,res5,res7],
14371 vect2: [res2,res4,res6,res8].
14373 When vectorizing outer-loops, we execute the inner-loop sequentially
14374 (each vectorized inner-loop iteration contributes to VF outer-loop
14375 iterations in parallel). We therefore don't allow to change the
14376 order of the computation in the inner-loop during outer-loop
14377 vectorization. */
14378 /* TODO: Another case in which order doesn't *really* matter is when we
14379 widen and then contract again, e.g. (short)((int)x * y >> 8).
14380 Normally, pack_trunc performs an even/odd permute, whereas the
14381 repack from an even/odd expansion would be an interleave, which
14382 would be significantly simpler for e.g. AVX2. */
14383 /* In any case, in order to avoid duplicating the code below, recurse
14384 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
14385 are properly set up for the caller. If we fail, we'll continue with
14386 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
14393 interm_types))
14394 {
14395 /* Elements in a vector with vect_used_by_reduction property cannot
14396 be reordered if the use chain with this property does not have the
14397 same operation. One such an example is s += a * b, where elements
14398 in a and b cannot be reordered. Here we check if the vector defined
14399 by STMT is only directly used in the reduction statement. */
14404 }
14420 /* Support the recursion induced just above. */
14430 CASE_CONVERT:
14447 }
14453 {
14454 /* The signedness is determined from output operand. */
14457 }
14463 {
14464 /* If the input and result modes are the same, a different optab
14465 is needed where we pass in the number of units in vectype. */
14468 }
14472 {
14473 /* If this is an internal fn then we must check whether the target
14474 supports either a low-high split or an even-odd split. */
14484 /* If we don't support low-high, then check for even-odd. */
14487 || !optab2
14489 {
14495 }
14496 }
14498 {
14500 {
14501 /* The signedness is determined from output operand. */
14504 }
14510 {
14511 /* If the input and result modes are the same, a different optab
14512 is needed where we pass in the number of units in vectype. */
14515 }
14516 else
14517 {
14520 }
14523 }
14535 {
14538 /* For scalar masks we may have different boolean
14539 vector types having the same QImode. Thus we
14540 add additional check for elements number. */
14544 }
14546 /* Check if it's a multi-step conversion that can be done using intermediate
14547 types. */
14555 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14556 intermediate steps in promotion sequence. We try
14557 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
14558 not. */
14561 {
14564 intermediate_type
14567 {
14568 tree intermediate_element_type
14571 intermediate_type
14573 intermediate_mode);
14574 }
14575 else
14576 intermediate_type
14584 {
14585 /* If the input and result modes are the same, a different optab
14586 is needed where we pass in the number of units in vectype. */
14589 }
14590 else
14591 {
14594 }
14612 {
14618 }
14622 }
14626 }
14629 /* Function supportable_narrowing_operation
14631 Check whether an operation represented by the code CODE is a
14632 narrowing operation that is supported by the target platform in
14633 vector form (i.e., when operating on arguments of type VECTYPE_IN
14634 and producing a result of type VECTYPE_OUT).
14636 Narrowing operations we currently support are NOP (CONVERT), FIX_TRUNC
14637 and FLOAT. This function checks if these operations are supported by
14638 the target platform directly via vector tree-codes.
14640 Output:
14641 - CODE1 is the code of a vector operation to be used when
14642 vectorizing the operation, if available.
14643 - MULTI_STEP_CVT determines the number of required intermediate steps in
14644 case of multi-step conversion (like int->short->char - in that case
14645 MULTI_STEP_CVT will be 1).
14646 - INTERM_TYPES contains the intermediate type required to perform the
14647 narrowing operation (short in the above example). */
14649 bool
14654 {
14655 machine_mode vec_mode;
14672 {
14673 CASE_CONVERT:
14681 else
14687 /* The signedness is determined from output operand. */
14698 }
14710 {
14713 /* For scalar masks we may have different boolean
14714 vector types having the same QImode. Thus we
14715 add additional check for elements number. */
14719 }
14724 /* Check if it's a multi-step conversion that can be done using intermediate
14725 types. */
14730 else
14733 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
14734 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
14735 costly than signed. */
14737 {
14740 intermediate_type
14742 interm_optab
14748 {
14752 }
14753 }
14755 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
14756 intermediate steps in promotion sequence. We try
14757 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
14760 {
14763 intermediate_type
14765 else
14766 intermediate_type
14774 else
14775 interm_optab
14777 optab_default);
14789 {
14795 }
14800 }
14804 }
14806 /* Function supportable_indirect_convert_operation
14808 Check whether an operation represented by the code CODE is single or multi
14809 operations that are supported by the target platform in
14810 vector form (i.e., when operating on arguments of type VECTYPE_IN
14811 producing a result of type VECTYPE_OUT).
14813 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
14814 This function checks if these operations are supported
14815 by the target platform directly (via vector tree-codes).
14817 Output:
14818 - converts contains some pairs to perform the convert operation,
14819 the pair's first is the intermediate type, and its second is the code of
14820 a vector operation to be used when converting the operation from the
14821 previous type to the intermediate type. */
14822 bool
14824 tree vectype_out,
14825 tree vectype_in,
14827 tree op0)
14828 {
14832 opt_scalar_mode mode_iter;
14839 vectype_out,
14840 vectype_in,
14842 {
14845 }
14847 /* For conversions between float and integer types try whether
14848 we can use intermediate signed integer types to support the
14849 conversion. */
14853 {
14857 scalar_mode intermediate_mode;
14859 {
14862 }
14863 else
14864 {
14869 }
14872 opt_scalar_mode mode_iter;
14874 {
14880 scalar_mode cvt_mode;
14885 cvt_type = build_nonstandard_integer_type
14888 /* Check if the intermediate type can hold OP0's range.
14889 When converting from float to integer this is not necessary
14890 because values that do not fit the (smaller) target type are
14891 unspecified anyway. */
14893 {
14895 /* For vector form, it looks like op0 doesn't have RANGE_INFO.
14896 In the future, if it is supported, changes may need to be made
14897 to this part, such as checking the RANGE of each element
14898 in the vector. */
14911 }
14914 cvt_type,
14915 nelts);
14916 /* This should only happened for SLP as long as loop vectorizer
14917 only supports same-sized vector. */
14921 vectype_out,
14924 cvt_type,
14930 }
14933 {
14938 }
14939 }
14941 }
14943 /* Generate and return a vector mask of MASK_TYPE such that
14944 mask[I] is true iff J + START_INDEX < END_INDEX for all J <= I.
14945 Add the statements to SEQ. */
14947 tree
14950 {
14954 OPTIMIZE_FOR_SPEED));
14958 tree tmp;
14961 else
14966 }
14968 /* Generate a vector mask of type MASK_TYPE for which index I is false iff
14969 J + START_INDEX < END_INDEX for all J <= I. Add the statements to SEQ. */
14971 tree
14973 tree end_index)
14974 {
14977 }
14979 /* Try to compute the vector types required to vectorize STMT_INFO,
14980 returning true on success and false if vectorization isn't possible.
14981 If GROUP_SIZE is nonzero and we're performing BB vectorization,
14982 take sure that the number of elements in the vectors is no bigger
14983 than GROUP_SIZE.
14985 On success:
14987 - Set *STMT_VECTYPE_OUT to:
14988 - NULL_TREE if the statement doesn't need to be vectorized;
14989 - the equivalent of STMT_VINFO_VECTYPE otherwise.
14991 - Set *NUNITS_VECTYPE_OUT to the vector type that contains the maximum
14992 number of units needed to vectorize STMT_INFO, or NULL_TREE if the
14993 statement does not help to determine the overall number of units. */
14995 opt_result
15000 {
15003 /* For BB vectorization, we should always have a group size once we've
15004 constructed the SLP tree; the only valid uses of zero GROUP_SIZEs
15005 are tentative requests during things like early data reference
15006 analysis and pattern recognition. */
15009 else
15016 /* Allow vector conditionals through here. */
15018 /* MASK_STORE and friends have no lhs, but are ok. */
15022 {
15024 {
15025 /* Ignore calls with no lhs. These must be calls to
15026 #pragma omp simd functions, and what vectorization factor
15027 it really needs can't be determined until
15028 vectorizable_simd_clone_call. */
15033 }
15037 }
15039 tree vectype;
15042 {
15047 }
15049 {
15058 }
15059 else
15060 {
15061 /* If we got here with a gcond it means that the target had no available vector
15062 mode for the scalar type. We can't vectorize so abort. */
15065 "not vectorized:"
15067 scalar_type);
15071 else
15075 {
15078 "get vectype for scalar type (group size %d):"
15080 else
15083 }
15087 "not vectorized:"
15089 scalar_type);
15093 }
15098 stmt);
15102 /* Don't try to compute scalar types if the stmt produces a boolean
15103 vector; use the existing vector type instead. */
15106 {
15107 /* The number of units is set according to the smallest scalar
15108 type (or the largest vector size, but we only support one
15109 vector size per vectorization). */
15113 {
15117 scalar_type);
15119 group_size);
15123 scalar_type);
15126 nunits_vectype);
15127 }
15128 }
15133 "Not vectorized: Incompatible number "
15138 {
15142 }
15146 }
15148 /* Generate and return statement sequence that sets vector length LEN that is:
15150 min_of_start_and_end = min (START_INDEX, END_INDEX);
15151 left_len = END_INDEX - min_of_start_and_end;
15152 rhs = min (left_len, LEN_LIMIT);
15153 LEN = rhs;
15155 Note: the cost of the code generated by this function is modeled
15156 by vect_estimate_min_profitable_iters, so changes here may need
15157 corresponding changes there. */
15159 gimple_seq
15161 {
15173 }