| blob | be99137b052d1692ce06c4b69ac8bca5a7422f89 |
1 /* Machine description for AArch64 architecture.
2 Copyright (C) 2009-2025 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 #define IN_TARGET_CODE 1
23 #define INCLUDE_STRING
24 #define INCLUDE_ALGORITHM
25 #define INCLUDE_VECTOR
101 /* This file should be included last. */
104 /* Defined for convenience. */
105 #define POINTER_BYTES (POINTER_SIZE / BITS_PER_UNIT)
107 /* Maximum bytes set for an inline memset expansion. With -Os use 3 STP
108 and 1 MOVI/DUP (same size as a call). */
109 #define MAX_SET_SIZE(speed) (speed ? 256 : 96)
111 /* Flags that describe how a function shares certain architectural state
112 with its callers.
114 - AARCH64_STATE_SHARED indicates that the function does share the state
115 with callers.
117 - AARCH64_STATE_IN indicates that the function reads (or might read) the
118 incoming state. The converse is that the function ignores the incoming
119 state.
121 - AARCH64_STATE_OUT indicates that the function returns new state.
122 The converse is that the state on return is the same as it was on entry.
124 A function that partially modifies the state treats it as both IN
125 and OUT (because the value on return depends to some extent on the
126 value on input). */
131 /* Enum to distinguish which type of check is to be done in
132 aarch64_simd_valid_imm. */
134 AARCH64_CHECK_MOV,
135 AARCH64_CHECK_ORR,
136 AARCH64_CHECK_AND,
137 AARCH64_CHECK_XOR
138 };
140 /* Information about a legitimate vector immediate operand. */
141 struct simd_immediate_info
142 {
154 /* The mode of the elements. */
155 scalar_mode elt_mode;
157 /* The instruction to use to move the immediate into a vector. */
158 insn_type insn;
160 union
161 {
162 /* For MOV and MVN. */
163 struct
164 {
165 /* The value of each element. */
166 rtx value;
168 /* The kind of shift modifier to use, and the number of bits to shift.
169 This is (LSL, 0) if no shift is needed. */
170 modifier_type modifier;
174 /* For INDEX. */
175 struct
176 {
177 /* The value of the first element and the step to be added for each
178 subsequent element. */
182 /* For PTRUE. */
183 aarch64_svpattern pattern;
185 };
187 /* Construct a floating-point immediate in which each element has mode
188 ELT_MODE_IN and value VALUE_IN. */
189 inline simd_immediate_info
192 {
196 }
198 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
199 and value VALUE_IN. The other parameters are as for the structure
200 fields. */
201 inline simd_immediate_info
207 {
211 }
213 /* Construct an integer immediate in which each element has mode ELT_MODE_IN
214 and where element I is equal to BASE_IN + I * STEP_IN. */
215 inline simd_immediate_info
218 {
221 }
223 /* Construct a predicate that controls elements of mode ELT_MODE_IN
224 and has PTRUE pattern PATTERN_IN. */
225 inline simd_immediate_info
227 aarch64_svpattern pattern_in)
229 {
231 }
235 /* Describes types that map to Pure Scalable Types (PSTs) in the AAPCS64. */
236 class pure_scalable_type_info
237 {
239 /* Represents the result of analyzing a type. All values are nonzero,
240 in the possibly forlorn hope that accidental conversions to bool
241 trigger a warning. */
242 enum analysis_result
243 {
244 /* The type does not have an ABI identity; i.e. it doesn't contain
245 at least one object whose type is a Fundamental Data Type. */
248 /* The type is definitely a Pure Scalable Type. */
249 IS_PST,
251 /* The type is definitely not a Pure Scalable Type. */
252 ISNT_PST,
254 /* It doesn't matter for PCS purposes whether the type is a Pure
255 Scalable Type or not, since the type will be handled the same
256 way regardless.
258 Specifically, this means that if the type is a Pure Scalable Type,
259 there aren't enough argument registers to hold it, and so it will
260 need to be passed or returned in memory. If the type isn't a
261 Pure Scalable Type, it's too big to be passed or returned in core
262 or SIMD&FP registers, and so again will need to go in memory. */
263 DOESNT_MATTER
264 };
266 /* Aggregates of 17 bytes or more are normally passed and returned
267 in memory, so aggregates of that size can safely be analyzed as
268 DOESNT_MATTER. We need to be able to collect enough pieces to
269 represent a PST that is smaller than that. Since predicates are
270 2 bytes in size for -msve-vector-bits=128, that means we need to be
271 able to store at least 8 pieces.
273 We also need to be able to store enough pieces to represent
274 a single vector in each vector argument register and a single
275 predicate in each predicate argument register. This means that
276 we need at least 12 pieces. */
280 /* Describes one piece of a PST. Each piece is one of:
282 - a single Scalable Vector Type (SVT)
283 - a single Scalable Predicate Type (SPT)
284 - a PST containing 2, 3 or 4 SVTs, with no padding
286 It either represents a single built-in type or a PST formed from
287 multiple homogeneous built-in types. */
288 struct piece
289 {
292 /* The number of vector and predicate registers that the piece
293 occupies. One of the two is always zero. */
297 /* The mode of the registers described above. */
298 machine_mode mode;
300 /* If this piece is formed from multiple homogeneous built-in types,
301 this is the mode of the built-in types, otherwise it is MODE. */
302 machine_mode orig_mode;
304 /* The offset in bytes of the piece from the start of the type. */
305 poly_uint64 offset;
306 };
308 /* Divides types analyzed as IS_PST into individual pieces. The pieces
309 are in memory order. */
324 };
325 }
327 /* The current code model. */
332 /* The number of 64-bit elements in an SVE vector. */
333 poly_uint16 aarch64_sve_vg;
335 #ifdef HAVE_AS_TLS
336 #undef TARGET_HAVE_TLS
337 #define TARGET_HAVE_TLS 1
338 #endif
343 const_tree,
352 const_tree type,
357 aarch64_addr_query_type);
359 /* The processor for which instructions should be scheduled. */
362 /* Global flag for PC relative loads. */
365 /* Global flag for whether frame pointer is enabled. */
368 /* Support for command line parsing of boolean flags in the tuning
369 structures. */
370 struct aarch64_flag_desc
371 {
374 };
376 #define AARCH64_FUSION_PAIR(name, internal_name) \
377 { name, AARCH64_FUSE_##internal_name },
379 {
384 };
386 #define AARCH64_EXTRA_TUNING_OPTION(name, internal_name) \
387 { name, AARCH64_EXTRA_TUNE_##internal_name },
389 {
394 };
396 /* Tuning parameters. */
430 /* Support for fine-grained override of the tuning structures. */
431 struct aarch64_tuning_override_function
432 {
435 };
441 static const struct aarch64_tuning_override_function
442 aarch64_tuning_override_functions[] =
443 {
448 };
450 /* A processor implementing AArch64. */
451 struct processor
452 {
454 aarch64_cpu ident;
455 aarch64_cpu sched_core;
456 aarch64_arch arch;
457 aarch64_feature_flags flags;
459 };
461 /* Architectures implementing AArch64. */
463 {
464 #define AARCH64_ARCH(NAME, CORE, ARCH_IDENT, D, E) \
465 {NAME, AARCH64_CPU_##CORE, AARCH64_CPU_##CORE, AARCH64_ARCH_##ARCH_IDENT, \
466 feature_deps::ARCH_IDENT ().enable, NULL},
469 };
471 /* Processor cores implementing AArch64. */
473 {
474 #define AARCH64_CORE(NAME, IDENT, SCHED, ARCH, E, COSTS, G, H, I) \
475 {NAME, AARCH64_CPU_##IDENT, AARCH64_CPU_##SCHED, AARCH64_ARCH_##ARCH, \
476 feature_deps::cpu_##IDENT, &COSTS##_tunings},
479 };
480 /* Internal representation of system registers. */
483 /* Stringified sysreg encoding values, represented as
484 s<sn>_<op1>_c<cn>_c<cm>_<op2>. */
486 /* Flags affecting sysreg usage, such as read/write-only. */
488 /* Architectural features implied by sysreg. */
489 aarch64_feature_flags arch_reqs;
492 /* An aarch64_feature_set initializer for a single feature,
493 AARCH64_FEATURE_<FEAT>. */
494 #define AARCH64_FEATURE(FEAT) AARCH64_FL_##FEAT
496 /* Used by AARCH64_FEATURES. */
497 #define AARCH64_OR_FEATURES_1(X, F1) \
498 AARCH64_FEATURE (F1)
499 #define AARCH64_OR_FEATURES_2(X, F1, F2) \
500 (AARCH64_FEATURE (F1) | AARCH64_OR_FEATURES_1 (X, F2))
501 #define AARCH64_OR_FEATURES_3(X, F1, ...) \
502 (AARCH64_FEATURE (F1) | AARCH64_OR_FEATURES_2 (X, __VA_ARGS__))
504 /* An aarch64_feature_set initializer for the N features listed in "...". */
505 #define AARCH64_FEATURES(N, ...) \
506 AARCH64_OR_FEATURES_##N (0, __VA_ARGS__)
508 #define AARCH64_NO_FEATURES 0
510 /* Flags associated with the properties of system registers. It mainly serves
511 to mark particular registers as read or write only. */
512 #define F_DEPRECATED (1 << 1)
513 #define F_REG_READ (1 << 2)
514 #define F_REG_WRITE (1 << 3)
515 #define F_ARCHEXT (1 << 4)
516 /* Flag indicating register name is alias for another system register. */
517 #define F_REG_ALIAS (1 << 5)
518 /* Flag indicatinig registers which may be implemented with 128-bits. */
519 #define F_REG_128 (1 << 6)
521 /* Database of system registers, their encodings and architectural
522 requirements. */
524 {
526 #define SYSREG(NAME, ENC, FLAGS, ARCH) \
527 { NAME, ENC, FLAGS, ARCH },
529 #undef CPENC
530 };
532 #undef AARCH64_NO_FEATURES
537 /* Map system register names to their hardware metadata: encoding,
538 feature flags and architectural feature requirements, all of which
539 are encoded in a sysreg_t struct. */
540 void
542 {
545 }
547 /* Lazily initialize hash table for system register validation,
548 checking the validity of supplied register name and returning
549 register's associated metadata. */
550 static void
552 {
558 {
561 }
562 }
564 /* No direct access to the sysreg hash-map should be made. Doing so
565 risks trying to acess an unitialized hash-map and dereferencing the
566 returned double pointer without due care risks dereferencing a
567 null-pointer. */
570 {
578 }
580 /* The current tuning set. */
583 /* If NAME is the name of an arm:: attribute that describes shared state,
584 return its associated AARCH64_STATE_* flags, otherwise return 0. */
585 static unsigned int
587 {
597 }
599 /* See whether attribute list ATTRS has any sharing information
600 for state STATE_NAME. Return the associated state flags if so,
601 otherwise return 0. */
602 static unsigned int
604 {
606 {
616 {
621 }
622 }
624 }
626 /* Return true if DECL creates a new scope for state STATE_STRING. */
627 static bool
629 {
632 {
637 }
639 }
641 /* Return true if attribute argument VALUE is a recognized state string,
642 otherwise report an error. NAME is the name of the attribute to which
643 VALUE is being passed. */
644 static bool
646 {
648 {
651 }
656 {
659 }
662 }
664 /* qsort callback to compare two STRING_CSTs. */
665 static int
667 {
670 }
672 /* Canonicalize a list of state strings. ARGS contains the arguments to
673 a new attribute while OLD_ATTR, if nonnull, contains a previous attribute
674 of the same type. If CAN_MERGE_IN_PLACE, it is safe to adjust OLD_ATTR's
675 arguments and drop the new attribute. Otherwise, the new attribute must
676 be kept and ARGS must include the information in OLD_ATTR.
678 In both cases, the new arguments must be a sorted list of state strings
679 with duplicates removed.
681 Return true if new attribute should be kept, false if it should be
682 dropped. */
683 static bool
686 {
687 /* Get a sorted list of all state strings (including duplicates). */
689 {
693 };
700 /* The list can be empty if there was no previous attribute and if all
701 the new arguments are erroneous. Drop the attribute in that case. */
705 /* Destructively modify one of the argument lists, removing duplicates
706 on the fly. */
711 {
717 else
720 }
723 }
725 /* Check whether an 'aarch64_vector_pcs' attribute is valid. */
727 static tree
730 {
731 /* Since we set fn_type_req to true, the caller should have checked
732 this for us. */
735 {
742 name);
749 }
751 }
753 /* Return true if arm::new(ARGS) is compatible with the type of decl DECL,
754 otherwise report an error. */
755 static bool
757 {
760 {
763 {
766 {
768 "cannot create a new %qs scope since %qs is shared"
771 }
772 }
773 }
775 }
777 /* Callback for arm::new attributes. */
778 static tree
780 {
783 {
787 }
789 {
792 }
798 {
801 }
803 /* If there is an old attribute, we should try to update it in-place,
804 so that there is only one (definitive) arm::new attribute on the decl. */
810 }
812 /* Callback for arm::{in,out,inout,preserves} attributes. */
813 static tree
816 {
821 {
824 {
827 state_name);
829 {
833 }
834 }
835 }
837 /* We can't update an old attribute in-place, since types are shared.
838 Instead make sure that this new attribute contains all the
839 information, so that the old attribute becomes redundant. */
841 old_attrs);
846 }
848 /* Mutually-exclusive function type attributes for controlling PSTATE.SM. */
850 {
851 /* Attribute name exclusion applies to:
852 function, type, variable */
856 };
858 /* Table of machine attributes. */
860 {
861 /* { name, min_len, max_len, decl_req, type_req, fn_type_req,
862 affects_type_identity, handler, exclude } */
868 NULL },
872 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
875 #endif
876 #ifdef SUBTARGET_ATTRIBUTE_TABLE
877 SUBTARGET_ATTRIBUTE_TABLE
878 #endif
879 };
882 {
884 };
887 {
903 };
906 {
908 };
911 {
913 &aarch64_arm_attribute_table
914 };
917 {
921 }
922 aarch64_cc;
924 #define AARCH64_INVERSE_CONDITION_CODE(X) ((aarch64_cc) (((int) X) ^ 1))
927 /* The condition codes of the processor, and the inverse function. */
929 {
932 };
934 /* The preferred condition codes for SVE conditions. */
936 {
939 };
941 /* Return the assembly token for svpattern value VALUE. */
945 {
947 {
948 #define CASE(UPPER, LOWER, VALUE) case AARCH64_SV_##UPPER: return #LOWER;
950 #undef CASE
953 }
955 }
957 /* Return the location of a piece that is known to be passed or returned
958 in registers. FIRST_ZR is the first unused vector argument register
959 and FIRST_PR is the first unused predicate argument register. */
961 rtx
964 {
976 }
978 /* Return the total number of vector registers required by the PST. */
980 unsigned int
982 {
987 }
989 /* Return the total number of predicate registers required by the PST. */
991 unsigned int
993 {
998 }
1000 /* Return the location of a PST that is known to be passed or returned
1001 in registers. FIRST_ZR is the first unused vector argument register
1002 and FIRST_PR is the first unused predicate argument register. */
1004 rtx
1008 {
1009 /* Try to return a single REG if possible. This leads to better
1010 code generation; it isn't required for correctness. */
1012 {
1015 }
1017 /* Build up a PARALLEL that contains the individual pieces. */
1020 {
1026 }
1028 }
1030 /* Analyze whether TYPE is a Pure Scalable Type according to the rules
1031 in the AAPCS64. */
1035 {
1036 /* Prevent accidental reuse. */
1039 /* No code will be generated for erroneous types, so we won't establish
1040 an ABI mapping. */
1044 /* Zero-sized types disappear in the language->ABI mapping. */
1048 /* Check for SVTs, SPTs, and built-in tuple types that map to PSTs. */
1049 piece p = {};
1051 {
1059 }
1061 /* Check for user-defined PSTs. */
1068 }
1070 /* Analyze a type that is known not to be passed or returned in memory.
1071 Return true if it has an ABI identity and is a Pure Scalable Type. */
1073 bool
1075 {
1079 }
1081 /* Subroutine of analyze for handling ARRAY_TYPEs. */
1085 {
1086 /* Analyze the element type. */
1087 pure_scalable_type_info element_info;
1092 /* An array of unknown, flexible or variable length will be passed and
1093 returned by reference whatever we do. */
1098 /* Likewise if the array is constant-sized but too big to be interesting.
1099 The double checks against MAX_PIECES are to protect against overflow. */
1107 /* The above checks should have weeded out elements of unknown size. */
1108 poly_uint64 element_bytes;
1112 /* Build up the list of individual vectors and predicates. */
1116 {
1120 }
1122 }
1124 /* Subroutine of analyze for handling RECORD_TYPEs. */
1128 {
1130 {
1134 /* Zero-sized fields disappear in the language->ABI mapping. */
1138 /* All fields with an ABI identity must be PSTs for the record as
1139 a whole to be a PST. If any individual field is too big to be
1140 interesting then the record is too. */
1141 pure_scalable_type_info field_info;
1148 /* Since all previous fields are PSTs, we ought to be able to track
1149 the field offset using poly_ints. */
1153 /* For the same reason, it shouldn't be possible to create a PST field
1154 whose offset isn't byte-aligned. */
1156 BITS_PER_UNIT);
1158 /* Punt if the record is too big to be interesting. */
1159 poly_uint64 bytepos;
1164 /* Add the individual vectors and predicates in the field to the
1165 record's list. */
1168 {
1172 }
1173 }
1174 /* Empty structures disappear in the language->ABI mapping. */
1176 }
1178 /* Add P to the list of pieces in the type. */
1180 void
1182 {
1183 /* Try to fold the new piece into the previous one to form a
1184 single-mode PST. For example, if we see three consecutive vectors
1185 of the same mode, we can represent them using the corresponding
1186 3-tuple mode.
1188 This is purely an optimization. */
1190 {
1203 {
1207 }
1208 }
1210 }
1212 /* Return true if at least one possible value of type TYPE includes at
1213 least one object of Pure Scalable Type, in the sense of the AAPCS64.
1215 This is a relatively expensive test for some types, so it should
1216 generally be made as late as possible. */
1218 static bool
1220 {
1237 }
1239 /* Return the descriptor of the SIMD ABI. */
1243 {
1246 {
1247 HARD_REG_SET full_reg_clobbers
1253 }
1255 }
1257 /* Return the descriptor of the SVE PCS. */
1261 {
1264 {
1265 HARD_REG_SET full_reg_clobbers
1272 }
1274 }
1276 /* If X is an UNSPEC_SALT_ADDR expression, return the address that it
1277 wraps, otherwise return X itself. */
1279 static rtx
1281 {
1288 }
1290 /* Like strip_offset, but also strip any UNSPEC_SALT_ADDR from the
1291 expression. */
1293 static rtx
1295 {
1297 }
1299 /* Generate code to enable conditional branches in functions over 1 MiB. */
1303 {
1320 }
1322 void
1324 {
1329 else
1332 else
1336 else
1339 }
1341 /* Report when we try to do something that requires SVE when SVE is disabled.
1342 This is an error of last resort and isn't very high-quality. It usually
1343 involves attempts to measure the vector length in some way. */
1344 static void
1346 {
1349 /* Avoid reporting a slew of messages for a single oversight. */
1355 " option %<-march%>, or by using the %<target%>"
1358 }
1360 /* Return true if REGNO is P0-P15 or one of the special FFR-related
1361 registers. */
1364 {
1366 }
1368 /* Implement TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS.
1369 The register allocator chooses POINTER_AND_FP_REGS if FP_REGS and
1370 GENERAL_REGS have the same cost - even if POINTER_AND_FP_REGS has a much
1371 higher cost. POINTER_AND_FP_REGS is also used if the cost of both FP_REGS
1372 and GENERAL_REGS is lower than the memory cost (in this case the best class
1373 is the lowest cost one). Using POINTER_AND_FP_REGS irrespectively of its
1374 cost results in bad allocations with many redundant int<->FP moves which
1375 are expensive on various cores.
1376 To avoid this we don't allow POINTER_AND_FP_REGS as the allocno class, but
1377 force a decision between FP_REGS and GENERAL_REGS. We use the allocno class
1378 if it isn't POINTER_AND_FP_REGS. Similarly, use the best class if it isn't
1379 POINTER_AND_FP_REGS. Otherwise set the allocno class depending on the mode.
1380 The result of this is that it is no longer inefficient to have a higher
1381 memory move cost than the register move cost.
1382 */
1384 static reg_class_t
1386 reg_class_t best_class)
1387 {
1388 machine_mode mode;
1400 }
1402 static unsigned int
1404 {
1408 }
1410 /* Return the reassociation width of treeop OPC with mode MODE. */
1411 static int
1413 {
1418 /* Reassociation reduces the number of FMAs which may result in worse
1419 performance. Use a per-CPU setting for FMA reassociation which allows
1420 narrow CPUs with few FP pipes to switch it off (value of 1), and wider
1421 CPUs with many FP pipes to enable reassociation.
1422 Since the reassociation pass doesn't understand FMA at all, assume
1423 that any FP addition might turn into FMA. */
1428 }
1430 /* Provide a mapping from gcc register numbers to dwarf register numbers. */
1431 unsigned
1433 {
1445 /* Return values >= DWARF_FRAME_REGISTERS indicate that there is no
1446 equivalent DWARF register. */
1448 }
1450 /* Implement TARGET_DWARF_FRAME_REG_MODE. */
1451 static machine_mode
1453 {
1454 /* Predicate registers are call-clobbered in the EH ABI (which is
1455 ARM_PCS_AAPCS64), so they should not be described by CFI.
1456 Their size changes as VL changes, so any values computed by
1457 __builtin_init_dwarf_reg_size_table might not be valid for
1458 all frames. */
1462 }
1464 /* Implement TARGET_OUTPUT_CFI_DIRECTIVE. */
1465 static bool
1467 {
1470 {
1473 }
1475 }
1477 /* Implement TARGET_DW_CFI_OPRND1_DESC. */
1478 static bool
1481 {
1483 {
1486 }
1488 }
1490 /* If X is a CONST_DOUBLE, return its bit representation as a constant
1491 integer, otherwise return X unmodified. */
1492 static rtx
1494 {
1498 }
1500 /* Return an estimate for the number of quadwords in an SVE vector. This is
1501 equivalent to the number of Advanced SIMD vectors in an SVE vector. */
1502 static unsigned int
1504 {
1506 }
1508 /* Return true if MODE is an SVE predicate mode. */
1509 static bool
1511 {
1517 }
1519 /* Three mutually-exclusive flags describing a vector or predicate type. */
1523 /* Indicates a structure of 2, 3 or 4 vectors or predicates. */
1525 /* Can be used in combination with VEC_SVE_DATA to indicate that the
1526 vector has fewer significant bytes than a full SVE vector. */
1528 /* Useful combinations of the above. */
1532 /* Return a set of flags describing the vector properties of mode MODE.
1533 If ANY_TARGET_P is false (the default), ignore modes that are not supported
1534 by the current target. Otherwise categorize the modes that can be used
1535 with the set of all targets supported by the port. */
1537 static unsigned int
1539 {
1543 /* Make the decision based on the mode's enum value rather than its
1544 properties, so that we keep the correct classification regardless
1545 of -msve-vector-bits. */
1547 {
1548 /* Partial SVE QI vectors. */
1552 /* Partial SVE HI vectors. */
1555 /* Partial SVE SI vector. */
1557 /* Partial SVE HF vectors. */
1560 /* Partial SVE BF vectors. */
1563 /* Partial SVE SF vector. */
1577 /* x2 SVE vectors. */
1586 /* x3 SVE vectors. */
1595 /* x4 SVE vectors. */
1611 /* Structures of 64-bit Advanced SIMD vectors. */
1639 /* Structures of 128-bit Advanced SIMD vectors. */
1666 /* 64-bit Advanced SIMD vectors. */
1675 /* 128-bit Advanced SIMD vectors. */
1692 }
1693 }
1695 /* Like aarch64_classify_vector_mode, but also include modes that are used
1696 for memory operands but not register operands. Such modes do not count
1697 as real vector modes; they are just an internal construct to make things
1698 easier to describe. */
1699 static unsigned int
1701 {
1703 {
1718 }
1719 }
1721 /* Return true if MODE is any of the Advanced SIMD structure modes. */
1722 bool
1724 {
1727 }
1729 /* Return true if MODE is an Advanced SIMD D-register structure mode. */
1730 static bool
1732 {
1735 }
1737 /* Return true if MODE is an Advanced SIMD Q-register structure mode. */
1738 static bool
1740 {
1742 }
1744 /* Return true if MODE is any of the data vector modes, including
1745 structure modes. */
1746 static bool
1748 {
1750 }
1752 /* Return true if MODE is any form of SVE mode, including predicates,
1753 vectors and structures. */
1754 bool
1756 {
1758 }
1760 /* Return true if MODE is an SVE data vector mode; either a single vector
1761 or a structure of vectors. */
1762 static bool
1764 {
1766 }
1768 /* Return the number of defined bytes in one constituent vector of
1769 SVE mode MODE, which has vector flags VEC_FLAGS. */
1770 static poly_int64
1772 {
1774 /* A single partial vector. */
1778 /* A single vector or a tuple. */
1781 /* A single predicate. */
1784 }
1786 /* If MODE holds an array of vectors, return the number of vectors
1787 in the array, otherwise return 1. */
1789 static unsigned int
1791 {
1801 }
1803 /* Given an Advanced SIMD vector mode MODE and a tuple size NELEMS, return the
1804 corresponding vector structure mode. */
1805 opt_machine_mode
1808 {
1813 machine_mode struct_mode;
1821 }
1823 /* Return the SVE vector mode that has NUNITS elements of mode INNER_MODE. */
1825 opt_machine_mode
1827 {
1830 machine_mode mode;
1837 }
1839 /* Implement target hook TARGET_ARRAY_MODE. */
1840 static opt_machine_mode
1842 {
1844 {
1845 /* Use VNx32BI and VNx64BI for tuples of predicates, but explicitly
1846 reject giving a mode to other array sizes. Using integer modes
1847 requires a round trip through memory and generates terrible code. */
1855 }
1866 }
1868 /* Implement target hook TARGET_ARRAY_MODE_SUPPORTED_P. */
1869 static bool
1872 {
1880 }
1882 /* MODE is some form of SVE vector mode. For data modes, return the number
1883 of vector register bits that each element of MODE occupies, such as 64
1884 for both VNx2DImode and VNx2SImode (where each 32-bit value is stored
1885 in a 64-bit container). For predicate modes, return the number of
1886 data bits controlled by each significant predicate bit. */
1888 static unsigned int
1890 {
1893 ? BITS_PER_SVE_VECTOR
1896 }
1898 /* Return the SVE predicate mode to use for elements that have
1899 ELEM_NBYTES bytes, if such a mode exists. */
1901 opt_machine_mode
1903 {
1905 {
1914 }
1916 }
1918 /* Return the SVE predicate mode that should be used to control
1919 SVE mode MODE. */
1921 machine_mode
1923 {
1926 }
1928 /* Implement TARGET_VECTORIZE_GET_MASK_MODE. */
1930 static opt_machine_mode
1932 {
1938 }
1940 /* Return the integer element mode associated with SVE mode MODE. */
1942 static scalar_int_mode
1944 {
1946 ? BITS_PER_SVE_VECTOR
1951 }
1953 /* Return an integer element mode that contains exactly
1954 aarch64_sve_container_bits (MODE) bits. This is wider than
1955 aarch64_sve_element_int_mode if MODE is a partial vector,
1956 otherwise it's the same. */
1958 static scalar_int_mode
1960 {
1962 }
1964 /* Return the integer vector mode associated with SVE mode MODE.
1965 Unlike related_int_vector_mode, this can handle the case in which
1966 MODE is a predicate (and thus has a different total size). */
1968 machine_mode
1970 {
1973 }
1975 /* Look for a vector mode with the same classification as VEC_MODE,
1976 but with each group of FACTOR elements coalesced into a single element.
1977 In other words, look for a mode in which the elements are FACTOR times
1978 larger and in which the number of elements is FACTOR times smaller.
1980 Return the mode found, if one exists. */
1982 static opt_machine_mode
1984 {
1989 {
1993 }
1995 scalar_mode new_elt_mode;
2000 {
2005 }
2007 {
2008 poly_uint64 new_nunits;
2011 }
2013 }
2015 /* Implement TARGET_VECTORIZE_RELATED_MODE. */
2017 static opt_machine_mode
2019 scalar_mode element_mode,
2020 poly_uint64 nunits)
2021 {
2024 /* If we're operating on SVE vectors, try to return an SVE mode. */
2025 poly_uint64 sve_nunits;
2029 {
2030 machine_mode sve_mode;
2032 {
2033 /* Try to find a full or partial SVE mode with exactly
2034 NUNITS units. */
2039 }
2040 else
2041 {
2042 /* Take the preferred number of units from the number of bytes
2043 that fit in VECTOR_MODE. We always start by "autodetecting"
2044 a full vector mode with preferred_simd_mode, so vectors
2045 chosen here will also be full vector modes. Then
2046 autovectorize_vector_modes tries smaller starting modes
2047 and thus smaller preferred numbers of units. */
2052 }
2053 }
2055 /* Prefer to use 1 128-bit vector instead of 2 64-bit vectors. */
2062 {
2066 }
2069 }
2071 /* Implement TARGET_VECTORIZE_PREFERRED_DIV_AS_SHIFTS_OVER_MULT. */
2073 static bool
2075 {
2082 }
2084 /* Implement TARGET_PREFERRED_ELSE_VALUE. For binary operations,
2085 prefer to use the first arithmetic operand as the else value if
2086 the else value doesn't matter, since that exactly matches the SVE
2087 destructive merging form. For ternary operations we could either
2088 pick the first operand and use FMAD-like instructions or the last
2089 operand and use FMLA-like instructions; the latter seems more
2090 natural. */
2092 static tree
2094 {
2096 }
2098 /* Implement TARGET_HARD_REGNO_NREGS. */
2100 static unsigned int
2102 {
2103 /* ??? Logically we should only need to provide a value when
2104 HARD_REGNO_MODE_OK says that the combination is valid,
2105 but at the moment we need to handle all modes. Just ignore
2106 any runtime parts for registers that can't store them. */
2109 {
2113 {
2121 }
2136 }
2138 }
2140 /* Implement TARGET_HARD_REGNO_MODE_OK. */
2142 static bool
2144 {
2153 /* This must have the same size as _Unwind_Word. */
2169 /* These registers are abstract; their modes don't matter. */
2174 /* The purpose of comparing with ptr_mode is to support the
2175 global register variable associated with the stack pointer
2176 register via the syntax of asm ("wsp") in ILP32. */
2183 {
2190 }
2192 {
2195 else
2197 }
2200 }
2202 /* Return true if a function with type FNTYPE returns its value in
2203 SVE vector or predicate registers. */
2205 static bool
2207 {
2210 pure_scalable_type_info pst_info;
2212 {
2224 }
2226 }
2228 /* Return true if a function with type FNTYPE takes arguments in
2229 SVE vector or predicate registers. */
2231 static bool
2233 {
2234 CUMULATIVE_ARGS args_so_far_v;
2242 {
2249 pure_scalable_type_info pst_info;
2251 {
2256 }
2259 }
2261 }
2263 /* Implement TARGET_FNTYPE_ABI. */
2267 {
2276 }
2278 /* Return the state of PSTATE.SM on entry to functions of type FNTYPE. */
2280 static aarch64_isa_mode
2282 {
2291 }
2293 /* Return state flags that describe whether and how functions of type
2294 FNTYPE share state STATE_NAME with their callers. */
2296 static unsigned int
2298 {
2300 state_name);
2301 }
2303 /* Return the state of PSTATE.ZA on entry to functions of type FNTYPE. */
2305 static aarch64_isa_mode
2307 {
2313 }
2315 /* Return the ISA mode on entry to functions of type FNTYPE. */
2317 static aarch64_isa_mode
2319 {
2322 }
2324 /* Return true if FNDECL uses streaming mode internally, as an
2325 implementation choice. */
2327 static bool
2329 {
2332 }
2334 /* Return the state of PSTATE.SM when compiling the body of
2335 function FNDECL. This might be different from the state of
2336 PSTATE.SM on entry. */
2338 static aarch64_isa_mode
2340 {
2345 }
2347 /* Return true if function FNDECL has state STATE_NAME, either by creating
2348 new state itself or by sharing state with callers. */
2350 static bool
2352 {
2356 }
2358 /* Return the state of PSTATE.ZA when compiling the body of function FNDECL.
2359 This might be different from the state of PSTATE.ZA on entry. */
2361 static aarch64_isa_mode
2363 {
2369 }
2371 /* Return the ISA mode that should be used to compile the body of
2372 function FNDECL. */
2374 static aarch64_isa_mode
2376 {
2379 }
2381 /* Return the state of PSTATE.SM on entry to the current function.
2382 This might be different from the state of PSTATE.SM in the function
2383 body. */
2385 static aarch64_isa_mode
2387 {
2389 }
2391 /* Return the state of PSTATE.ZA on entry to the current function.
2392 This might be different from the state of PSTATE.ZA in the function
2393 body. */
2395 static aarch64_isa_mode
2397 {
2399 }
2401 /* Return state flags that describe whether and how the current function shares
2402 state STATE_NAME with callers. */
2404 static unsigned int
2406 {
2408 }
2410 /* Return true if the current function creates new state of type STATE_NAME
2411 (as opposed to sharing the state with its callers or ignoring the state
2412 altogether). */
2414 static bool
2416 {
2418 }
2420 /* Return true if PSTATE.SM is 1 in the body of the current function,
2421 but is not guaranteed to be 1 on entry. */
2423 static bool
2425 {
2428 }
2430 /* Return true if the current function has state STATE_NAME, either by
2431 creating new state itself or by sharing state with callers. */
2433 static bool
2435 {
2437 }
2439 /* Return true if a call from the current function to a function with
2440 ISA mode CALLEE_MODE would involve a change to PSTATE.SM around
2441 the BL instruction. */
2443 static bool
2445 {
2447 }
2449 /* Implement TARGET_COMPATIBLE_VECTOR_TYPES_P. */
2451 static bool
2453 {
2456 }
2458 /* Return true if we should emit CFI for register REGNO. */
2460 static bool
2462 {
2465 }
2467 /* Return the mode we should use to save and restore register REGNO. */
2469 static machine_mode
2471 {
2477 {
2479 /* Only the low 64 bits are saved by the base PCS. */
2483 /* The vector PCS saves the low 128 bits (which is the full
2484 register on non-SVE targets). */
2488 /* Use vectors of DImode for registers that need frame
2489 information, so that the first 64 bytes of the save slot
2490 are always the equivalent of what storing D<n> would give. */
2494 /* Use vectors of bytes otherwise, so that the layout is
2495 endian-agnostic, and so that we can use LDR and STR for
2496 big-endian targets. */
2502 }
2505 /* Save the full predicate register. */
2509 }
2511 /* Return the CONST_INT that should be placed in an UNSPEC_CALLEE_ABI rtx.
2512 This value encodes the following information:
2513 - the ISA mode on entry to a callee (ISA_MODE)
2514 - the ABI of the callee (PCS_VARIANT)
2515 - whether the callee has an indirect_return
2516 attribute (INDIRECT_RETURN). */
2518 rtx
2521 {
2527 }
2529 /* COOKIE is a CONST_INT from an UNSPEC_CALLEE_ABI rtx. Return the
2530 callee's ABI. */
2534 {
2537 }
2539 /* COOKIE is a CONST_INT from an UNSPEC_CALLEE_ABI rtx. Return the
2540 required ISA mode on entry to the callee, which is also the ISA
2541 mode on return from the callee. */
2543 static aarch64_isa_mode
2545 {
2547 }
2549 /* COOKIE is a CONST_INT from an UNSPEC_CALLEE_ABI rtx. Return
2550 whether function was marked with an indirect_return attribute. */
2552 static bool
2554 {
2556 }
2558 /* INSN is a call instruction. Return the CONST_INT stored in its
2559 UNSPEC_CALLEE_ABI rtx. */
2561 static rtx
2563 {
2570 }
2572 /* INSN is a call instruction. Return true if the callee has an
2573 indirect_return attribute. */
2575 bool
2577 {
2580 }
2582 /* Implement TARGET_INSN_CALLEE_ABI. */
2586 {
2588 }
2590 /* INSN is a call instruction. Return the required ISA mode on entry to
2591 the callee, which is also the ISA mode on return from the callee. */
2593 static aarch64_isa_mode
2595 {
2597 }
2599 /* Implement TARGET_HARD_REGNO_CALL_PART_CLOBBERED. The callee only saves
2600 the lower 64 bits of a 128-bit register. Tell the compiler the callee
2601 clobbers the top 64 bits when restoring the bottom 64 bits. */
2603 static bool
2606 machine_mode mode)
2607 {
2609 {
2617 }
2619 }
2621 /* Implement REGMODE_NATURAL_SIZE. */
2622 poly_uint64
2624 {
2625 /* The natural size for SVE data modes is one SVE data vector,
2626 and similarly for predicates. We can't independently modify
2627 anything smaller than that. */
2628 /* ??? For now, only do this for variable-width SVE registers.
2629 Doing it for constant-sized registers breaks lower-subreg.cc. */
2630 /* ??? And once that's fixed, we should probably have similar
2631 code for Advanced SIMD. */
2633 {
2634 /* REGMODE_NATURAL_SIZE influences general subreg validity rules,
2635 so we need to handle memory-only modes as well. */
2641 }
2643 }
2645 /* Implement HARD_REGNO_CALLER_SAVE_MODE. */
2646 machine_mode
2648 machine_mode mode)
2649 {
2650 /* The predicate mode determines which bits are significant and
2651 which are "don't care". Decreasing the number of lanes would
2652 lose data while increasing the number of lanes would make bits
2653 unnecessarily significant. */
2661 }
2663 /* Return true if I's bits are consecutive ones from the MSB. */
2664 bool
2666 {
2668 }
2670 /* Implement TARGET_CONSTANT_ALIGNMENT. Make strings word-aligned so
2671 that strcpy from constants will be faster. */
2673 static HOST_WIDE_INT
2675 {
2679 }
2681 /* Align definitions of arrays, unions and structures so that
2682 initializations and copies can be made more efficient. This is not
2683 ABI-changing, so it only affects places where we can see the
2684 definition. Increasing the alignment tends to introduce padding,
2685 so don't do this when optimizing for size/conserving stack space. */
2687 unsigned
2689 {
2694 {
2701 /* Align small structs/arrays to 32 bits, or 64 bits if larger. */
2706 }
2709 }
2711 unsigned
2713 {
2718 {
2725 /* Align small structs/arrays to 32 bits, or 64 bits if larger. */
2730 }
2733 }
2735 /* Return true if calls to DECL should be treated as
2736 long-calls (ie called via a register). */
2737 static bool
2739 {
2741 }
2743 /* Return true if calls to symbol-ref SYM should be treated as
2744 long-calls (ie called via a register). */
2745 bool
2747 {
2749 }
2751 /* Return true if calls to symbol-ref SYM should not go through
2752 plt stubs. */
2754 bool
2756 {
2760 && decl
2761 && (!flag_plt
2767 }
2769 /* Emit an insn that's a simple single-set. Both the operands must be
2770 known to be valid. */
2773 {
2775 }
2777 /* X and Y are two things to compare using CODE. Emit the compare insn and
2778 return the rtx for register 0 in the proper mode. */
2779 rtx
2781 {
2783 machine_mode cc_mode;
2784 rtx cc_reg;
2787 {
2802 }
2803 else
2804 {
2808 }
2810 }
2812 /* Similarly, but maybe zero-extend Y if Y_MODE < SImode. */
2814 static rtx
2816 machine_mode y_mode)
2817 {
2819 {
2821 {
2824 }
2825 else
2826 {
2828 machine_mode cc_mode;
2836 }
2837 }
2843 }
2845 /* Generate conditional branch to LABEL, comparing X to 0 using CODE.
2846 Return the jump instruction. */
2848 static rtx
2851 {
2853 {
2854 /* Emit an explicit compare instruction, so that we can correctly
2855 track the condition codes. */
2858 }
2859 else
2865 }
2867 /* Return an rtx that branches to LABEL based on the value of bit BITNUM of X.
2868 If CODE is NE, it branches to LABEL when the bit is set; if CODE is EQ,
2869 it branches to LABEL when the bit is clear. */
2871 static rtx
2874 {
2877 {
2883 }
2886 }
2888 /* Consider the operation:
2890 OPERANDS[0] = CODE (OPERANDS[1], OPERANDS[2]) + OPERANDS[3]
2892 where:
2894 - CODE is [SU]MAX or [SU]MIN
2895 - OPERANDS[2] and OPERANDS[3] are constant integers
2896 - OPERANDS[3] is a positive or negative shifted 12-bit immediate
2897 - all operands have mode MODE
2899 Decide whether it is possible to implement the operation using:
2901 SUBS <tmp>, OPERANDS[1], -OPERANDS[3]
2902 or
2903 ADDS <tmp>, OPERANDS[1], OPERANDS[3]
2905 followed by:
2907 <insn> OPERANDS[0], <tmp>, [wx]zr, <cond>
2909 where <insn> is one of CSEL, CSINV or CSINC. Return true if so.
2910 If GENERATE_P is true, also update OPERANDS as follows:
2912 OPERANDS[4] = -OPERANDS[3]
2913 OPERANDS[5] = the rtl condition representing <cond>
2914 OPERANDS[6] = <tmp>
2915 OPERANDS[7] = 0 for CSEL, -1 for CSINV or 1 for CSINC. */
2916 bool
2918 {
2925 /* max (x, y) - z == (x >= y + 1 ? x : y) - z
2926 == (x >= y ? x : y) - z
2927 == (x > y ? x : y) - z
2928 == (x > y - 1 ? x : y) - z
2930 min (x, y) - z == (x <= y - 1 ? x : y) - z
2931 == (x <= y ? x : y) - z
2932 == (x < y ? x : y) - z
2933 == (x < y + 1 ? x : y) - z
2935 Check whether z is in { y - 1, y, y + 1 } and pick the form(s) for
2936 which x is compared with z. Set DIFF to y - z. Thus the supported
2937 combinations are as follows, with DIFF being the value after the ":":
2939 max (x, y) - z == x >= y + 1 ? x - (y + 1) : -1 [z == y + 1]
2940 == x >= y ? x - y : 0 [z == y]
2941 == x > y ? x - y : 0 [z == y]
2942 == x > y - 1 ? x - (y - 1) : 1 [z == y - 1]
2944 min (x, y) - z == x <= y - 1 ? x - (y - 1) : 1 [z == y - 1]
2945 == x <= y ? x - y : 0 [z == y]
2946 == x < y ? x - y : 0 [z == y]
2947 == x < y + 1 ? x - (y + 1) : -1 [z == y + 1]. */
2960 rtx_code cmp;
2962 {
2977 }
2984 else
2989 }
2992 /* Build the SYMBOL_REF for __tls_get_addr. */
2996 rtx
2998 {
3002 }
3004 /* Return the TLS model to use for ADDR. */
3006 static enum tls_model
3008 {
3010 poly_int64 offset;
3016 }
3018 /* We'll allow lo_sum's in addresses in our legitimate addresses
3019 so that combine would take care of combining addresses where
3020 necessary, but for generation purposes, we'll generate the address
3021 as :
3022 RTL Absolute
3023 tmp = hi (symbol_ref); adrp x1, foo
3024 dest = lo_sum (tmp, symbol_ref); add dest, x1, :lo_12:foo
3025 nop
3027 PIC TLS
3028 adrp x1, :got:foo adrp tmp, :tlsgd:foo
3029 ldr x1, [:got_lo12:foo] add dest, tmp, :tlsgd_lo12:foo
3030 bl __tls_get_addr
3031 nop
3033 Load TLS symbol, depending on TLS mechanism and TLS access model.
3035 Global Dynamic - Traditional TLS:
3036 adrp tmp, :tlsgd:imm
3037 add dest, tmp, #:tlsgd_lo12:imm
3038 bl __tls_get_addr
3040 Global Dynamic - TLS Descriptors:
3041 adrp dest, :tlsdesc:imm
3042 ldr tmp, [dest, #:tlsdesc_lo12:imm]
3043 add dest, dest, #:tlsdesc_lo12:imm
3044 blr tmp
3045 mrs tp, tpidr_el0
3046 add dest, dest, tp
3048 Initial Exec:
3049 mrs tp, tpidr_el0
3050 adrp tmp, :gottprel:imm
3051 ldr dest, [tmp, #:gottprel_lo12:imm]
3052 add dest, dest, tp
3054 Local Exec:
3055 mrs tp, tpidr_el0
3056 add t0, tp, #:tprel_hi12:imm, lsl #12
3057 add t0, t0, #:tprel_lo12_nc:imm
3058 */
3060 static void
3063 {
3064 #if TARGET_PECOFF
3067 {
3070 }
3071 #endif
3074 {
3076 {
3077 /* In ILP32, the mode of dest can be either SImode or DImode. */
3088 {
3089 poly_int64 offset;
3092 HOST_WIDE_INT const_offset;
3094 /* Written this way for the sake of negative offsets. */
3096 }
3104 }
3111 {
3114 rtx insn;
3115 rtx mem;
3117 /* NOTE: pic_offset_table_rtx can be NULL_RTX, because we can reach
3118 here before rtl expand. Tree IVOPT will generate rtl pattern to
3119 decide rtx costs, in which case pic_offset_table_rtx is not
3120 initialized. For that case no need to generate the first adrp
3121 instruction as the final cost for global variable access is
3122 one instruction. */
3124 {
3125 /* -fpic for -mcmodel=small allow 32K GOT table size (but we are
3126 using the page base as GOT base, the first page may be wasted,
3127 in the worst scenario, there is only 28K space for GOT).
3129 The generate instruction sequence for accessing global variable
3130 is:
3132 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym]
3134 Only one instruction needed. But we must initialize
3135 pic_offset_table_rtx properly. We generate initialize insn for
3136 every global access, and allow CSE to remove all redundant.
3138 The final instruction sequences will look like the following
3139 for multiply global variables access.
3141 adrp pic_offset_table_rtx, _GLOBAL_OFFSET_TABLE_
3143 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym1]
3144 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym2]
3145 ldr reg, [pic_offset_table_rtx, #:gotpage_lo15:sym3]
3146 ... */
3155 }
3158 {
3161 else
3165 }
3166 else
3167 {
3172 }
3174 /* The operand is expected to be MEM. Whenever the related insn
3175 pattern changed, above code which calculate mem should be
3176 updated. */
3182 }
3189 {
3191 /* The return type of __tls_get_addr is the C pointer type
3192 so use ptr_mode. */
3202 else
3209 /* Convert back to the mode of the dest adding a zero_extend
3210 from SImode (ptr_mode) to DImode (Pmode). */
3214 }
3217 {
3220 rtx tp;
3224 /* In ILP32, the got entry is always of SImode size. Unlike
3225 small GOT, the dest is fixed at reg 0. */
3228 else
3239 }
3242 {
3243 /* In ILP32, the mode of dest can be either SImode or DImode,
3244 while the got entry is always of SImode size. The mode of
3245 dest depends on how dest is used: if dest is assigned to a
3246 pointer (e.g. in the memory), it has SImode; it may have
3247 DImode if dest is dereferenced to access the memeory.
3248 This is why we have to handle three different tlsie_small
3249 patterns here (two patterns for ILP32). */
3255 {
3258 else
3259 {
3262 }
3263 }
3264 else
3265 {
3268 }
3274 }
3280 {
3288 {
3311 }
3316 }
3319 {
3320 rtx insn;
3325 else
3326 {
3329 }
3333 }
3336 {
3341 {
3344 else
3345 {
3348 }
3349 }
3350 else
3351 {
3354 }
3359 }
3363 }
3364 }
3366 /* Emit a move from SRC to DEST. Assume that the move expanders can
3367 handle all moves if !can_create_pseudo_p (). The distinction is
3368 important because, unlike emit_move_insn, the move expanders know
3369 how to force Pmode objects into the constant pool even when the
3370 constant pool address is not itself legitimate. */
3371 static rtx
3373 {
3377 }
3379 /* Apply UNOPTAB to OP and store the result in DEST. */
3381 static void
3383 {
3387 }
3389 /* Apply BINOPTAB to OP0 and OP1 and store the result in DEST. */
3391 static void
3393 {
3395 OPTAB_DIRECT);
3398 }
3400 /* Split a move from SRC to DST into multiple moves of mode SINGLE_MODE. */
3402 void
3404 {
3411 {
3415 }
3417 /* At most one pairing may overlap. */
3421 else
3424 }
3426 /* Split a 128-bit move operation into two 64-bit move operations,
3427 taking care to handle partial overlap of register to register
3428 copies. Special cases are needed when moving between GP regs and
3429 FP regs. SRC can be a register, constant or memory; DST a register
3430 or memory. If either operand is memory it must not have any side
3431 effects. */
3432 void
3434 {
3442 {
3446 /* Handle FP <-> GP regs. */
3448 {
3455 }
3457 {
3464 }
3465 }
3468 }
3470 /* Return true if we should split a move from 128-bit value SRC
3471 to 128-bit register DEST. */
3473 bool
3475 {
3478 /* All moves to GPRs need to be split. */
3480 }
3482 /* Split a complex SIMD move. */
3484 void
3486 {
3493 {
3496 }
3497 }
3499 /* Return a register that contains SVE value X reinterpreted as SVE mode MODE.
3500 The semantics of those of svreinterpret rather than those of subregs;
3501 see the comment at the head of aarch64-sve.md for details about the
3502 difference. */
3504 rtx
3506 {
3510 /* can_change_mode_class must only return true if subregs and svreinterprets
3511 have the same semantics. */
3519 }
3521 bool
3524 {
3528 }
3530 /* Return TARGET if it is nonnull and a register of mode MODE.
3531 Otherwise, return a fresh register of mode MODE if we can,
3532 or TARGET reinterpreted as MODE if we can't. */
3534 static rtx
3536 {
3540 {
3543 }
3545 }
3547 /* Return a register that contains the constant in BUILDER, given that
3548 the constant is a legitimate move operand. Use TARGET as the register
3549 if it is nonnull and convenient. */
3551 static rtx
3553 {
3558 }
3560 static rtx
3562 {
3565 else
3566 {
3570 }
3571 }
3573 /* Return true if predicate value X is a constant in which every element
3574 is a CONST_INT. When returning true, describe X in BUILDER as a VNx16BI
3575 value, i.e. as a predicate in which all bits are significant. */
3577 static bool
3579 {
3591 {
3599 }
3602 }
3604 /* BUILDER contains a predicate constant of mode VNx16BI. Return the
3605 widest predicate element size it can have (that is, the largest size
3606 for which each element would still be 0 or 1). */
3608 unsigned int
3610 {
3611 /* Start with the most optimistic assumption: that we only need
3612 one bit per pattern. This is what we will use if only the first
3613 bit in each pattern is ever set. */
3617 /* Look for set bits. */
3621 {
3625 }
3627 }
3629 /* If VNx16BImode rtx X is a canonical PTRUE for a predicate mode,
3630 return that predicate mode, otherwise return opt_machine_mode (). */
3632 opt_machine_mode
3634 {
3648 }
3650 /* BUILDER is a predicate constant of mode VNx16BI. Consider the value
3651 that the constant would have with predicate element size ELT_SIZE
3652 (ignoring the upper bits in each element) and return:
3654 * -1 if all bits are set
3655 * N if the predicate has N leading set bits followed by all clear bits
3656 * 0 if the predicate does not have any of these forms. */
3658 int
3661 {
3662 /* If nelts_per_pattern is 3, we have set bits followed by clear bits
3663 followed by set bits. */
3667 /* Skip over leading set bits. */
3675 /* Check for the all-true case. */
3679 /* If nelts_per_pattern is 1, then either VL is zero, or we have a
3680 repeating pattern of set bits followed by clear bits. */
3684 /* We have a "foreground" value and a duplicated "background" value.
3685 If the background might repeat and the last set bit belongs to it,
3686 we might have set bits followed by clear bits followed by set bits. */
3690 /* Make sure that the rest are all clear. */
3696 }
3698 /* See if there is an svpattern that encodes an SVE predicate of mode
3699 PRED_MODE in which the first VL bits are set and the rest are clear.
3700 Return the pattern if so, otherwise return AARCH64_NUM_SVPATTERNS.
3701 A VL of -1 indicates an all-true vector. */
3703 aarch64_svpattern
3705 {
3720 {
3723 /* These would only trigger for non-power-of-2 lengths. */
3730 }
3732 }
3734 /* Return a VNx16BImode constant in which every sequence of ELT_SIZE
3735 bits has the lowest bit set and the upper bits clear. This is the
3736 VNx16BImode equivalent of a PTRUE for controlling elements of
3737 ELT_SIZE bytes. However, because the constant is VNx16BImode,
3738 all bits are significant, even the upper zeros. */
3740 rtx
3742 {
3748 }
3750 /* Return an all-true predicate register of mode MODE. */
3752 rtx
3754 {
3758 }
3760 /* Return an all-true (restricted to the leading VL bits) predicate register of
3761 mode MODE. */
3763 rtx
3765 {
3779 }
3781 /* Return a register of mode PRED_MODE for controlling data of mode DATA_MODE.
3783 DATA_MODE can be a scalar, an Advanced SIMD vector, or an SVE vector.
3784 If it's an N-byte scalar or an Advanced SIMD vector, the first N bits
3785 of the predicate will be active and the rest will be inactive.
3786 If DATA_MODE is an SVE mode, every bit of the predicate will be active. */
3787 rtx
3789 {
3795 }
3797 /* Return an all-false predicate register of mode MODE. */
3799 rtx
3801 {
3805 }
3807 /* PRED1[0] is a PTEST predicate and PRED1[1] is an aarch64_sve_ptrue_flag
3808 for it. PRED2[0] is the predicate for the instruction whose result
3809 is tested by the PTEST and PRED2[1] is again an aarch64_sve_ptrue_flag
3810 for it. Return true if we can prove that the two predicates are
3811 equivalent for PTEST purposes; that is, if we can replace PRED2[0]
3812 with PRED1[0] without changing behavior. */
3814 bool
3816 {
3828 }
3830 /* Emit a comparison CMP between OP0 and OP1, both of which have mode
3831 DATA_MODE, and return the result in a predicate of mode PRED_MODE.
3832 Use TARGET as the target register if nonnull and convenient. */
3834 static rtx
3837 {
3847 }
3849 /* Use a comparison to convert integer vector SRC into MODE, which is
3850 the corresponding SVE predicate mode. Use TARGET for the result
3851 if it's nonnull and convenient. */
3853 rtx
3855 {
3859 }
3861 /* Return the assembly token for svprfop value PRFOP. */
3865 {
3867 {
3868 #define CASE(UPPER, LOWER, VALUE) case AARCH64_SV_##UPPER: return #LOWER;
3870 #undef CASE
3873 }
3875 }
3877 /* Return the assembly string for an SVE prefetch operation with
3878 mnemonic MNEMONIC, given that PRFOP_RTX is the prefetch operation
3879 and that SUFFIX is the format for the remaining operands. */
3884 {
3891 }
3893 /* Check whether we can calculate the number of elements in PATTERN
3894 at compile time, given that there are NELTS_PER_VQ elements per
3895 128-bit block. Return the value if so, otherwise return -1. */
3897 HOST_WIDE_INT
3899 {
3906 {
3907 /* There are two vector granules per quadword. */
3910 {
3916 }
3917 }
3918 else
3921 /* There are two vector granules per quadword. */
3926 /* Requesting more elements than are available results in a PFALSE. */
3931 }
3933 /* Return true if a single CNT[BHWD] instruction can multiply FACTOR
3934 by the number of 128-bit quadwords in an SVE vector. */
3936 static bool
3938 {
3939 /* The coefficient must be [1, 16] * {2, 4, 8, 16}. */
3943 }
3945 /* Return true if we can move VALUE into a register using a single
3946 CNT[BHWD] instruction. */
3948 static bool
3950 {
3953 }
3955 /* Likewise for rtx X. */
3957 bool
3959 {
3960 poly_int64 value;
3962 }
3964 /* Return the asm string for an instruction with a CNT-like vector size
3965 operand (a vector pattern followed by a multiplier in the range [1, 16]).
3966 PREFIX is the mnemonic without the size suffix and OPERANDS is the
3967 first part of the operands template (the part that comes before the
3968 vector size itself). PATTERN is the pattern to use. FACTOR is the
3969 number of quadwords. NELTS_PER_VQ, if nonzero, is the number of elements
3970 in each quadword. If it is zero, we can use any element size. */
3974 aarch64_svpattern pattern,
3977 {
3981 /* There is some overlap in the ranges of the four CNT instructions.
3982 Here we always use the smallest possible element size, so that the
3983 multiplier is 1 whereever possible. */
3997 else
4000 factor);
4003 }
4005 /* Return the asm string for an instruction with a CNT-like vector size
4006 operand (a vector pattern followed by a multiplier in the range [1, 16]).
4007 PREFIX is the mnemonic without the size suffix and OPERANDS is the
4008 first part of the operands template (the part that comes before the
4009 vector size itself). X is the value of the vector size operand,
4010 as a polynomial integer rtx; we need to convert this into an "all"
4011 pattern with a multiplier. */
4015 rtx x)
4016 {
4021 }
4023 /* Return the asm string for an instruction with a CNT-like vector size
4024 operand (a vector pattern followed by a multiplier in the range [1, 16]).
4025 PREFIX is the mnemonic without the size suffix and OPERANDS is the
4026 first part of the operands template (the part that comes before the
4027 vector size itself). CNT_PAT[0..2] are the operands of the
4028 UNSPEC_SVE_CNT_PAT; see aarch64_sve_cnt_pat for details. */
4033 {
4039 }
4041 /* Return true if we can add X using a single SVE INC or DEC instruction. */
4043 bool
4045 {
4046 poly_int64 value;
4050 }
4052 /* Return the asm string for adding SVE INC/DEC immediate OFFSET to
4053 operand 0. */
4057 {
4063 else
4066 }
4068 /* Return true if a single RDVL instruction can multiply FACTOR by the
4069 number of 128-bit quadwords in an SVE vector. This is also the
4070 range of ADDVL. */
4072 static bool
4074 {
4077 }
4079 /* Return true if ADDPL can be used to add FACTOR multiplied by the number
4080 of quadwords in an SVE vector. */
4082 static bool
4084 {
4087 }
4089 /* Return true if we can move VALUE into a register using a single
4090 RDVL instruction. */
4092 static bool
4094 {
4097 }
4099 /* Likewise for rtx X. */
4101 bool
4103 {
4104 poly_int64 value;
4106 }
4108 /* Return the asm string for moving RDVL immediate OFFSET into register
4109 operand 0. */
4113 {
4121 }
4123 /* Return true if we can add VALUE to a register using a single ADDVL
4124 or ADDPL instruction. */
4126 static bool
4128 {
4134 }
4136 /* Likewise for rtx X. */
4138 bool
4140 {
4141 poly_int64 value;
4144 }
4146 /* Return the asm string for adding ADDVL or ADDPL immediate OFFSET
4147 to operand 1 and storing the result in operand 0. */
4151 {
4159 else
4162 }
4164 /* Return true if X is a valid immediate for an SVE vector INC or DEC
4165 instruction. If it is, store the number of elements in each vector
4166 quadword in *NELTS_PER_VQ_OUT (if nonnull) and store the multiplication
4167 factor in *FACTOR_OUT (if nonnull). */
4169 bool
4172 {
4173 rtx elt;
4174 poly_int64 value;
4182 /* There's no vector INCB. */
4189 /* The coefficient must be [1, 16] * NELTS_PER_VQ. */
4199 }
4201 /* Return true if X is a valid immediate for an SVE vector INC or DEC
4202 instruction. */
4204 bool
4206 {
4208 }
4210 /* Return the asm template for an SVE vector INC or DEC instruction.
4211 OPERANDS gives the operands before the vector count and X is the
4212 value of the vector count operand itself. */
4216 {
4224 else
4227 }
4229 /* Return a constant that represents FACTOR multiplied by the
4230 number of 128-bit quadwords in an SME vector. ISA_MODE is the
4231 ISA mode in which the calculation is being performed. */
4233 rtx
4235 aarch64_isa_mode isa_mode)
4236 {
4239 /* We're in streaming mode, so we can use normal poly-int values. */
4245 }
4247 /* Return true if X is a constant that represents some number X
4248 multiplied by the number of quadwords in an SME vector. Store this X
4249 in *FACTOR if so. */
4251 static bool
4253 {
4269 }
4271 /* Return true if X is a constant that represents some number Y
4272 multiplied by the number of quadwords in an SME vector, and if
4273 that Y is in the range of RDSVL. */
4275 bool
4277 {
4278 HOST_WIDE_INT factor;
4281 }
4283 /* Return the asm string for an RDSVL instruction that calculates X,
4284 which is a constant that satisfies aarch64_rdsvl_immediate_p. */
4288 {
4295 }
4297 /* Return true if X is a constant that can be added using ADDSVL or ADDSPL. */
4299 bool
4301 {
4302 HOST_WIDE_INT factor;
4306 }
4308 /* X is a constant that satisfies aarch64_addsvl_addspl_immediate_p.
4309 Return the asm string for the associated instruction. */
4313 {
4315 HOST_WIDE_INT factor;
4324 else
4327 }
4329 /* Multipliers for repeating bitmasks of width 32, 16, 8, 4, and 2. */
4332 {
4338 };
4342 /* Return true if 64-bit VAL is a valid bitmask immediate. */
4343 static bool
4345 {
4349 /* Check for a single sequence of one bits and return quickly if so.
4350 The special cases of all ones and all zeroes returns false. */
4356 /* Invert if the immediate doesn't start with a zero bit - this means we
4357 only need to search for sequences of one bits. */
4361 /* Find the first set bit and set tmp to val with the first sequence of one
4362 bits removed. Return success if there is a single sequence of ones. */
4369 /* Find the next set bit and compute the difference in bit position. */
4374 /* Check the bit position difference is a power of 2, and that the first
4375 sequence of one bits fits within 'bits' bits. */
4379 /* Check the sequence of one bits is repeated 64/bits times. */
4381 }
4384 /* Return true if VAL is a valid bitmask immediate for MODE. */
4385 bool
4387 {
4394 /* Replicate small immediates to fit 64 bits. */
4400 }
4403 /* Return true if the immediate VAL can be a bitfield immediate
4404 by changing the given MASK bits in VAL to zeroes, ones or bits
4405 from the other half of VAL. Return the new immediate in VAL2. */
4410 {
4425 }
4428 /* Return true if VAL is a valid MOVZ immediate. */
4431 {
4433 }
4436 /* Return true if immediate VAL can be created by a 64-bit MOVI/MOVN/MOVZ. */
4437 bool
4439 {
4442 }
4445 /* Return true if VAL is an immediate that can be created by a single
4446 MOV instruction. */
4447 bool
4449 {
4463 }
4466 static int
4468 machine_mode mode)
4469 {
4480 {
4484 }
4487 {
4489 {
4494 else
4497 }
4499 }
4501 /* Remaining cases are all for DImode. */
4509 /* Try a bitmask immediate and a movk to generate the immediate
4510 in 2 instructions. */
4513 {
4515 {
4522 }
4525 {
4527 {
4531 }
4533 }
4535 /* Try 2 bitmask immediates which are xor'd together. */
4537 {
4543 }
4546 {
4548 {
4551 }
4553 }
4554 }
4556 /* Try a bitmask plus 2 movk to generate the immediate in 3 instructions. */
4558 {
4562 {
4564 {
4570 }
4572 }
4574 /* Try shifting and inserting the bottom 32-bits into the top bits. */
4580 {
4582 {
4587 }
4589 }
4591 {
4593 {
4598 dest));
4599 }
4601 }
4602 }
4604 /* Generate 2-4 instructions, skipping 16 bits of all zeroes or ones which
4605 are emitted by the initial mov. If one_match > zero_match, skip set bits,
4606 otherwise skip zero bits. */
4618 {
4624 num_insns ++;
4625 }
4628 }
4630 /* Return whether imm is a 128-bit immediate which is simple enough to
4631 expand inline. */
4632 bool
4634 {
4645 }
4648 /* Return true if val can be encoded as a 12-bit unsigned immediate with
4649 a left shift of 0 or 12 bits. */
4650 bool
4652 {
4654 }
4656 /* Returns the nearest value to VAL that will fit as a 12-bit unsigned immediate
4657 that can be created with a left shift of 0 or 12. */
4658 static HOST_WIDE_INT
4660 {
4661 /* Check to see if the value fits in 24 bits, as that is the maximum we can
4662 handle correctly. */
4669 }
4672 /* Test whether:
4674 X = (X & AND_VAL) | IOR_VAL;
4676 can be implemented using:
4678 MOVK X, #(IOR_VAL >> shift), LSL #shift
4680 Return the shift if so, otherwise return -1. */
4681 int
4684 {
4688 {
4692 }
4694 }
4696 /* Create mask of ones, covering the lowest to highest bits set in VAL_IN.
4697 Assumed precondition: VAL_IN Is not zero. */
4699 unsigned HOST_WIDE_INT
4701 {
4708 }
4710 /* Create constant where bits outside of lowest bit set to highest bit set
4711 are set to 1. */
4713 unsigned HOST_WIDE_INT
4715 {
4717 }
4719 /* Return true if VAL_IN is a valid 'and' bitmask immediate. */
4721 bool
4723 {
4724 scalar_int_mode int_mode;
4737 }
4739 /* Return the number of temporary registers that aarch64_add_offset_1
4740 would need to add OFFSET to a register. */
4742 static unsigned int
4744 {
4746 }
4748 /* A subroutine of aarch64_add_offset. Set DEST to SRC + OFFSET for
4749 a non-polynomial OFFSET. MODE is the mode of the addition.
4750 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
4751 be set and CFA adjustments added to the generated instructions.
4753 TEMP1, if nonnull, is a register of mode MODE that can be used as a
4754 temporary if register allocation is already complete. This temporary
4755 register may overlap DEST but must not overlap SRC. If TEMP1 is known
4756 to hold abs (OFFSET), EMIT_MOVE_IMM can be set to false to avoid emitting
4757 the immediate again.
4759 Since this function may be used to adjust the stack pointer, we must
4760 ensure that it cannot cause transient stack deallocation (for example
4761 by first incrementing SP and then decrementing when adjusting by a
4762 large immediate). */
4764 static void
4768 {
4776 {
4778 {
4781 }
4783 }
4785 /* Single instruction adjustment. */
4787 {
4791 }
4793 /* Emit 2 additions/subtractions if the adjustment is less than 24 bits
4794 and either:
4796 a) the offset cannot be loaded by a 16-bit move or
4797 b) there is no spare register into which we can move it. */
4801 {
4810 }
4812 /* Emit a move immediate if required and an addition/subtraction. */
4814 {
4818 }
4823 {
4827 }
4828 }
4830 /* Return the number of temporary registers that aarch64_add_offset
4831 would need to move OFFSET into a register or add OFFSET to a register;
4832 ADD_P is true if we want the latter rather than the former. */
4834 static unsigned int
4836 {
4837 /* This follows the same structure as aarch64_add_offset. */
4846 /* Need one register for the ADDVL/ADDPL result. */
4849 {
4852 /* Need one register for the CNT or RDVL result and one for the
4853 multiplication factor. If necessary, the second temporary
4854 can be reused for the constant part of the offset. */
4856 /* Need one register for the CNT or RDVL result (which might then
4857 be shifted). */
4859 }
4861 }
4863 /* If X can be represented as a poly_int64, return the number
4864 of temporaries that are required to add it to a register.
4865 Return -1 otherwise. */
4867 int
4869 {
4870 poly_int64 offset;
4874 }
4876 /* Set DEST to SRC + OFFSET. MODE is the mode of the addition.
4877 FRAME_RELATED_P is true if the RTX_FRAME_RELATED flag should
4878 be set and CFA adjustments added to the generated instructions.
4880 TEMP1, if nonnull, is a register of mode MODE that can be used as a
4881 temporary if register allocation is already complete. This temporary
4882 register may overlap DEST if !FRAME_RELATED_P but must not overlap SRC.
4883 If TEMP1 is known to hold abs (OFFSET), EMIT_MOVE_IMM can be set to
4884 false to avoid emitting the immediate again.
4886 TEMP2, if nonnull, is a second temporary register that doesn't
4887 overlap either DEST or REG.
4889 FORCE_ISA_MODE is AARCH64_ISA_MODE_SM_ON if any variable component of OFFSET
4890 is measured relative to the SME vector length instead of the current
4891 prevailing vector length. It is 0 otherwise.
4893 Since this function may be used to adjust the stack pointer, we must
4894 ensure that it cannot cause transient stack deallocation (for example
4895 by first incrementing SP and then decrementing when adjusting by a
4896 large immediate). */
4898 static void
4901 aarch64_isa_mode force_isa_mode,
4903 {
4907 || !frame_related_p
4911 /* Try using ADDVL or ADDPL to add the whole value. */
4913 {
4915 rtx offset_rtx;
4918 else
4925 offset)));
4927 }
4929 /* Coefficient 1 is multiplied by the number of 128-bit blocks in an
4930 SVE vector register, over and above the minimum size of 128 bits.
4931 This is equivalent to half the value returned by CNTD with a
4932 vector shape of ALL. */
4936 /* Try using ADDVL or ADDPL to add the VG-based part. */
4940 {
4941 rtx offset_rtx;
4944 else
4947 {
4953 poly_offset)));
4955 }
4956 else
4957 {
4962 }
4963 }
4964 /* Otherwise use a CNT-based sequence. */
4966 {
4967 /* Calculate CNTB * FACTOR / 16 as CNTB * REL_FACTOR * 2**SHIFT,
4968 with negative shifts indicating a shift right. */
4974 /* Set CODE, VAL and SHIFT so that [+-] VAL * 2**SHIFT is
4975 equal to CNTB * FACTOR / 16, with CODE being the [+-].
4977 We can avoid a multiplication if REL_FACTOR is in the range
4978 of RDVL, although there are then various optimizations that
4979 we can try on top. */
4981 rtx val;
4983 {
4985 {
4986 /* Try to use an unshifted RDSVL, otherwise fall back on
4987 a shifted RDSVL #1. */
4990 else
4993 }
4994 /* Try to use an unshifted CNT[BHWD] or RDVL. */
4997 {
5000 }
5001 /* Try to subtract an unshifted CNT[BHWD]. */
5003 {
5007 }
5008 /* If subtraction is free, prefer to load a positive constant.
5009 In the best case this will fit a shifted CNTB. */
5011 {
5014 }
5015 /* Otherwise use a shifted RDVL or CNT[BHWD]. */
5016 else
5018 }
5019 else
5020 {
5021 /* If we can calculate CNTB << SHIFT directly, prefer to do that,
5022 since it should increase the chances of being able to use
5023 a shift and add sequence for the multiplication.
5024 If CNTB << SHIFT is out of range, stick with the current
5025 shift factor. */
5028 {
5031 }
5034 {
5037 }
5038 else
5043 /* Prefer to multiply by a positive factor and subtract rather
5044 than multiply by a negative factor and add, since positive
5045 values are usually easier to move. */
5047 {
5050 }
5053 {
5056 }
5057 else
5058 {
5062 }
5063 }
5065 /* Multiply by 2 ** SHIFT. */
5067 {
5070 }
5072 {
5075 }
5077 /* Add the result to SRC or subtract the result from SRC. */
5079 {
5082 }
5084 {
5087 }
5090 {
5093 {
5097 poly_offset)));
5098 }
5102 }
5103 else
5104 {
5108 }
5111 }
5115 }
5117 /* Like aarch64_add_offset, but the offset is given as an rtx rather
5118 than a poly_int64. */
5120 void
5123 {
5126 }
5128 /* Add DELTA to the stack pointer, marking the instructions frame-related.
5129 TEMP1 is available as a temporary if nonnull. FORCE_ISA_MODE is as
5130 for aarch64_add_offset. EMIT_MOVE_IMM is false if TEMP1 already
5131 contains abs (DELTA). */
5136 {
5139 }
5141 /* Subtract DELTA from the stack pointer, marking the instructions
5142 frame-related if FRAME_RELATED_P. FORCE_ISA_MODE is as for
5143 aarch64_add_offset. TEMP1 is available as a temporary if nonnull. */
5147 aarch64_isa_mode force_isa_mode,
5149 {
5152 emit_move_imm);
5153 }
5155 /* A streaming-compatible function needs to switch temporarily to the known
5156 PSTATE.SM mode described by LOCAL_MODE. The low bit of OLD_SVCR contains
5157 the runtime state of PSTATE.SM in the streaming-compatible code, before
5158 the start of the switch to LOCAL_MODE.
5160 Emit instructions to branch around the mode switch if PSTATE.SM already
5161 matches LOCAL_MODE. Return the label that the branch jumps to. */
5165 {
5171 label);
5175 }
5177 /* Emit code to switch from the PSTATE.SM state in OLD_MODE to the PSTATE.SM
5178 state in NEW_MODE. This is known to involve either an SMSTART SM or
5179 an SMSTOP SM. */
5181 static void
5183 {
5191 else
5193 }
5195 /* As a side-effect, SMSTART SM and SMSTOP SM clobber the contents of all
5196 FP and predicate registers. This class emits code to preserve any
5197 necessary registers around the mode switch.
5199 The class uses four approaches to saving and restoring contents, enumerated
5200 by group_type:
5202 - GPR: save and restore the contents of FP registers using GPRs.
5203 This is used if the FP register contains no more than 64 significant
5204 bits. The registers used are FIRST_GPR onwards.
5206 - MEM_128: save and restore 128-bit SIMD registers using memory.
5208 - MEM_SVE_PRED: save and restore full SVE predicate registers using memory.
5210 - MEM_SVE_DATA: save and restore full SVE vector registers using memory.
5212 The save slots within each memory group are consecutive, with the
5213 MEM_SVE_PRED slots occupying a region below the MEM_SVE_DATA slots.
5215 There will only be two mode switches for each use of SME, so they should
5216 not be particularly performance-sensitive. It's also rare for SIMD, SVE
5217 or predicate registers to be live across mode switches. We therefore
5218 don't preallocate the save slots but instead allocate them locally on
5219 demand. This makes the code emitted by the class self-contained. */
5221 class aarch64_sme_mode_switch_regs
5222 {
5235 /* The number of GPRs needed to save FP registers, starting from
5236 FIRST_GPR. */
5243 /* Information about the save location for one FP, SIMD, SVE data, or
5244 SVE predicate register. */
5246 /* The register to be saved. */
5247 rtx reg;
5249 /* Which group the save location belongs to. */
5250 group_type group;
5252 /* A zero-based index of the register within the group. */
5254 };
5267 /* All save locations, in no particular order. */
5270 /* The number of registers in each group. */
5272 };
5274 /* Record that (reg:MODE REGNO) needs to be preserved around the mode
5275 switch. */
5277 void
5279 {
5287 {
5288 /* Force the mode of SVE saves and restores even for single registers.
5289 This is necessary because big-endian targets only allow LDR Z and
5290 STR Z to be used with byte modes. */
5297 {
5300 else
5302 }
5303 save_location loc;
5306 {
5309 }
5310 else
5311 {
5317 else
5319 }
5322 }
5323 }
5325 /* Record that the arguments to CALL_INSN need to be preserved around
5326 the mode switch. */
5328 void
5330 {
5333 {
5341 }
5342 }
5344 /* Record that the return value from CALL_INSN (if any) needs to be
5345 preserved around the mode switch. */
5347 void
5349 {
5358 {
5363 }
5364 else
5366 }
5368 /* REGNO is a register that is call-preserved under the current function's ABI.
5369 Record that it must be preserved around the mode switch. */
5371 void
5373 {
5376 {
5388 }
5391 }
5393 /* The hard registers in REGS are call-preserved under the current function's
5394 ABI. Record that they must be preserved around the mode switch. */
5396 void
5398 {
5399 bitmap_iterator bi;
5404 else
5406 }
5408 /* Emit code to save registers before the mode switch. */
5410 void
5412 {
5418 }
5420 /* Emit code to restore registers after the mode switch. */
5422 void
5424 {
5430 }
5432 /* The SVE predicate registers are stored below the SVE data registers,
5433 with the predicate save area being padded to a data-register-sized
5434 boundary. Return the size of this padded area as a whole number
5435 of data register slots. */
5437 unsigned int
5439 {
5441 }
5443 /* Return a memory reference of mode MODE to OFFSET bytes from the
5444 stack pointer. */
5446 rtx
5448 poly_int64 offset)
5449 {
5452 }
5454 /* Allocate or deallocate SIZE bytes of stack space: SEQ decides which. */
5456 void
5458 poly_int64 size)
5459 {
5464 }
5466 /* Save or restore the register in LOC, whose slot is OFFSET bytes from
5467 the stack pointer. SEQ chooses between saving and restoring. */
5469 void
5472 poly_int64 offset)
5473 {
5477 else
5479 }
5481 /* Emit instructions to save or restore the GPR group. SEQ chooses between
5482 saving and restoring. */
5484 void
5486 {
5489 {
5494 else
5496 }
5497 }
5499 /* Emit instructions to save or restore the MEM_128 group. SEQ chooses
5500 between saving and restoring. */
5502 void
5504 {
5512 /* Pick a common mode that supports LDR & STR with pre/post-modification
5513 and LDP & STP with pre/post-modification. */
5516 /* An instruction pattern that should be emitted at the end. */
5519 /* A previous MEM_128 location that hasn't been handled yet. */
5522 /* Look for LDP/STPs and record any leftover LDR/STR in PREV_LOC. */
5525 {
5527 {
5530 }
5533 /* The offset of the base of the save area from the current
5534 stack pointer. */
5539 /* Get the two sets in the LDP/STP. */
5540 rtx ops[] = {
5545 };
5550 /* Combine the sets with any stack allocation/deallocation. */
5551 rtx pat;
5553 {
5557 }
5560 else
5563 /* Queue a deallocation to the end, otherwise emit the
5564 instruction now. */
5567 else
5570 }
5572 /* Handle any leftover LDR/STR. */
5574 {
5576 rtx addr;
5580 {
5583 }
5584 else
5585 {
5588 }
5592 else
5594 }
5598 }
5600 /* Allocate or deallocate the stack space needed by the SVE groups.
5601 SEQ chooses between allocating and deallocating. */
5603 void
5605 {
5608 }
5610 /* Save or restore the MEM_SVE_DATA group. SEQ chooses between saving
5611 and restoring. */
5613 void
5615 {
5618 {
5621 }
5622 }
5624 /* Save or restore the MEM_SVE_PRED group. SEQ chooses between saving
5625 and restoring. */
5627 void
5629 {
5633 }
5635 /* Set DEST to (vec_series BASE STEP). */
5637 static void
5639 {
5643 /* Each operand can be a register or an immediate in the range [-16, 15]. */
5650 }
5652 /* Duplicate 128-bit Advanced SIMD vector SRC so that it fills an SVE
5653 register of mode MODE. Use TARGET for the result if it's nonnull
5654 and convenient.
5656 The two vector modes must have the same element mode. The behavior
5657 is to duplicate architectural lane N of SRC into architectural lanes
5658 N + I * STEP of the result. On big-endian targets, architectural
5659 lane 0 of an Advanced SIMD vector is the last element of the vector
5660 in memory layout, so for big-endian targets this operation has the
5661 effect of reversing SRC before duplicating it. Callers need to
5662 account for this. */
5664 rtx
5666 {
5669 insn_code icode = (BYTES_BIG_ENDIAN
5678 {
5679 /* Create a PARALLEL describing the reversal of SRC. */
5684 }
5687 }
5689 /* Try to force 128-bit vector value SRC into memory and use LD1RQ to fetch
5690 the memory image into DEST. Return true on success. */
5692 static bool
5694 {
5699 /* Make sure that the address is legitimate. */
5701 {
5704 }
5711 }
5713 /* SRC is an SVE CONST_VECTOR that contains N "foreground" values followed
5714 by N "background" values. Try to move it into TARGET using:
5716 PTRUE PRED.<T>, VL<N>
5717 MOV TRUE.<T>, #<foreground>
5718 MOV FALSE.<T>, #<background>
5719 SEL TARGET.<T>, PRED.<T>, TRUE.<T>, FALSE.<T>
5721 The PTRUE is always a single instruction but the MOVs might need a
5722 longer sequence. If the background value is zero (as it often is),
5723 the sequence can sometimes collapse to a PTRUE followed by a
5724 zero-predicated move.
5726 Return the target on success, otherwise return null. */
5728 static rtx
5730 {
5733 /* Make sure that the PTRUE is valid. */
5745 {
5748 }
5750 {
5753 }
5761 }
5763 /* Return a register containing CONST_VECTOR SRC, given that SRC has an
5764 SVE data mode and isn't a legitimate constant. Use TARGET for the
5765 result if convenient.
5767 The returned register can have whatever mode seems most natural
5768 given the contents of SRC. */
5770 static rtx
5772 {
5784 {
5785 /* We have a partial vector mode and a constant whose full-vector
5786 equivalent would occupy a repeating 128-bit sequence. Build that
5787 full-vector equivalent instead, so that we have the option of
5788 using LD1RQ and Advanced SIMD operations. */
5797 }
5800 {
5801 /* The constant is a duplicated quadword but can't be narrowed
5802 beyond a quadword. Get the memory image of the first quadword
5803 as a 128-bit vector and try using LD1RQ to load it from memory.
5805 The effect for both endiannesses is to load memory lane N into
5806 architectural lanes N + I * STEP of the result. On big-endian
5807 targets, the layout of the 128-bit vector in an Advanced SIMD
5808 register would be different from its layout in an SVE register,
5809 but this 128-bit vector is a memory value only. */
5814 }
5817 {
5818 /* The vector is a repeating sequence of 64 bits or fewer.
5819 See if we can load them using an Advanced SIMD move and then
5820 duplicate it to fill a vector. This is better than using a GPR
5821 move because it keeps everything in the same register file. */
5825 {
5826 /* We want memory lane N to go into architectural lane N,
5827 so reverse for big-endian targets. The DUP .Q pattern
5828 has a compensating reverse built-in. */
5831 }
5834 {
5837 }
5839 /* Get an integer representation of the repeating part of Advanced
5840 SIMD vector VQ_SRC. This preserves the endianness of VQ_SRC,
5841 which for big-endian targets is lane-swapped wrt a normal
5842 Advanced SIMD vector. This means that for both endiannesses,
5843 memory lane N of SVE vector SRC corresponds to architectural
5844 lane N of a register holding VQ_SRC. This in turn means that
5845 memory lane 0 of SVE vector SRC is in the lsb of VQ_SRC (viewed
5846 as a single 128-bit value) and thus that memory lane 0 of SRC is
5847 in the lsb of the integer. Duplicating the integer therefore
5848 ensures that memory lane N of SRC goes into architectural lane
5849 N + I * INDEX of the SVE register. */
5853 {
5854 /* Pretend that we had a vector of INT_MODE to start with. */
5858 /* If the integer can be moved into a general register by a
5859 single instruction, do that and duplicate the result. */
5863 {
5866 }
5867 }
5869 /* We're duplicating a single value, but can't do better than
5870 force it to memory and load from there. This handles things
5871 like symbolic constants. */
5875 {
5876 /* Load the element from memory if we can, otherwise move it into
5877 a register and use a DUP. */
5882 }
5883 }
5885 /* Try using INDEX. */
5888 {
5891 }
5893 /* From here on, it's better to force the whole constant to memory
5894 if we can. */
5902 /* Expand each pattern individually. */
5904 rtx_vector_builder builder;
5907 {
5912 }
5914 /* Use permutes to interleave the separate vectors. */
5916 {
5919 {
5924 }
5925 }
5928 }
5930 /* Use WHILE to set a predicate register of mode MODE in which the first
5931 VL bits are set and the rest are clear. Use TARGET for the register
5932 if it's nonnull and convenient. */
5934 static rtx
5937 {
5943 }
5945 static rtx
5948 /* BUILDER is a constant predicate in which the index of every set bit
5949 is a multiple of ELT_SIZE (which is <= 8). Try to load the constant
5950 by inverting every element at a multiple of ELT_SIZE and EORing the
5951 result with an ELT_SIZE PTRUE.
5953 Return a register that contains the constant on success, otherwise
5954 return null. Use TARGET as the register if it is nonnull and
5955 convenient. */
5957 static rtx
5960 {
5961 /* Invert every element at a multiple of ELT_SIZE, keeping the
5962 other bits zero. */
5968 else
5972 /* See if we can load the constant cheaply. */
5977 /* EOR the result with an ELT_SIZE PTRUE. */
5984 }
5986 /* BUILDER is a constant predicate in which the index of every set bit
5987 is a multiple of ELT_SIZE (which is <= 8). Try to load the constant
5988 using a TRN1 of size PERMUTE_SIZE, which is >= ELT_SIZE. Return the
5989 register on success, otherwise return null. Use TARGET as the register
5990 if nonnull and convenient. */
5992 static rtx
5996 {
5997 /* We're going to split the constant into two new constants A and B,
5998 with element I of BUILDER going into A if (I & PERMUTE_SIZE) == 0
5999 and into B otherwise. E.g. for PERMUTE_SIZE == 4 && ELT_SIZE == 1:
6001 A: { 0, 1, 2, 3, _, _, _, _, 8, 9, 10, 11, _, _, _, _ }
6002 B: { 4, 5, 6, 7, _, _, _, _, 12, 13, 14, 15, _, _, _, _ }
6004 where _ indicates elements that will be discarded by the permute.
6006 First calculate the ELT_SIZEs for A and B. */
6011 {
6014 else
6016 }
6020 /* Now construct the vectors themselves. */
6028 {
6031 }
6033 {
6034 /* The A and B elements are significant. */
6037 }
6038 else
6039 {
6040 /* The A and B elements are going to be discarded, so pick whatever
6041 is likely to give a nice constant. We are targeting element
6042 sizes A_ELT_SIZE and B_ELT_SIZE for A and B respectively,
6043 with the aim of each being a sequence of ones followed by
6044 a sequence of zeros. So:
6046 * if X_ELT_SIZE <= PERMUTE_SIZE, the best approach is to
6047 duplicate the last X_ELT_SIZE element, to extend the
6048 current sequence of ones or zeros.
6050 * if X_ELT_SIZE > PERMUTE_SIZE, the best approach is to add a
6051 zero, so that the constant really does have X_ELT_SIZE and
6052 not a smaller size. */
6055 else
6059 else
6061 }
6065 /* Try loading A into a register. */
6071 /* Try loading B into a register. */
6074 {
6077 {
6080 }
6081 }
6083 /* Emit the TRN1 itself. We emit a TRN that operates on VNx16BI
6084 operands but permutes them as though they had mode MODE. */
6090 }
6092 /* Subroutine of aarch64_expand_sve_const_pred. Try to load the VNx16BI
6093 constant in BUILDER into an SVE predicate register. Return the register
6094 on success, otherwise return null. Use TARGET for the register if
6095 nonnull and convenient.
6097 ALLOW_RECURSE_P is true if we can use methods that would call this
6098 function recursively. */
6100 static rtx
6103 {
6105 /* A PFALSE or a PTRUE .B ALL. */
6110 {
6111 /* If we can load the constant using PTRUE, use it as-is. */
6116 /* Otherwise use WHILE to set the first VL bits. */
6118 }
6123 /* Try inverting the vector in element size ELT_SIZE and then EORing
6124 the result with an ELT_SIZE PTRUE. */
6127 elt_size))
6130 /* Try using TRN1 to permute two simpler constants. */
6137 }
6139 /* Return an SVE predicate register that contains the VNx16BImode
6140 constant in BUILDER, without going through the move expanders.
6142 The returned register can have whatever mode seems most natural
6143 given the contents of BUILDER. Use TARGET for the result if
6144 convenient. */
6146 static rtx
6148 {
6149 /* Try loading the constant using pure predicate operations. */
6153 /* Try forcing the constant to memory. */
6156 {
6160 }
6162 /* The last resort is to load the constant as an integer and then
6163 compare it against zero. Use -1 for set bits in order to increase
6164 the changes of using SVE DUPM or an Advanced SIMD byte mask. */
6172 }
6174 /* Set DEST to immediate IMM. */
6176 void
6178 {
6181 /* Check on what type of symbol it is. */
6182 scalar_int_mode int_mode;
6188 {
6189 rtx mem;
6190 poly_int64 offset;
6191 HOST_WIDE_INT const_offset;
6194 /* If we have (const (plus symbol offset)), separate out the offset
6195 before we start classifying the symbol. */
6198 /* We must always add an offset involving VL separately, rather than
6199 folding it into the relocation. */
6201 {
6203 {
6206 }
6211 else
6212 {
6213 /* Do arithmetic on 32-bit values if the result is smaller
6214 than that. */
6216 {
6217 /* It is invalid to do symbol calculations in modes
6218 narrower than SImode. */
6222 }
6224 {
6228 }
6229 else
6232 }
6234 }
6237 {
6238 /* We could handle non-constant offsets if they are ever
6239 generated. */
6243 }
6247 {
6254 {
6260 }
6265 /* If we aren't generating PC relative literals, then
6266 we need to expand the literal pool access carefully.
6267 This is something that needs to be done in a number
6268 of places, so could well live as a separate function. */
6270 {
6277 }
6294 {
6300 }
6301 /* FALLTHRU */
6314 }
6315 }
6318 {
6320 {
6321 /* Only the low bit of each .H, .S and .D element is defined,
6322 so we can set the upper bits to whatever we like. If the
6323 predicate is all-true in MODE, prefer to set all the undefined
6324 bits as well, so that we can share a single .B predicate for
6325 all modes. */
6329 /* All methods for constructing predicate modes wider than VNx16BI
6330 will set the upper bits of each element to zero. Expose this
6331 by moving such constants as a VNx16BI, so that all bits are
6332 significant and so that constants for different modes can be
6333 shared. The wider constant will still be available as a
6334 REG_EQUAL note. */
6335 rtx_vector_builder builder;
6337 {
6342 }
6343 }
6346 {
6349 }
6353 {
6357 }
6363 }
6366 }
6368 /* Return the MEM rtx that provides the canary value that should be used
6369 for stack-smashing protection. MODE is the mode of the memory.
6370 For SSP_GLOBAL, DECL_RTL is the MEM rtx for the canary variable
6371 (__stack_chk_guard), otherwise it has no useful value. SALT_TYPE
6372 indicates whether the caller is performing a SET or a TEST operation. */
6374 rtx
6376 aarch64_salt_type salt_type)
6377 {
6378 rtx addr;
6380 {
6383 poly_int64 offset;
6392 }
6393 else
6394 {
6395 /* Calculate the address from the system register. */
6400 else
6401 {
6404 }
6406 }
6408 }
6410 /* Emit an SVE predicated move from SRC to DEST. PRED is a predicate
6411 that is known to contain PTRUE. */
6413 void
6415 {
6423 }
6425 /* Expand a pre-RA SVE data move from SRC to DEST in which at least one
6426 operand is in memory. In this case we need to use the predicated LD1
6427 and ST1 instead of LDR and STR, both for correctness on big-endian
6428 targets and because LD1 and ST1 support a wider range of addressing modes.
6429 PRED_MODE is the mode of the predicate.
6431 See the comment at the head of aarch64-sve.md for details about the
6432 big-endian handling. */
6434 void
6436 {
6441 {
6445 else
6448 }
6450 }
6452 /* Called only on big-endian targets. See whether an SVE vector move
6453 from SRC to DEST is effectively a REV[BHW] instruction, because at
6454 least one operand is a subreg of an SVE vector that has wider or
6455 narrower elements. Return true and emit the instruction if so.
6457 For example:
6459 (set (reg:VNx8HI R1) (subreg:VNx8HI (reg:VNx16QI R2) 0))
6461 represents a VIEW_CONVERT between the following vectors, viewed
6462 in memory order:
6464 R2: { [0].high, [0].low, [1].high, [1].low, ... }
6465 R1: { [0], [1], [2], [3], ... }
6467 The high part of lane X in R2 should therefore correspond to lane X*2
6468 of R1, but the register representations are:
6470 msb lsb
6471 R2: ...... [1].high [1].low [0].high [0].low
6472 R1: ...... [3] [2] [1] [0]
6474 where the low part of lane X in R2 corresponds to lane X*2 in R1.
6475 We therefore need a reverse operation to swap the high and low values
6476 around.
6478 This is purely an optimization. Without it we would spill the
6479 subreg operand to the stack in one mode and reload it in the
6480 other mode, which has the same effect as the REV. */
6482 bool
6484 {
6487 /* Do not try to optimize subregs that LRA has created for matched
6488 reloads. These subregs only exist as a temporary measure to make
6489 the RTL well-formed, but they are exempt from the usual
6490 TARGET_CAN_CHANGE_MODE_CLASS rules.
6492 For example, if we have:
6494 (set (reg:VNx8HI R1) (foo:VNx8HI (reg:VNx4SI R2)))
6496 and the constraints require R1 and R2 to be in the same register,
6497 LRA may need to create RTL such as:
6499 (set (subreg:VNx4SI (reg:VNx8HI TMP) 0) (reg:VNx4SI R2))
6500 (set (reg:VNx8HI TMP) (foo:VNx8HI (subreg:VNx4SI (reg:VNx8HI TMP) 0)))
6501 (set (reg:VNx8HI R1) (reg:VNx8HI TMP))
6503 which forces both the input and output of the original instruction
6504 to use the same hard register. But for this to work, the normal
6505 rules have to be suppressed on the subreg input, otherwise LRA
6506 would need to reload that input too, meaning that the process
6507 would never terminate. To compensate for this, the normal rules
6508 are also suppressed for the subreg output of the first move.
6509 Ignoring the special case and handling the first move normally
6510 would therefore generate wrong code: we would reverse the elements
6511 for the first subreg but not reverse them back for the second subreg. */
6517 /* The optimization handles two single SVE REGs with different element
6518 sizes. */
6527 /* Generate *aarch64_sve_mov<mode>_subreg_be. */
6530 UNSPEC_REV_SUBREG);
6533 }
6535 /* Return a copy of X with mode MODE, without changing its other
6536 attributes. Unlike gen_lowpart, this doesn't care whether the
6537 mode change is valid. */
6539 rtx
6541 {
6548 }
6550 /* Return the SVE REV[BHW] unspec for reversing quantites of mode MODE
6551 stored in wider integer containers. */
6553 static unsigned int
6555 {
6557 {
6561 }
6563 }
6565 /* Split a *aarch64_sve_mov<mode>_subreg_be pattern with the given
6566 operands. */
6568 void
6570 {
6571 /* Decide which REV operation we need. The mode with wider elements
6572 determines the mode of the operands and the mode with the narrower
6573 elements determines the reverse width. */
6583 /* Get the operands in the appropriate modes and emit the instruction. */
6589 }
6591 static bool
6593 {
6597 /* ARM_PCS_SVE preserves strictly more than ARM_PCS_SIMD, which in
6598 turn preserves strictly more than the base PCS. The callee must
6599 preserve everything that the caller is required to preserve. */
6615 /* BTI J is needed where indirect_return functions may return
6616 if bti is enabled there. */
6623 }
6625 /* Subroutine of aarch64_pass_by_reference for arguments that are not
6626 passed in SVE registers. */
6628 static bool
6631 {
6632 HOST_WIDE_INT size;
6633 machine_mode dummymode;
6636 /* GET_MODE_SIZE (BLKmode) is useless since it is 0. */
6639 else
6640 /* No frontends can create types with variable-sized modes, so we
6641 shouldn't be asked to pass or return them. */
6644 /* Aggregates are passed by reference based on their size. */
6648 /* Variable sized arguments are always returned by reference. */
6652 /* Can this be a candidate to be passed in fp/simd register(s)? */
6658 /* Arguments which are variable sized or larger than 2 registers are
6659 passed by reference unless they are a homogenous floating point
6660 aggregate. */
6662 }
6664 /* Implement TARGET_PASS_BY_REFERENCE. */
6666 static bool
6669 {
6675 pure_scalable_type_info pst_info;
6677 {
6680 /* We can't gracefully recover at this point, so make this a
6681 fatal error. */
6685 /* Variadic SVE types are passed by reference. Normal non-variadic
6686 arguments are too if we've run out of registers. */
6698 }
6700 }
6702 /* Return TRUE if VALTYPE is padded to its least significant bits. */
6703 static bool
6705 {
6706 machine_mode dummy_mode;
6709 /* Never happens in little-endian mode. */
6713 /* Only composite types smaller than or equal to 16 bytes can
6714 be potentially returned in registers. */
6720 /* But not a composite that is an HFA (Homogeneous Floating-point Aggregate)
6721 or an HVA (Homogeneous Short-Vector Aggregate); such a special composite
6722 is always passed/returned in the least significant bits of fp/simd
6723 register(s). */
6729 /* Likewise pure scalable types for SVE vector and predicate registers. */
6730 pure_scalable_type_info pst_info;
6735 }
6737 /* Implement TARGET_FUNCTION_VALUE.
6738 Define how to find the value returned by a function. */
6740 static rtx
6743 {
6744 machine_mode mode;
6751 pure_scalable_type_info pst_info;
6755 /* Generic vectors that map to full SVE modes with -msve-vector-bits=N
6756 are returned in memory, not by value. */
6761 {
6765 {
6768 }
6769 }
6772 machine_mode ag_mode;
6775 {
6778 {
6781 }
6788 else
6789 {
6791 rtx par;
6795 {
6800 }
6802 }
6803 }
6804 else
6805 {
6807 {
6808 /* Vector types can acquire a partial SVE mode using things like
6809 __attribute__((vector_size(N))), and this is potentially useful.
6810 However, the choice of mode doesn't affect the type's ABI
6811 identity, so we should treat the types as though they had
6812 the associated integer mode, just like they did before SVE
6813 was introduced.
6815 We know that the vector must be 128 bits or smaller,
6816 otherwise we'd have returned it in memory instead. */
6825 }
6827 }
6828 }
6830 /* Implements TARGET_FUNCTION_VALUE_REGNO_P.
6831 Return true if REGNO is the number of a hard register in which the values
6832 of called function may come back. */
6834 static bool
6836 {
6837 /* Maximum of 16 bytes can be returned in the general registers. Examples
6838 of 16-byte return values are: 128-bit integers and 16-byte small
6839 structures (excluding homogeneous floating-point aggregates). */
6843 /* Up to four fp/simd registers can return a function value, e.g. a
6844 homogeneous floating-point aggregate having four members. */
6852 }
6854 /* Subroutine for aarch64_return_in_memory for types that are not returned
6855 in SVE registers. */
6857 static bool
6859 {
6860 HOST_WIDE_INT size;
6861 machine_mode ag_mode;
6868 /* Simple scalar types always returned in registers. */
6875 /* Types larger than 2 registers returned in memory. */
6878 }
6880 /* Implement TARGET_RETURN_IN_MEMORY.
6882 If the type T of the result of a function is such that
6883 void func (T arg)
6884 would require that arg be passed as a value in a register (or set of
6885 registers) according to the parameter passing rules, then the result
6886 is returned in the same registers as would be used for such an
6887 argument. */
6889 static bool
6891 {
6892 pure_scalable_type_info pst_info;
6894 {
6906 }
6908 }
6910 static bool
6913 {
6918 }
6920 /* Given MODE and TYPE of a function argument, return the alignment in
6921 bits. The idea is to suppress any stronger alignment requested by
6922 the user and opt for the natural alignment (specified in AAPCS64 \S
6923 4.1). ABI_BREAK_GCC_9 is set to the old alignment if the alignment
6924 was incorrectly calculated in versions of GCC prior to GCC 9.
6925 ABI_BREAK_GCC_13 is set to the old alignment if it was incorrectly
6926 calculated in versions between GCC 9 and GCC 13. If the alignment
6927 might have changed between GCC 13 and GCC 14, ABI_BREAK_GCC_14
6928 is the old GCC 13 alignment, otherwise it is zero.
6930 This is a helper function for local use only. */
6932 static unsigned int
6937 {
6950 {
6951 /* The ABI alignment is the natural alignment of the type, without
6952 any attributes applied. Normally this is the alignment of the
6953 TYPE_MAIN_VARIANT, but not always; see PR108910 for a counterexample.
6954 For now we just handle the known exceptions explicitly. */
6957 {
6960 }
6962 {
6965 }
6968 }
6978 {
6979 /* Note that we explicitly consider zero-sized fields here,
6980 even though they don't map to AAPCS64 machine types.
6981 For example, in:
6983 struct __attribute__((aligned(8))) empty {};
6985 struct s {
6986 [[no_unique_address]] empty e;
6987 int x;
6988 };
6990 "s" contains only one Fundamental Data Type (the int field)
6991 but gains 8-byte alignment and size thanks to "e". */
6994 {
6995 /* Take the bit-field type's alignment into account only
6996 if the user didn't reduce this field's alignment with
6997 the packed attribute. */
6999 bitfield_alignment
7003 /* Compute the alignment even if the bit-field is
7004 packed, so that we can emit a warning in case the
7005 alignment changed between GCC versions. */
7006 bitfield_alignment_with_packed
7009 }
7010 }
7012 /* Emit a warning if the alignment is different when taking the
7013 'packed' attribute into account. */
7019 {
7022 }
7025 }
7027 /* Return true if TYPE describes a _BitInt(N) or an angreggate that uses the
7028 _BitInt(N) type. These include ARRAY_TYPE's with an element that is a
7029 _BitInt(N) or an aggregate that uses it, and a RECORD_TYPE or a UNION_TYPE
7030 with a field member that is a _BitInt(N) or an aggregate that uses it.
7031 Return false otherwise. */
7033 static bool
7035 {
7042 /* If ARRAY_TYPE, check it's element type. */
7046 /* If RECORD_TYPE or UNION_TYPE, check the fields' types. */
7049 {
7054 }
7056 }
7058 /* Layout a function argument according to the AAPCS64 rules. The rule
7059 numbers refer to the rule numbers in the AAPCS64. ORIG_MODE is the
7060 mode that was originally given to us by the target hook, whereas the
7061 mode in ARG might be the result of replacing partial SVE modes with
7062 the equivalent integer mode. */
7064 static void
7066 {
7072 HOST_WIDE_INT size;
7077 /* We need to do this once per argument. */
7081 bool warn_pcs_change
7082 = (warn_psabi
7084 && (currently_expanding_function_start
7087 /* HFAs and HVAs can have an alignment greater than 16 bytes. For example:
7089 typedef struct foo {
7090 __Int8x16_t foo[2] __attribute__((aligned(32)));
7091 } foo;
7093 is still a HVA despite its larger-than-normal alignment.
7094 However, such over-aligned HFAs and HVAs are guaranteed to have
7095 no padding.
7097 If we exclude HFAs and HVAs from the discussion below, then there
7098 are several things to note:
7100 - Both the C and AAPCS64 interpretations of a type's alignment should
7101 give a value that is no greater than the type's size.
7103 - Types bigger than 16 bytes are passed indirectly.
7105 - If an argument of type T is passed indirectly, TYPE and MODE describe
7106 a pointer to T rather than T iself.
7108 It follows that the AAPCS64 alignment of TYPE must be no greater
7109 than 16 bytes.
7111 Versions prior to GCC 9.1 ignored a bitfield's underlying type
7112 and so could calculate an alignment that was too small. If this
7113 happened for TYPE then ABI_BREAK_GCC_9 is this older, too-small alignment.
7115 Although GCC 9.1 fixed that bug, it introduced a different one:
7116 it would consider the alignment of a bitfield's underlying type even
7117 if the field was packed (which should have the effect of overriding
7118 the alignment of the underlying type). This was fixed in GCC 13.1.
7120 As a result of this bug, GCC 9 to GCC 12 could calculate an alignment
7121 that was too big. If this happened for TYPE, ABI_BREAK_GCC_13 is
7122 this older, too-big alignment.
7124 Also, the fact that GCC 9 to GCC 12 considered irrelevant
7125 alignments meant they could calculate type alignments that were
7126 bigger than the type's size, contrary to the assumption above.
7127 The handling of register arguments was nevertheless (and justifiably)
7128 written to follow the assumption that the alignment can never be
7129 greater than the size. The same was not true for stack arguments;
7130 their alignment was instead handled by MIN bounds in
7131 aarch64_function_arg_boundary.
7133 The net effect is that, if GCC 9 to GCC 12 incorrectly calculated
7134 an alignment of more than 16 bytes for TYPE then:
7136 - If the argument was passed in registers, these GCC versions
7137 would treat the alignment as though it was *less than* 16 bytes.
7139 - If the argument was passed on the stack, these GCC versions
7140 would treat the alignment as though it was *equal to* 16 bytes.
7142 Both behaviors were wrong, but in different cases. */
7146 pure_scalable_type_info pst_info;
7148 {
7149 /* aarch64_function_arg_alignment has never had an effect on
7150 this case. */
7152 /* The PCS says that it is invalid to pass an SVE value to an
7153 unprototyped function. There is no ABI-defined location we
7154 can return in this case, so we have no real choice but to raise
7155 an error immediately, even though this is only a query function. */
7157 {
7161 /* Avoid repeating the message, and avoid tripping the assert
7162 below. */
7164 }
7166 /* We would have converted the argument into pass-by-reference
7167 form if it didn't fit in registers. */
7177 }
7179 /* Generic vectors that map to full SVE modes with -msve-vector-bits=N
7180 are passed by reference, not by value. */
7184 /* Vector types can acquire a partial SVE mode using things like
7185 __attribute__((vector_size(N))), and this is potentially useful.
7186 However, the choice of mode doesn't affect the type's ABI
7187 identity, so we should treat the types as though they had
7188 the associated integer mode, just like they did before SVE
7189 was introduced.
7191 We know that the vector must be 128 bits or smaller,
7192 otherwise we'd have passed it in memory instead. */
7197 /* Size in bytes, rounded to the nearest multiple of 8 bytes. */
7200 else
7201 /* No frontends can create types with variable-sized modes, so we
7202 shouldn't be asked to pass or return them. */
7208 mode,
7209 type,
7213 unsigned int alignment
7221 /* _BitInt(N) was only added in GCC 14. */
7222 bool warn_pcs_change_le_gcc14
7225 /* allocate_ncrn may be false-positive, but allocate_nvrn is quite reliable.
7226 The following code thus handles passing by SIMD/FP registers first. */
7230 /* C1 - C5 for floating point, homogenous floating point aggregates (HFA)
7231 and homogenous short-vector aggregates (HVA). */
7233 {
7234 /* aarch64_function_arg_alignment has never had an effect on
7235 this case. */
7240 {
7243 {
7246 }
7253 else
7254 {
7255 rtx par;
7259 {
7262 rtx offset = gen_int_mode
7266 }
7268 }
7270 }
7271 else
7272 {
7273 /* C.3 NSRN is set to 8. */
7276 }
7277 }
7282 /* C6 - C9. though the sign and zero extension semantics are
7283 handled elsewhere. This is the case where the argument fits
7284 entirely general registers. */
7286 {
7289 /* C.8 if the argument has an alignment of 16 then the NGRN is
7290 rounded up to the next even number. */
7293 {
7294 /* Emit a warning if the alignment changed when taking the
7295 'packed' attribute into account. */
7297 && abi_break_gcc_13
7304 && abi_break_gcc_14
7310 /* The == 16 * BITS_PER_UNIT instead of >= 16 * BITS_PER_UNIT
7311 comparison is there because for > 16 * BITS_PER_UNIT
7312 alignment nregs should be > 2 and therefore it should be
7313 passed by reference rather than value. */
7315 {
7322 }
7323 }
7325 /* If an argument with an SVE mode needs to be shifted up to the
7326 high part of the register, treat it as though it had an integer mode.
7327 Using the normal (parallel [...]) would suppress the shifting. */
7329 && BYTES_BIG_ENDIAN
7332 {
7335 }
7337 /* NREGS can be 0 when e.g. an empty structure is to be passed.
7338 A reg is still generated for it, but the caller should be smart
7339 enough not to use it. */
7344 else
7345 {
7346 rtx par;
7351 {
7359 }
7361 }
7365 }
7367 /* C.11 */
7370 /* The argument is passed on stack; record the needed number of words for
7371 this argument and align the total size if necessary. */
7372 on_stack:
7376 && abi_break_gcc_13
7383 && abi_break_gcc_14
7390 {
7393 {
7399 }
7400 }
7402 }
7404 /* Add the current argument register to the set of those that need
7405 to be saved and restored around a change to PSTATE.SM. */
7407 static void
7409 {
7412 {
7415 {
7419 }
7420 }
7421 }
7423 /* Return a parallel that contains all the registers that need to be
7424 saved around a change to PSTATE.SM. Return const0_rtx if there is
7425 no such mode switch, or if no registers need to be saved. */
7427 static rtx
7429 {
7436 }
7438 /* Implement TARGET_FUNCTION_ARG. */
7440 static rtx
7442 {
7449 {
7457 sme_mode_switch_args,
7458 shared_za_flags,
7459 shared_zt0_flags));
7460 }
7464 }
7466 void
7468 const_tree fntype,
7469 rtx libname ATTRIBUTE_UNUSED,
7470 const_tree fndecl,
7473 {
7481 {
7486 }
7487 else
7488 {
7492 }
7498 pcum->shared_za_flags
7500 pcum->shared_zt0_flags
7505 && !TARGET_FLOAT
7507 {
7514 }
7517 && !TARGET_SVE
7519 {
7520 /* We can't gracefully recover at this point, so make this a
7521 fatal error. */
7524 fndecl);
7525 else
7528 }
7529 }
7531 static void
7534 {
7539 {
7554 }
7555 }
7557 bool
7559 {
7563 }
7565 /* Implement FUNCTION_ARG_BOUNDARY. Every parameter gets at least
7566 PARM_BOUNDARY bits of alignment, but will be given anything up
7567 to STACK_BOUNDARY bits if the type requires it. This makes sure
7568 that both before and after the layout of each argument, the Next
7569 Stacked Argument Address (NSAA) will have a minimum alignment of
7570 8 bytes. */
7572 static unsigned int
7574 {
7582 /* We rely on aarch64_layout_arg and aarch64_gimplify_va_arg_expr
7583 to emit warnings about ABI incompatibility. */
7586 }
7588 /* Implement TARGET_GET_RAW_RESULT_MODE and TARGET_GET_RAW_ARG_MODE. */
7590 static fixed_size_mode
7592 {
7593 /* Don't use any non GP registers for __builtin_apply and
7594 __builtin_return if general registers only mode is requested. */
7598 /* Don't use the SVE part of the register for __builtin_apply and
7599 __builtin_return. The SVE registers aren't used by the normal PCS,
7600 so using them there would be a waste of time. The PCS extensions
7601 for SVE types are fundamentally incompatible with the
7602 __builtin_return/__builtin_apply interface. */
7605 /* For SVE PR regs, indicate that they should be ignored for
7606 __builtin_apply/__builtin_return. */
7609 }
7611 /* Implement TARGET_FUNCTION_ARG_PADDING.
7613 Small aggregate types are placed in the lowest memory address.
7615 The related parameter passing rules are B.4, C.3, C.5 and C.14. */
7617 static pad_direction
7619 {
7620 /* On little-endian targets, the least significant byte of every stack
7621 argument is passed at the lowest byte address of the stack slot. */
7625 /* Otherwise, integral, floating-point and pointer types are padded downward:
7626 the least significant byte of a stack argument is passed at the highest
7627 byte address of the stack slot. */
7634 /* Everything else padded upward, i.e. data in first byte of stack slot. */
7636 }
7638 /* Similarly, for use by BLOCK_REG_PADDING (MODE, TYPE, FIRST).
7640 It specifies padding for the last (may also be the only)
7641 element of a block move between registers and memory. If
7642 assuming the block is in the memory, padding upward means that
7643 the last element is padded after its highest significant byte,
7644 while in downward padding, the last element is padded at the
7645 its least significant byte side.
7647 Small aggregates and small complex types are always padded
7648 upwards.
7650 We don't need to worry about homogeneous floating-point or
7651 short-vector aggregates; their move is not affected by the
7652 padding direction determined here. Regardless of endianness,
7653 each element of such an aggregate is put in the least
7654 significant bits of a fp/simd register.
7656 Return !BYTES_BIG_ENDIAN if the least significant byte of the
7657 register has useful data, and return the opposite if the most
7658 significant byte does. */
7660 bool
7663 {
7665 /* Aside from pure scalable types, small composite types are always
7666 padded upward. */
7668 {
7669 HOST_WIDE_INT size;
7672 else
7673 /* No frontends can create types with variable-sized modes, so we
7674 shouldn't be asked to pass or return them. */
7677 {
7678 pure_scalable_type_info pst_info;
7682 }
7683 }
7685 /* Otherwise, use the default padding. */
7687 }
7689 static scalar_int_mode
7691 {
7693 }
7695 #define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP)
7697 /* We use the 12-bit shifted immediate arithmetic instructions so values
7698 must be multiple of (1 << 12), i.e. 4096. */
7699 #define ARITH_FACTOR 4096
7701 #if (PROBE_INTERVAL % ARITH_FACTOR) != 0
7702 #error Cannot use simple address calculation for stack probing
7703 #endif
7705 /* Emit code to probe a range of stack addresses from FIRST to FIRST+POLY_SIZE,
7706 inclusive. These are offsets from the current stack pointer. */
7708 static void
7710 {
7711 HOST_WIDE_INT size;
7713 {
7716 }
7720 /* See the same assertion on PROBE_INTERVAL above. */
7723 /* See if we have a constant small number of probes to generate. If so,
7724 that's the easy case. */
7726 {
7733 }
7735 /* The run-time loop is made up of 8 insns in the generic case while the
7736 compile-time loop is made up of 4+2*(n-2) insns for n # of intervals. */
7738 {
7743 stack_pointer_rtx,
7747 /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 2 until
7748 it exceeds SIZE. If only two probes are needed, this will not
7749 generate any code. Then probe at FIRST + SIZE. */
7751 {
7755 }
7759 {
7764 }
7765 else
7767 }
7769 /* Otherwise, do the same as above, but in a loop. Note that we must be
7770 extra careful with variables wrapping around because we might be at
7771 the very top (or the very bottom) of the address space and we have
7772 to be able to handle this case properly; in particular, we use an
7773 equality test for the loop condition. */
7774 else
7775 {
7778 /* Step 1: round SIZE to the previous multiple of the interval. */
7783 /* Step 2: compute initial and final value of the loop counter. */
7785 /* TEST_ADDR = SP + FIRST. */
7789 /* LAST_ADDR = SP + FIRST + ROUNDED_SIZE. */
7792 {
7796 }
7797 else
7801 /* Step 3: the loop
7803 do
7804 {
7805 TEST_ADDR = TEST_ADDR + PROBE_INTERVAL
7806 probe at TEST_ADDR
7807 }
7808 while (TEST_ADDR != LAST_ADDR)
7810 probes at FIRST + N * PROBE_INTERVAL for values of N from 1
7811 until it is equal to ROUNDED_SIZE. */
7816 /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time
7817 that SIZE is equal to ROUNDED_SIZE. */
7820 {
7824 {
7829 }
7830 else
7832 }
7833 }
7835 /* Make sure nothing is scheduled before we are done. */
7837 }
7839 /* Probe a range of stack addresses from REG1 to REG2 inclusive. These are
7840 absolute addresses. */
7844 {
7851 /* Loop. */
7854 HOST_WIDE_INT stack_clash_probe_interval
7857 /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */
7859 HOST_WIDE_INT interval;
7862 else
7870 /* If doing stack clash protection then we probe up by the ABI specified
7871 amount. We do this because we're dropping full pages at a time in the
7872 loop. But if we're doing non-stack clash probing, probe at SP 0. */
7875 else
7878 /* Probe at TEST_ADDR. If we're inside the loop it is always safe to probe
7879 by this amount for each iteration. */
7882 /* Test if TEST_ADDR == LAST_ADDR. */
7886 /* Branch. */
7892 }
7894 /* Emit the probe loop for doing stack clash probes and stack adjustments for
7895 SVE. This emits probes from BASE to BASE - ADJUSTMENT based on a guard size
7896 of GUARD_SIZE. When a probe is emitted it is done at most
7897 MIN_PROBE_THRESHOLD bytes from the current BASE at an interval of
7898 at most MIN_PROBE_THRESHOLD. By the end of this function
7899 BASE = BASE - ADJUSTMENT. */
7904 {
7905 /* This function is not allowed to use any instruction generation function
7906 like gen_ and friends. If you do you'll likely ICE during CFG validation,
7907 so instead emit the code you want using output_asm_insn. */
7912 /* The minimum required allocation before the residual requires probing. */
7915 /* Clamp the value down to the nearest value that can be used with a cmp. */
7930 /* Emit loop start label. */
7933 /* ADJUSTMENT < RESIDUAL_PROBE_GUARD. */
7938 /* Branch to end if not enough adjustment to probe. */
7943 /* BASE = BASE - RESIDUAL_PROBE_GUARD. */
7948 /* Probe at BASE. */
7952 /* ADJUSTMENT = ADJUSTMENT - RESIDUAL_PROBE_GUARD. */
7957 /* Branch to start if still more bytes to allocate. */
7962 /* No probe leave. */
7965 /* BASE = BASE - ADJUSTMENT. */
7970 }
7972 /* Determine whether a frame chain needs to be generated. */
7973 static bool
7975 {
7979 /* A leaf function cannot have calls or write LR. */
7982 /* Don't use a frame chain in leaf functions if leaf frame pointers
7983 are disabled. */
7988 }
7990 /* Return true if the current function should save registers above
7991 the locals area, rather than below it. */
7993 static bool
7995 {
7996 /* When using stack smash protection, make sure that the canary slot
7997 comes between the locals and the saved registers. Otherwise,
7998 it would be possible for a carefully sized smash attack to change
7999 the saved registers (particularly LR and FP) without reaching the
8000 canary. */
8002 }
8004 /* Return true if the current function needs to record the incoming
8005 value of PSTATE.SM. */
8006 static bool
8008 {
8009 /* Exit early if the incoming value of PSTATE.SM is known at
8010 compile time. */
8017 /* Non-local goto receivers are entered with PSTATE.SM equal to 0,
8018 but the function needs to return with PSTATE.SM unchanged. */
8022 /* Likewise for exception handlers. */
8023 eh_landing_pad lp;
8028 /* Non-local gotos need to set PSTATE.SM to zero. It's possible to call
8029 streaming-compatible functions without SME being available, so PSTATE.SM
8030 should only be changed if it is currently set to one. */
8038 {
8039 /* Return true if there is a call to a non-streaming-compatible
8040 function. */
8044 }
8046 }
8048 /* Mark the registers that need to be saved by the callee and calculate
8049 the size of the callee-saved registers area and frame record (both FP
8050 and LR may be omitted). */
8051 static void
8053 {
8068 /* Adjust the outgoing arguments size if required. Keep it in sync with what
8069 the mid-end is doing. */
8072 #define SLOT_NOT_REQUIRED (-2)
8073 #define SLOT_REQUIRED (-1)
8079 /* First mark all the registers that really need to be saved... */
8084 /* ... that includes the eh data registers (if needed)... */
8089 /* ... and any callee saved register that dataflow says is live. */
8101 {
8106 }
8108 /* Big-endian SVE frames need a spare predicate register in order
8109 to save Z8-Z15. Decide which register they should use. Prefer
8110 an unused argument register if possible, so that we don't force P4
8111 to be saved unnecessarily. */
8115 {
8124 }
8137 {
8141 }
8145 /* Now assign stack slots for the registers. Start with the predicate
8146 registers, since predicate LDR and STR have a relatively small
8147 offset range. These saves happen below the hard frame pointer. */
8150 {
8156 }
8160 {
8161 /* If we have any vector registers to save above the predicate registers,
8162 the offset of the vector register save slots need to be a multiple
8163 of the vector size. This lets us use the immediate forms of LDR/STR
8164 (or LD1/ST1 for big-endian).
8166 A vector register is 8 times the size of a predicate register,
8167 and we need to save a maximum of 12 predicate registers, so the
8168 first vector register will be at either #1, MUL VL or #2, MUL VL.
8170 If we don't have any vector registers to save, and we know how
8171 big the predicate save area is, we can just round it up to the
8172 next 16-byte boundary. */
8175 else
8176 {
8181 else
8183 }
8184 }
8186 /* If we need to save any SVE vector registers, add them next. */
8190 {
8196 }
8198 /* OFFSET is now the offset of the hard frame pointer from the bottom
8199 of the callee save area. */
8211 {
8215 };
8218 {
8219 /* FP and LR are placed in the linkage record. */
8222 }
8225 /* Put the LR save slot first, since it makes a good choice of probe
8226 for stack clash purposes. The idea is that the link register usually
8227 has to be saved before a call anyway, and so we lose little by
8228 stopping it from being individually shrink-wrapped. */
8236 {
8239 }
8241 /* If the current function changes the SVE vector length, ensure that the
8242 old value of the DWARF VG register is saved and available in the CFI,
8243 so that outer frames with VL-sized offsets can be processed correctly. */
8246 {
8249 }
8257 {
8259 /* If there is an alignment gap between integer and fp callee-saves,
8260 allocate the last fp register to it if possible. */
8262 && has_align_gap
8265 {
8268 }
8272 }
8282 {
8287 }
8289 /* With stack-clash, a register must be saved in non-leaf functions.
8290 The saving of the bottommost register counts as an implicit probe,
8291 which allows us to maintain the invariant described in the comment
8292 at expand_prologue. */
8296 {
8300 }
8317 /* Shadow call stack only deals with functions where the LR is pushed
8318 onto the stack and without specifying the "no_sanitize" attribute
8319 with the argument "shadow-call-stack". */
8320 frame.is_scs_enabled
8325 /* When shadow call stack is enabled, the scs_pop in the epilogue will
8326 restore x30, and we don't need to pop x30 again in the traditional
8327 way. Pop candidates record the registers that need to be popped
8328 eventually. */
8330 {
8335 }
8337 /* If candidate2 is INVALID_REGNUM, we need to adjust max_push_offset to
8338 256 to ensure that the offset meets the requirements of emit_move_insn.
8339 Similarly, if candidate1 is INVALID_REGNUM, we need to set
8340 max_push_offset to 0, because no registers are popped at this time,
8341 so callee_adjust cannot be adjusted. */
8344 {
8347 else
8349 }
8352 HOST_WIDE_INT const_saved_regs_size;
8358 {
8359 /* Simple, small frame with no data below the saved registers.
8361 stp reg1, reg2, [sp, -frame_size]!
8362 stp reg3, reg4, [sp, 16] */
8364 }
8368 /* We could handle this case even with data below the saved
8369 registers, provided that that data left us with valid offsets
8370 for all predicate and vector save slots. It's such a rare
8371 case that it hardly seems worth the effort though. */
8376 {
8377 /* Frame with small area below the saved registers:
8379 sub sp, sp, frame_size
8380 stp reg1, reg2, [sp, bytes_below_saved_regs]
8381 stp reg3, reg4, [sp, bytes_below_saved_regs + 16] */
8383 }
8386 {
8387 /* Frame in which all saves are SVE saves:
8389 sub sp, sp, frame_size - bytes_below_saved_regs
8390 save SVE registers relative to SP
8391 sub sp, sp, bytes_below_saved_regs */
8394 }
8398 {
8399 /* Frame with large area below the saved registers, or with SVE saves,
8400 but with a small area above:
8402 stp reg1, reg2, [sp, -hard_fp_offset]!
8403 stp reg3, reg4, [sp, 16]
8404 [sub sp, sp, below_hard_fp_saved_regs_size]
8405 [save SVE registers relative to SP]
8406 sub sp, sp, bytes_below_saved_regs */
8410 }
8411 else
8412 {
8413 /* General case:
8415 sub sp, sp, hard_fp_offset
8416 stp x29, x30, [sp, 0]
8417 add x29, sp, 0
8418 stp reg3, reg4, [sp, 16]
8419 [sub sp, sp, below_hard_fp_saved_regs_size]
8420 [save SVE registers relative to SP]
8421 sub sp, sp, bytes_below_saved_regs */
8425 }
8427 /* The frame is allocated in pieces, with each non-final piece
8428 including a register save at offset 0 that acts as a probe for
8429 the following piece. In addition, the save of the bottommost register
8430 acts as a probe for callees and allocas. Roll back any probes that
8431 aren't needed.
8433 A probe isn't needed if it is associated with the final allocation
8434 (including callees and allocas) that happens before the epilogue is
8435 executed. */
8439 {
8442 else
8444 }
8446 /* Make sure the individual adjustments add up to the full frame size. */
8453 {
8454 /* We've decided not to do a "real" push and pop. However,
8455 setting up the frame chain is treated as being essentially
8456 a multi-instruction push. */
8460 }
8463 }
8465 /* Return true if the register REGNO is saved on entry to
8466 the current function. */
8468 static bool
8470 {
8472 }
8474 /* Push the register number REGNO of mode MODE to the stack with write-back
8475 adjusting the stack by ADJUSTMENT. */
8477 static void
8479 HOST_WIDE_INT adjustment)
8480 {
8491 }
8493 /* Generate and return an instruction to store the pair of registers
8494 REG and REG2 of mode MODE to location BASE with write-back adjusting
8495 the stack location BASE by ADJUSTMENT. */
8497 static rtx
8499 HOST_WIDE_INT adjustment)
8500 {
8510 }
8512 /* Push registers numbered REGNO1 and REGNO2 to the stack, adjusting the
8513 stack pointer by ADJUSTMENT. */
8515 static void
8517 {
8532 }
8534 /* Load the pair of register REG, REG2 of mode MODE from stack location BASE,
8535 adjusting it by ADJUSTMENT afterwards. */
8537 static rtx
8539 HOST_WIDE_INT adjustment)
8540 {
8550 }
8552 /* Pop the two registers numbered REGNO1, REGNO2 from the stack, adjusting it
8553 afterwards by ADJUSTMENT and writing the appropriate REG_CFA_RESTORE notes
8554 into CFI_OPS. */
8556 static void
8559 {
8566 {
8570 }
8571 else
8572 {
8577 }
8578 }
8580 /* Given an ldp/stp register operand mode MODE, return a suitable mode to use
8581 for a mem rtx representing the entire pair. */
8583 static machine_mode
8585 {
8592 else
8594 }
8596 /* Given a base mem MEM with mode and address suitable for a single ldp/stp
8597 operand, return an rtx like MEM which instead represents the entire pair. */
8599 static rtx
8601 {
8606 }
8608 /* Generate and return a store pair instruction to store REG1 and REG2
8609 into memory starting at BASE_MEM. All three rtxes should have modes of the
8610 same size. */
8612 rtx
8614 {
8620 UNSPEC_STP));
8621 }
8623 /* Generate and return a load pair instruction to load a pair of
8624 registers starting at BASE_MEM into REG1 and REG2. If CODE is
8625 UNKNOWN, all three rtxes should have modes of the same size.
8626 Otherwise, CODE is {SIGN,ZERO}_EXTEND, base_mem should be in SImode,
8627 and REG{1,2} should be in DImode. */
8629 rtx
8631 {
8639 else
8645 UNSPEC_LDP_FST),
8648 UNSPEC_LDP_SND)
8649 };
8659 }
8661 /* Return TRUE if return address signing should be enabled for the current
8662 function, otherwise return FALSE. */
8664 bool
8666 {
8667 /* This function should only be called after frame laid out. */
8670 /* If signing scope is AARCH_FUNCTION_NON_LEAF, we only sign a leaf function
8671 if its LR is pushed onto stack. */
8675 }
8677 /* Only used by the arm backend. */
8679 {}
8681 /* Return TRUE if Branch Target Identification Mechanism is enabled. */
8682 bool
8684 {
8686 }
8688 /* Check if INSN is a BTI J insn. */
8689 bool
8691 {
8697 }
8699 /* Return TRUE if Guarded Control Stack is enabled. */
8700 bool
8702 {
8704 }
8706 /* Check if X (or any sub-rtx of X) is a PACIASP/PACIBSP instruction. */
8707 bool
8709 {
8715 {
8718 {
8721 {
8728 }
8730 }
8731 }
8733 }
8736 {
8738 }
8741 {
8743 }
8745 /* The caller is going to use ST1D or LD1D to save or restore an SVE
8746 register in mode MODE at BASE_RTX + OFFSET, where OFFSET is in
8747 the range [1, 16] * GET_MODE_SIZE (MODE). Prepare for this by:
8749 (1) updating BASE_RTX + OFFSET so that it is a legitimate ST1D
8750 or LD1D address
8752 (2) setting PRED to a valid predicate register for the ST1D or LD1D,
8753 if the variable isn't already nonnull
8755 (1) is needed when OFFSET is in the range [8, 16] * GET_MODE_SIZE (MODE).
8756 Handle this case using a temporary base register that is suitable for
8757 all offsets in that range. Use ANCHOR_REG as this base register if it
8758 is nonnull, otherwise create a new register and store it in ANCHOR_REG. */
8764 {
8766 {
8767 /* This is the maximum valid offset of the anchor from the base.
8768 Lower values would be valid too. */
8771 {
8775 }
8778 }
8780 {
8785 }
8786 }
8788 /* Add a REG_CFA_EXPRESSION note to INSN to say that register REG
8789 is saved at BASE + OFFSET. */
8791 static void
8794 {
8798 }
8800 /* Emit code to save the callee-saved registers in REGS. Skip any
8801 write-back candidates if SKIP_WB is true, otherwise consider only
8802 write-back candidates.
8804 The stack pointer is currently BYTES_BELOW_SP bytes above the bottom
8805 of the static frame. HARD_FP_VALID_P is true if the hard frame pointer
8806 has been set up. */
8808 static void
8812 {
8818 {
8827 };
8830 {
8832 poly_int64 offset;
8843 {
8846 }
8850 HOST_WIDE_INT const_offset;
8856 {
8860 else
8861 {
8863 {
8867 }
8869 }
8871 }
8874 stack_pointer_rtx,
8875 sp_offset));
8886 {
8892 rtx cfi_mem2
8895 stack_pointer_rtx,
8899 /* The first part of a frame-related parallel insn is always
8900 assumed to be relevant to the frame calculations;
8901 subsequent parts, are only frame-related if
8902 explicitly marked. */
8906 /* Add a REG_FRAME_RELATED_EXPR note since the unspec
8907 representation of stp cannot be understood directly by
8908 dwarf2cfi. */
8915 }
8916 else
8917 {
8919 {
8923 }
8926 else
8931 }
8935 /* Emit a fake instruction to indicate that the VG save slot has
8936 been initialized. */
8939 }
8940 }
8942 /* Emit code to restore the callee registers in REGS, ignoring pop candidates
8943 and any other registers that are handled separately. Write the appropriate
8944 REG_CFA_RESTORE notes into CFI_OPS.
8946 The stack pointer is currently BYTES_BELOW_SP bytes above the bottom
8947 of the static frame. */
8949 static void
8952 {
8954 poly_int64 offset;
8958 {
8966 /* The shadow call stack code restores LR separately. */
8971 };
8974 {
8995 {
9004 }
9009 else
9013 }
9014 }
9016 /* Return true if OFFSET is a signed 4-bit value multiplied by the size
9017 of MODE. */
9021 {
9022 HOST_WIDE_INT multiple;
9025 }
9027 /* Return true if OFFSET is a signed 6-bit value multiplied by the size
9028 of MODE. */
9032 {
9033 HOST_WIDE_INT multiple;
9036 }
9038 /* Return true if OFFSET is an unsigned 6-bit value multiplied by the size
9039 of MODE. */
9043 {
9044 HOST_WIDE_INT multiple;
9047 }
9049 /* Return true if OFFSET is a signed 7-bit value multiplied by the size
9050 of MODE. */
9052 bool
9054 {
9055 HOST_WIDE_INT multiple;
9058 }
9060 /* Return true if OFFSET is a signed 9-bit value. */
9062 bool
9064 poly_int64 offset)
9065 {
9066 HOST_WIDE_INT const_offset;
9069 }
9071 /* Return true if OFFSET is a signed 9-bit value multiplied by the size
9072 of MODE. */
9076 {
9077 HOST_WIDE_INT multiple;
9080 }
9082 /* Return true if OFFSET is an unsigned 12-bit value multiplied by the size
9083 of MODE. */
9087 {
9088 HOST_WIDE_INT multiple;
9091 }
9093 /* Implement TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS. */
9095 static sbitmap
9097 {
9102 /* The registers we need saved to the frame. */
9106 {
9107 /* Disallow shrink wrapping for registers that will be clobbered
9108 by an SMSTART SM in the prologue. */
9113 /* Punt on saves and restores that use ST1D and LD1D. We could
9114 try to be smarter, but it would involve making sure that the
9115 spare predicate register itself is safe to use at the save
9116 and restore points. Also, when a frame pointer is being used,
9117 the slots are often out of reach of ST1D and LD1D anyway. */
9124 /* Get the offset relative to the register we'll use. */
9128 /* Check that we can access the stack slot of the register with one
9129 direct load with no adjustments needed. */
9134 }
9136 /* Don't mess with the hard frame pointer. */
9140 /* If the spare predicate register used by big-endian SVE code
9141 is call-preserved, it must be saved in the main prologue
9142 before any saves that use it. */
9148 /* If registers have been chosen to be stored/restored with
9149 writeback don't interfere with them to avoid having to output explicit
9150 stack adjustment instructions. */
9159 {
9164 }
9166 /* The VG save sequence needs a temporary GPR. Punt for now on trying
9167 to find one. */
9171 }
9173 /* Implement TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB. */
9175 static sbitmap
9177 {
9185 /* Clobbered registers don't generate values in any meaningful sense,
9186 since nothing after the clobber can rely on their value. And we can't
9187 say that partially-clobbered registers are unconditionally killed,
9188 because whether they're killed or not depends on the mode of the
9189 value they're holding. Thus partially call-clobbered registers
9190 appear in neither the kill set nor the gen set.
9192 Check manually for any calls that clobber more of a register than the
9193 current function can. */
9194 function_abi_aggregator callee_abis;
9201 /* GPRs are used in a bb if they are in the IN, GEN, or KILL sets. */
9209 {
9212 /* If there is a callee-save at an adjacent offset, add it too
9213 to increase the use of LDP/STP. */
9218 {
9224 }
9225 }
9228 }
9230 /* Implement TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS.
9231 Nothing to do for aarch64. */
9233 static void
9235 {
9236 }
9238 /* Return the next set bit in BMP from START onwards. Return the total number
9239 of bits in BMP if no set bit is found at or after START. */
9241 static unsigned int
9243 {
9254 }
9256 /* Do the work for aarch64_emit_prologue_components and
9257 aarch64_emit_epilogue_components. COMPONENTS is the bitmap of registers
9258 to save/restore, PROLOGUE_P indicates whether to emit the prologue sequence
9259 for these components or the epilogue sequence. That is, it determines
9260 whether we should emit stores or loads and what kind of CFA notes to attach
9261 to the insns. Otherwise the logic for the two sequences is very
9262 similar. */
9264 static void
9266 {
9269 ? HARD_FRAME_POINTER_REGNUM
9277 {
9291 /* No more registers to handle after REGNO.
9292 Emit a single save/restore and exit. */
9294 {
9297 {
9301 else
9303 }
9305 }
9308 /* The next register is not of the same class or its offset is not
9309 mergeable with the current one into a pair. */
9316 {
9319 {
9323 else
9325 }
9329 }
9333 /* REGNO2 can be saved/restored in a pair with REGNO. */
9344 else
9348 {
9351 {
9356 }
9357 else
9358 {
9363 }
9364 }
9367 }
9368 }
9370 /* Implement TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS. */
9372 static void
9374 {
9376 }
9378 /* Implement TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS. */
9380 static void
9382 {
9384 }
9386 /* Implement TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS. */
9388 static void
9390 {
9394 }
9396 /* On AArch64 we have an ABI defined safe buffer. This constant is used to
9397 determining the probe offset for alloca. */
9399 static HOST_WIDE_INT
9401 {
9403 }
9405 /* Emit a stack tie that acts as a scheduling barrier for all previous and
9406 subsequent memory accesses and that requires the stack pointer and REG
9407 to have their current values. REG can be stack_pointer_rtx if no
9408 other register's value needs to be fixed. */
9410 static void
9412 {
9414 }
9416 /* Allocate POLY_SIZE bytes of stack space using TEMP1 and TEMP2 as scratch
9417 registers. If POLY_SIZE is not large enough to require a probe this function
9418 will only adjust the stack. When allocating the stack space
9419 FRAME_RELATED_P is then used to indicate if the allocation is frame related.
9420 FINAL_ADJUSTMENT_P indicates whether we are allocating the area below
9421 the saved registers. If we are then we ensure that any allocation
9422 larger than the ABI defined buffer needs a probe so that the
9423 invariant of having a 1KB buffer is maintained.
9425 We emit barriers after each stack adjustment to prevent optimizations from
9426 breaking the invariant that we never drop the stack more than a page. This
9427 invariant is needed to make it easier to correctly handle asynchronous
9428 events, e.g. if we were to allow the stack to be dropped by more than a page
9429 and then have multiple probes up and we take a signal somewhere in between
9430 then the signal handler doesn't know the state of the stack and can make no
9431 assumptions about which pages have been probed.
9433 FORCE_ISA_MODE is AARCH64_ISA_MODE_SM_ON if any variable component of
9434 POLY_SIZE is measured relative to the SME vector length instead of the
9435 current prevailing vector length. It is 0 otherwise. */
9437 static void
9439 poly_int64 poly_size,
9440 aarch64_isa_mode force_isa_mode,
9443 {
9445 HOST_WIDE_INT guard_size
9450 HOST_WIDE_INT min_probe_threshold
9451 = (final_adjustment_p
9456 /* We should always have a positive probe threshold. */
9460 {
9466 {
9468 }
9472 {
9474 }
9475 }
9477 /* If SIZE is not large enough to require probing, just adjust the stack and
9478 exit. */
9481 {
9483 frame_related_p);
9485 }
9487 HOST_WIDE_INT size;
9488 /* Handle the SVE non-constant case first. */
9490 {
9492 {
9496 }
9498 /* First calculate the amount of bytes we're actually spilling. */
9506 {
9507 /* This is done to provide unwinding information for the stack
9508 adjustments we're about to do, however to prevent the optimizers
9509 from removing the R11 move and leaving the CFA note (which would be
9510 very wrong) we tie the old and new stack pointer together.
9511 The tie will expand to nothing but the optimizers will not touch
9512 the instruction. */
9517 /* We want the CFA independent of the stack pointer for the
9518 duration of the loop. */
9521 }
9530 /* Now reset the CFA register if needed. */
9532 {
9537 }
9540 }
9544 "Stack clash AArch64 prologue: " HOST_WIDE_INT_PRINT_DEC
9547 /* Round size to the nearest multiple of guard_size, and calculate the
9548 residual as the difference between the original size and the rounded
9549 size. */
9553 /* We can handle a small number of allocations/probes inline. Otherwise
9554 punt to a loop. */
9556 {
9558 {
9561 guard_used_by_caller));
9563 }
9565 }
9566 else
9567 {
9568 /* Compute the ending address. */
9573 /* For the initial allocation, we don't have a frame pointer
9574 set up, so we always need CFI notes. If we're doing the
9575 final allocation, then we may have a frame pointer, in which
9576 case it is the CFA, otherwise we need CFI notes.
9578 We can determine which allocation we are doing by looking at
9579 the value of FRAME_RELATED_P since the final allocations are not
9580 frame related. */
9582 {
9583 /* We want the CFA independent of the stack pointer for the
9584 duration of the loop. */
9588 }
9590 /* This allocates and probes the stack. Note that this re-uses some of
9591 the existing Ada stack protection code. However we are guaranteed not
9592 to enter the non loop or residual branches of that code.
9594 The non-loop part won't be entered because if our allocation amount
9595 doesn't require a loop, the case above would handle it.
9597 The residual amount won't be entered because TEMP1 is a mutliple of
9598 the allocation size. The residual will always be 0. As such, the only
9599 part we are actually using from that code is the loop setup. The
9600 actual probing is done in aarch64_output_probe_stack_range. */
9604 /* Now reset the CFA register if needed. */
9606 {
9610 }
9614 }
9616 /* Handle any residuals. Residuals of at least MIN_PROBE_THRESHOLD have to
9617 be probed. This maintains the requirement that each page is probed at
9618 least once. For initial probing we probe only if the allocation is
9619 more than GUARD_SIZE - buffer, and below the saved registers we probe
9620 if the amount is larger than buffer. GUARD_SIZE - buffer + buffer ==
9621 GUARD_SIZE. This works that for any allocation that is large enough to
9622 trigger a probe here, we'll have at least one, and if they're not large
9623 enough for this code to emit anything for them, The page would have been
9624 probed by the saving of FP/LR either by this function or any callees. If
9625 we don't have any callees then we won't have more stack adjustments and so
9626 are still safe. */
9628 {
9631 /* If we're doing final adjustments, and we've done any full page
9632 allocations then any residual needs to be probed. */
9638 {
9641 "Stack clash AArch64 prologue residuals: "
9642 HOST_WIDE_INT_PRINT_DEC " bytes, probing will be required."
9646 guard_used_by_caller));
9648 }
9649 }
9650 }
9652 /* Implement TARGET_EXTRA_LIVE_ON_ENTRY. */
9654 void
9656 {
9658 {
9665 /* The only time ZA can't have live contents on entry is when
9666 the function explicitly treats it as a pure output. */
9671 /* Since ZT0 is call-clobbered, it is only live on input if
9672 it is explicitly shared, and is not a pure output. */
9677 }
9678 }
9680 /* Return 1 if the register is used by the epilogue. We need to say the
9681 return register is used, but only after epilogue generation is complete.
9682 Note that in the case of sibcalls, the values "used by the epilogue" are
9683 considered live at the start of the called function. */
9685 int
9687 {
9689 {
9692 }
9699 /* If the function shares SME state with its caller, ensure that that
9700 data is not in the lazy save buffer on exit. */
9708 }
9710 /* Implement TARGET_USE_LATE_PROLOGUE_EPILOGUE. */
9712 static bool
9714 {
9716 }
9718 /* The current function's frame has a save slot for the incoming state
9719 of SVCR. Return a legitimate memory for the slot, based on the hard
9720 frame pointer. */
9722 static rtx
9724 {
9729 /* hard fp -> bottom of frame. */
9731 /* bottom of frame -> save slot. */
9734 }
9736 /* The current function's frame has a save slot for the incoming state
9737 of SVCR. Load the slot into register REGNO and return the register. */
9739 static rtx
9741 {
9745 }
9747 /* Like the rtx version of aarch64_guard_switch_pstate_sm, but first
9748 load the incoming value of SVCR from its save slot into temporary
9749 register REGNO. */
9753 aarch64_isa_mode local_mode)
9754 {
9757 }
9759 /* AArch64 stack frames generated by this compiler look like:
9761 +-------------------------------+
9762 | |
9763 | incoming stack arguments |
9764 | |
9765 +-------------------------------+
9766 | | <-- incoming stack pointer (aligned)
9767 | callee-allocated save area |
9768 | for register varargs |
9769 | |
9770 +-------------------------------+
9771 | local variables (1) | <-- frame_pointer_rtx
9772 | |
9773 +-------------------------------+
9774 | padding (1) |
9775 +-------------------------------+
9776 | callee-saved registers |
9777 +-------------------------------+
9778 | LR' |
9779 +-------------------------------+
9780 | FP' |
9781 +-------------------------------+ <-- hard_frame_pointer_rtx (aligned)
9782 | SVE vector registers |
9783 +-------------------------------+
9784 | SVE predicate registers |
9785 +-------------------------------+
9786 | local variables (2) |
9787 +-------------------------------+
9788 | padding (2) |
9789 +-------------------------------+
9790 | dynamic allocation |
9791 +-------------------------------+
9792 | padding |
9793 +-------------------------------+
9794 | outgoing stack arguments | <-- arg_pointer
9795 | |
9796 +-------------------------------+
9797 | | <-- stack_pointer_rtx (aligned)
9799 The regions marked (1) and (2) are mutually exclusive. (2) is used
9800 when aarch64_save_regs_above_locals_p is true.
9802 Dynamic stack allocations via alloca() decrease stack_pointer_rtx
9803 but leave frame_pointer_rtx and hard_frame_pointer_rtx
9804 unchanged.
9806 By default for stack-clash we assume the guard is at least 64KB, but this
9807 value is configurable to either 4KB or 64KB. We also force the guard size to
9808 be the same as the probing interval and both values are kept in sync.
9810 With those assumptions the callee can allocate up to 63KB (or 3KB depending
9811 on the guard size) of stack space without probing.
9813 When probing is needed, we emit a probe at the start of the prologue
9814 and every PARAM_STACK_CLASH_PROTECTION_GUARD_SIZE bytes thereafter.
9816 We can also use register saves as probes. These are stored in
9817 sve_save_and_probe and hard_fp_save_and_probe.
9819 For outgoing arguments we probe if the size is larger than 1KB, such that
9820 the ABI specified buffer is maintained for the next callee.
9822 The following registers are reserved during frame layout and should not be
9823 used for any other purpose:
9825 - r11: Used by stack clash protection when SVE is enabled, and also
9826 as an anchor register when saving and restoring registers
9827 - r12(EP0) and r13(EP1): Used as temporaries for stack adjustment.
9828 - r14 and r15: Used for speculation tracking.
9829 - r16(IP0), r17(IP1): Used by indirect tailcalls.
9830 - r30(LR), r29(FP): Used by standard frame layout.
9832 These registers must be avoided in frame layout related code unless the
9833 explicit intention is to interact with one of the features listed above. */
9835 /* Generate the prologue instructions for entry into a function.
9836 Establish the stack frame by decreasing the stack pointer with a
9837 properly calculated size and, if necessary, create a frame record
9838 filled with the values of LR and previous frame pointer. The
9839 current FP is also set up if it is in use. */
9841 void
9843 {
9861 {
9862 /* Fold the SVE allocation into the initial allocation.
9863 We don't do this in aarch64_layout_arg to avoid pessimizing
9864 the epilogue code. */
9867 }
9869 /* Sign return address for functions. */
9871 {
9873 {
9882 }
9885 }
9887 /* Push return address to shadow call stack. */
9895 {
9897 {
9901 (frame_size
9903 }
9906 }
9911 /* In theory we should never have both an initial adjustment
9912 and a callee save adjustment. Verify that is the case since the
9913 code below does not handle it for -fstack-clash-protection. */
9916 /* Will only probe if the initial adjustment is larger than the guard
9917 less the amount of the guard reserved for use by the caller's
9918 outgoing args. */
9925 /* The offset of the current SP from the bottom of the static frame. */
9929 {
9930 /* The offset of the frame chain record (if any) from the current SP. */
9939 else
9944 frame_pointer_needed);
9946 {
9947 /* Variable-sized frames need to describe the save slot
9948 address using DW_CFA_expression rather than DW_CFA_offset.
9949 This means that, without taking further action, the
9950 locations of the registers that we've already saved would
9951 remain based on the stack pointer even after we redefine
9952 the CFA based on the frame pointer. We therefore need new
9953 DW_CFA_expressions to re-express the save slots with addresses
9954 based on the frame pointer. */
9958 /* Add an explicit CFA definition if this was previously
9959 implicit. */
9961 {
9965 }
9967 /* Change the save slot expressions for the registers that
9968 we've already saved. */
9973 }
9975 }
9978 emit_frame_chain);
9980 {
9983 emit_frame_chain);
9984 }
9986 {
9989 /* The VG save isn't shrink-wrapped and so serves as
9990 a probe of the initial allocation. */
9993 sve_callee_adjust,
9994 force_isa_mode,
9997 }
9999 emit_frame_chain);
10001 emit_frame_chain);
10003 /* We may need to probe the final adjustment if it is larger than the guard
10004 that is assumed by the called. */
10007 force_isa_mode,
10012 /* Save the incoming value of PSTATE.SM, if required. Code further
10013 down does this for locally-streaming functions. */
10016 {
10020 {
10024 }
10025 else
10026 {
10030 {
10033 }
10035 {
10038 }
10045 }
10046 }
10048 /* Enable PSTATE.SM, if required. */
10050 {
10053 {
10054 /* The current function is streaming-compatible. Save the
10055 original state of PSTATE.SM. */
10060 AARCH64_ISA_MODE);
10061 }
10062 aarch64_sme_mode_switch_regs args_switch;
10065 {
10068 }
10074 }
10075 }
10077 /* Return TRUE if we can use a simple_return insn.
10079 This function checks whether the callee saved stack is empty, which
10080 means no restore actions are need. The pro_and_epilogue will use
10081 this to check whether shrink-wrapping opt is feasible. */
10083 bool
10085 {
10093 }
10095 /* Generate the epilogue instructions for returning from a function.
10096 This is almost exactly the reverse of the prolog sequence, except
10097 that we need to insert barriers to avoid scheduling loads that read
10098 from a deallocated stack, and we optimize the unwind records by
10099 emitting them all together if possible. */
10100 void
10102 {
10113 /* A stack clash protection prologue may not have left EP0_REGNUM or
10114 EP1_REGNUM in a usable state. The same is true for allocations
10115 with an SVE component, since we then need both temporary registers
10116 for each allocation. For stack clash we are in a usable state if
10117 the adjustment is less than GUARD_SIZE - GUARD_USED_BY_CALLER. */
10118 HOST_WIDE_INT guard_size
10125 /* We can re-use the registers when:
10127 (a) the deallocation amount is the same as the corresponding
10128 allocation amount (which is false if we combine the initial
10129 and SVE callee save allocations in the prologue); and
10131 (b) the allocation amount doesn't need a probe (which is false
10132 if the amount is guard_size - guard_used_by_caller or greater).
10134 In such situations the register should remain live with the correct
10135 value. */
10138 && (!flag_stack_clash_protection
10143 /* We need to add memory barrier to prevent read from deallocated stack. */
10144 bool need_barrier_p
10148 /* Reset PSTATE.SM, if required. */
10150 {
10154 AARCH64_ISA_MODE);
10155 aarch64_sme_mode_switch_regs return_switch;
10166 }
10168 /* Emit a barrier to prevent loads from a deallocated stack. */
10172 {
10175 }
10177 /* Restore the stack pointer from the frame pointer if it may not
10178 be the same as the stack pointer. */
10183 /* If writeback is used when restoring callee-saves, the CFA
10184 is restored on the instruction doing the writeback. */
10186 hard_frame_pointer_rtx,
10190 else
10191 /* The case where we need to re-use the register here is very rare, so
10192 avoid the complicated condition and just always emit a move if the
10193 immediate doesn't fit. */
10196 /* Restore the vector registers before the predicate registers,
10197 so that we can use P4 as a temporary for big-endian SVE frames. */
10204 /* When shadow call stack is enabled, the scs_pop in the epilogue will
10205 restore x30, we don't need to restore x30 again in the traditional
10206 way. */
10216 /* If we have no register restore information, the CFA must have been
10217 defined in terms of the stack pointer since the end of the prologue. */
10221 {
10222 /* Emit delayed restores and set the CFA to be SP + initial_adjust. */
10228 }
10230 /* Liveness of EP0_REGNUM can not be trusted across function calls either, so
10231 add restriction on emit_move optimization to leaf functions. */
10237 {
10238 /* Emit delayed restores and reset the CFA to be SP. */
10243 }
10245 /* Pop return address from shadow call stack. */
10247 {
10254 }
10256 /* Stack adjustment for exception handler. */
10258 {
10259 /* If the EH_RETURN_TAKEN_RTX flag is set then we need
10260 to unwind the stack and jump to the handler, otherwise
10261 skip this eh_return logic and continue with normal
10262 return after the label. We have already reset the CFA
10263 to be SP; letting the CFA move during this adjustment
10264 is just as correct as retaining the CFA from the body
10265 of the function. Therefore, do nothing special. */
10268 label);
10273 EH_RETURN_STACKADJ_RTX));
10277 }
10279 /* We prefer to emit the combined return/authenticate instruction RETAA,
10280 however there are three cases in which we must instead emit an explicit
10281 authentication instruction.
10283 1) Sibcalls don't return in a normal way, so if we're about to call one
10284 we must authenticate.
10286 2) The RETAA instruction is not available without FEAT_PAuth, so if we
10287 are generating code for !TARGET_PAUTH we can't use it and must
10288 explicitly authenticate.
10289 */
10292 {
10294 {
10303 }
10306 }
10311 }
10313 /* Output code to add DELTA to the first argument, and then jump
10314 to FUNCTION. Used for C++ multiple inheritance. */
10315 static void
10317 HOST_WIDE_INT delta,
10318 HOST_WIDE_INT vcall_offset,
10319 tree function)
10320 {
10321 /* The this pointer is always in x0. Note that this differs from
10322 Arm where the this pointer maybe bumped to r1 if r0 is required
10323 to return a pointer to an aggregate. On AArch64 a result value
10324 pointer will be in x8. */
10343 else
10344 {
10349 {
10353 else
10356 }
10360 else
10367 else
10368 {
10370 Pmode);
10372 }
10376 else
10382 }
10384 /* Generate a tail call to the target function. */
10386 {
10389 }
10409 /* Stop pretending to be a post-reload pass. */
10411 }
10413 static bool
10415 {
10420 {
10424 /* Don't recurse into UNSPEC_TLS looking for TLS symbols; these are
10425 TLS offsets, not real symbol references. */
10428 }
10430 }
10433 static bool
10435 {
10439 /* There's no way to calculate VL-based values using relocations. */
10441 HOST_WIDE_INT factor;
10447 poly_int64 offset;
10450 {
10451 /* We checked for POLY_INT_CST offsets above. */
10455 else
10456 /* Avoid generating a 64-bit relocation in ILP32; leave
10457 to aarch64_expand_mov_immediate to handle it properly. */
10459 }
10462 }
10464 /* Implement TARGET_CASE_VALUES_THRESHOLD.
10465 The expansion for a table switch is quite expensive due to the number
10466 of instructions, the table lookup and hard to predict indirect jump.
10467 When optimizing for speed, and -O3 enabled, use the per-core tuning if
10468 set, otherwise use tables for >= 11 cases as a tradeoff between size and
10469 performance. When optimizing for size, use 8 for smallest codesize. */
10471 static unsigned int
10473 {
10474 /* Use the specified limit for the number of cases before using jump
10475 tables at higher optimization levels. */
10479 else
10481 }
10483 /* Return true if register REGNO is a valid index register.
10484 STRICT_P is true if REG_OK_STRICT is in effect. */
10486 bool
10488 {
10490 {
10498 }
10500 }
10502 /* Return true if register REGNO is a valid base register for mode MODE.
10503 STRICT_P is true if REG_OK_STRICT is in effect. */
10505 bool
10507 {
10509 {
10517 }
10519 /* The fake registers will be eliminated to either the stack or
10520 hard frame pointer, both of which are usually valid base registers.
10521 Reload deals with the cases where the eliminated form isn't valid. */
10526 }
10528 /* Return true if X is a valid base register for mode MODE.
10529 STRICT_P is true if REG_OK_STRICT is in effect. */
10531 static bool
10533 {
10540 }
10542 /* Return true if address offset is a valid index. If it is, fill in INFO
10543 appropriately. STRICT_P is true if REG_OK_STRICT is in effect. */
10545 static bool
10548 {
10550 rtx index;
10553 /* (reg:P) */
10556 {
10560 }
10561 /* (sign_extend:DI (reg:SI)) */
10566 {
10571 }
10572 /* (mult:DI (sign_extend:DI (reg:SI)) (const_int scale)) */
10579 {
10584 }
10585 /* (ashift:DI (sign_extend:DI (reg:SI)) (const_int shift)) */
10592 {
10597 }
10598 /* (and:DI (mult:DI (reg:DI) (const_int scale))
10599 (const_int 0xffffffff<<shift)) */
10606 {
10610 /* Avoid undefined code dealing with shift being -1. */
10614 }
10615 /* (and:DI (ashift:DI (reg:DI) (const_int shift))
10616 (const_int 0xffffffff<<shift)) */
10623 {
10629 }
10630 /* (mult:P (reg:P) (const_int scale)) */
10635 {
10639 }
10640 /* (ashift:P (reg:P) (const_int shift)) */
10645 {
10649 }
10650 else
10660 {
10664 }
10665 else
10666 {
10671 }
10675 {
10680 }
10683 }
10685 /* Return true if MODE is one of the modes for which we
10686 support LDP/STP operations. */
10688 static bool
10690 {
10697 }
10699 /* Return true if REGNO is a virtual pointer register, or an eliminable
10700 "soft" frame register. Like REGNO_PTR_FRAME_P except that we don't
10701 include stack_pointer or hard_frame_pointer. */
10702 static bool
10704 {
10709 }
10711 /* Return true if X is a valid address of type TYPE for machine mode MODE.
10712 If it is, fill in INFO appropriately. STRICT_P is true if
10713 REG_OK_STRICT is in effect. */
10715 bool
10718 aarch64_addr_query_type type)
10719 {
10722 poly_int64 offset;
10724 HOST_WIDE_INT const_size;
10726 /* Whether a vector mode is partial doesn't affect address legitimacy.
10727 Partial vectors like VNx8QImode allow the same indexed addressing
10728 mode and MUL VL addressing mode as full vectors like VNx16QImode;
10729 in both cases, MUL VL counts multiples of GET_MODE_SIZE. */
10733 /* We use load/store pair for all large int mode load/stores.
10734 TI/TF/TDmode may also use a load/store pair. */
10742 /* If we are dealing with ADDR_QUERY_LDP_STP_N that means the incoming mode
10743 corresponds to the actual size of the memory being loaded/stored and the
10744 mode of the corresponding addressing mode is half of that. */
10746 {
10753 else
10756 /* This isn't really an Advanced SIMD struct mode, but a mode
10757 used to represent the complete mem in a load/store pair. */
10759 }
10767 /* For SVE, only accept [Rn], [Rn, #offset, MUL VL] and [Rn, Rm, LSL #shift].
10768 The latter is not valid for SVE predicates, and that's rejected through
10769 allow_reg_index_p above. */
10778 {
10795 {
10802 }
10807 {
10813 /* TImode, TFmode and TDmode values are allowed in both pairs of X
10814 registers and individual Q registers. The available
10815 address modes are:
10816 X,X: 7-bit signed scaled offset
10817 Q: 9-bit signed offset
10818 We conservatively require an offset representable in either mode.
10819 When performing the check for pairs of X registers i.e. LDP/STP
10820 pass down DImode since that is the natural size of the LDP/STP
10821 instruction memory accesses. */
10831 /* A 7bit offset check because OImode will emit a ldp/stp
10832 instruction (only !TARGET_SIMD or big endian will get here).
10833 For ldp/stp instructions, the offset is scaled for the size of a
10834 single element of the pair. */
10842 /* Three 9/12 bit offsets checks because CImode will emit three
10843 ldr/str instructions (only !TARGET_SIMD or big endian will
10844 get here). */
10860 /* Two 7bit offsets checks because XImode will emit two ldp/stp
10861 instructions (only big endian will get here). */
10873 /* Make "m" use the LD1 offset range for SVE data modes, so
10874 that pre-RTL optimizers like ivopts will work to that
10875 instead of the wider LDR/STR range. */
10882 {
10883 poly_int64 end_offset = (offset
10890 end_offset)));
10891 }
10897 {
10898 poly_int64 end_offset = (offset
10903 }
10910 else
10913 }
10916 {
10917 /* Look for base + (scaled/extended) index register. */
10920 {
10923 }
10926 {
10929 }
10930 }
10951 {
10955 /* TImode, TFmode and TDmode values are allowed in both pairs of X
10956 registers and individual Q registers. The available
10957 address modes are:
10958 X,X: 7-bit signed scaled offset
10959 Q: 9-bit signed offset
10960 We conservatively require an offset representable in either mode.
10961 */
10971 else
10973 }
10979 /* load literal: pc-relative constant pool entry. Only supported
10980 for SI mode or larger. */
10986 {
10987 poly_int64 offset;
10993 }
11002 {
11003 poly_int64 offset;
11004 HOST_WIDE_INT const_offset;
11010 {
11011 /* The symbol and offset must be aligned to the access size. */
11017 {
11021 }
11027 else
11036 }
11037 }
11042 }
11043 }
11045 /* Return true if the address X is valid for a PRFM instruction.
11046 STRICT_P is true if we should do strict checking with
11047 aarch64_classify_address. */
11049 bool
11051 {
11054 /* PRFM accepts the same addresses as DImode... */
11059 /* ... except writeback forms. */
11061 }
11063 bool
11065 {
11066 poly_int64 offset;
11069 }
11071 /* Classify the base of symbolic expression X. */
11073 enum aarch64_symbol_type
11075 {
11076 rtx offset;
11080 }
11083 /* Return TRUE if X is a legitimate address for accessing memory in
11084 mode MODE. */
11085 static bool
11088 {
11092 }
11094 /* Return TRUE if X is a legitimate address of type TYPE for accessing
11095 memory in mode MODE. STRICT_P is true if REG_OK_STRICT is in effect. */
11096 bool
11098 aarch64_addr_query_type type)
11099 {
11103 }
11105 /* Implement TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT. */
11107 static bool
11109 poly_int64 orig_offset,
11110 machine_mode mode)
11111 {
11112 HOST_WIDE_INT size;
11114 {
11117 /* A general SVE offset is A * VQ + B. Remove the A component from
11118 coefficient 0 in order to get the constant B. */
11121 /* Split an out-of-range address displacement into a base and
11122 offset. Use 4KB range for 1- and 2-byte accesses and a 16KB
11123 range otherwise to increase opportunities for sharing the base
11124 address of different sizes. Unaligned accesses use the signed
11125 9-bit range, TImode/TFmode/TDmode use the intersection of signed
11126 scaled 7-bit and signed 9-bit offset. */
11131 else
11137 /* Split the offset into second_offset and the rest. */
11141 }
11142 else
11143 {
11144 /* Get the mode we should use as the basis of the range. For structure
11145 modes this is the mode of one vector. */
11147 machine_mode step_mode
11150 /* Get the "mul vl" multiplier we'd like to use. */
11154 /* LDR supports a 9-bit range, but the move patterns for
11155 structure modes require all vectors to be in range of the
11156 same base. The simplest way of accomodating that while still
11157 promoting reuse of anchor points between different modes is
11158 to use an 8-bit range unconditionally. */
11160 else
11161 /* Predicates are only handled singly, so we might as well use
11162 the full range. */
11167 /* Convert the "mul vl" multiplier into a byte offset. */
11172 /* Split the offset into second_offset and the rest. */
11176 }
11177 }
11179 /* Return the binary representation of floating point constant VALUE in INTVAL.
11180 If the value cannot be converted, return false without setting INTVAL.
11181 The conversion is done in the given MODE. */
11182 bool
11184 {
11186 /* We make a general exception for 0. */
11188 {
11191 }
11193 scalar_float_mode mode;
11197 /* Only support up to DF mode. */
11209 {
11213 }
11214 else
11219 }
11221 /* Return TRUE if rtx X is an immediate constant that can be moved using a
11222 single MOV(+MOVK) followed by an FMOV. */
11223 bool
11225 {
11230 /* Determine whether it's cheaper to write float constants as
11231 mov/movk pairs over ldr/adrp pairs. */
11237 {
11242 }
11245 }
11247 /* Return TRUE if rtx X is immediate constant 0.0 (but not in Decimal
11248 Floating Point). */
11249 bool
11251 {
11252 /* 0.0 in Decimal Floating Point cannot be represented by #0 or
11253 zr as our callers expect, so no need to check the actual
11254 value if X is of Decimal Floating Point type. */
11261 }
11263 /* Return true if X is any kind of constant zero rtx. */
11265 bool
11267 {
11270 }
11272 /* Return TRUE if rtx X is immediate constant that fits in a single
11273 MOVI immediate operation. */
11274 bool
11276 {
11280 machine_mode vmode;
11281 scalar_int_mode imode;
11286 {
11290 /* We make a general exception for 0. */
11295 }
11299 else
11302 /* use a 64 bit mode for everything except for DI/DF/DD mode, where we use
11303 a 128 bit vector mode. */
11310 }
11312 /* Return TRUE if DST and SRC with mode MODE is a valid fp move. */
11313 bool
11315 {
11329 {
11335 /* Block FP immediates which are split during expand. */
11338 }
11341 }
11343 /* Return the fixed registers used for condition codes. */
11345 static bool
11347 {
11351 }
11353 /* Return a fresh memory reference to the current function's TPIDR2 block,
11354 creating a block if necessary. */
11356 static rtx
11358 {
11360 /* The TPIDR2 block is 16 bytes in size and must be aligned to a 128-bit
11361 boundary. */
11364 }
11366 /* Return a fresh register that points to the current function's
11367 TPIDR2 block, creating a block if necessary. */
11369 static rtx
11371 {
11374 }
11376 /* Emit instructions to allocate a ZA lazy save buffer and initialize the
11377 current function's TPIDR2 block. */
11379 static void
11381 {
11384 /* The ZA save buffer is SVL.B*SVL.B bytes in size. */
11394 /* The first word of the block points to the save buffer and the second
11395 word is the number of ZA slices to save. */
11402 }
11404 /* Restore the contents of ZA from the lazy save buffer, given that
11405 register TPIDR2_BLOCK points to the current function's TPIDR2 block.
11406 PSTATE.ZA is known to be 0 and TPIDR2_EL0 is known to be null. */
11408 void
11410 {
11415 }
11417 /* Return the ZT0 save buffer, creating one if necessary. */
11419 static rtx
11421 {
11425 }
11427 /* Save ZT0 to the current function's save buffer. */
11429 static void
11431 {
11435 }
11437 /* Restore ZT0 from the current function's save buffer. FROM_LAZY_SAVE_P
11438 is true if the load is happening after a call to a private-ZA function,
11439 false if it can be treated as a normal load. */
11441 static void
11443 {
11449 }
11451 /* Implement TARGET_START_CALL_ARGS. */
11453 static void
11455 {
11459 {
11463 " option %<-march%>, or by using the %<target%>"
11465 }
11479 /* If this is a call to a private ZA function, emit a marker to
11480 indicate where any necessary set-up code could be inserted.
11481 The code itself is inserted by the mode-switching pass. */
11485 /* If this is a call to a shared-ZA function that doesn't share ZT0,
11486 save and restore ZT0 around the call. */
11491 }
11493 /* This function is used by the call expanders of the machine description.
11494 RESULT is the register in which the result is returned. It's NULL for
11495 "call" and "sibcall".
11496 MEM is the location of the function call.
11497 COOKIE is either:
11498 - a const_int that gives the argument to the call's UNSPEC_CALLEE_ABI.
11499 - a PARALLEL that contains such a const_int as its first element.
11500 The second element is a PARALLEL that lists all the argument
11501 registers that need to be saved and restored around a change
11502 in PSTATE.SM, or const0_rtx if no such switch is needed.
11503 The third and fourth elements are const_ints that contain the
11504 sharing flags for ZA and ZT0 respectively.
11505 SIBCALL indicates whether this function call is normal call or sibling call.
11506 It will generate different pattern accordingly. */
11508 void
11510 {
11512 rtvec vec;
11513 machine_mode mode;
11520 {
11525 }
11533 {
11538 }
11543 #if TARGET_PECOFF
11547 #endif
11552 /* Decide if we should generate indirect calls by loading the
11553 address of the callee into a register before performing
11554 the branch-and-link. */
11561 /* Accumulate the return values, including state that is shared via
11562 attributes. */
11565 {
11569 else
11571 }
11575 /* When calling private-ZA functions from functions with ZA state,
11576 we want to know whether the call committed a lazy save. */
11582 /* Create the new return value, if necessary. */
11584 {
11587 else
11588 {
11595 }
11596 }
11605 else
11609 UNSPEC_CALLEE_ABI);
11616 /* Check whether the call requires a change to PSTATE.SM. We can't
11617 emit the instructions to change PSTATE.SM yet, since they involve
11618 a change in vector length and a change in instruction set, which
11619 cannot be represented in RTL.
11621 For now, just record which registers will be clobbered and used
11622 by the changes to PSTATE.SM. */
11624 {
11625 aarch64_sme_mode_switch_regs args_switch;
11627 {
11630 {
11633 }
11634 }
11636 aarch64_sme_mode_switch_regs result_switch;
11654 /* Ensure that the VG save slot has been initialized. Also emit
11655 an instruction to model the effect of the temporary clobber
11656 of VG, so that the prologue/epilogue pass sees the need to
11657 save the old value. */
11663 }
11665 /* Add any ZA-related information.
11667 ZA_REGNUM represents the current function's ZA state, rather than
11668 the contents of the ZA register itself. We ensure that the function's
11669 ZA state is preserved by private-ZA call sequences, so the call itself
11670 does not use or clobber ZA_REGNUM. The same thing applies to
11671 ZT0_REGNUM. */
11673 {
11674 /* The callee requires ZA to be active if the callee is shared-ZA,
11675 otherwise it requires ZA to be dormant or off. The state of ZA is
11676 captured by a combination of SME_STATE_REGNUM, TPIDR2_SETUP_REGNUM,
11677 and ZA_SAVED_REGNUM. */
11685 /* Keep the aarch64_start/end_private_za_call markers live. */
11690 /* If the callee is a shared-ZA function, record whether it uses the
11691 current value of ZA and ZT0. */
11699 }
11700 }
11702 /* Implement TARGET_END_CALL_ARGS. */
11704 static void
11706 {
11709 /* If this is a call to a private ZA function, emit a marker to
11710 indicate where any necessary restoration code could be inserted.
11711 The code itself is inserted by the mode-switching pass. */
11715 /* If this is a call to a shared-ZA function that doesn't share ZT0,
11716 save and restore ZT0 around the call. */
11721 }
11723 /* Emit call insn with PAT and do aarch64-specific handling. */
11725 rtx_call_insn *
11727 {
11734 }
11736 machine_mode
11738 {
11742 /* All floating point compares return CCFP if it is an equality
11743 comparison, and CCFPE otherwise. */
11745 {
11747 {
11768 }
11769 }
11771 /* Equality comparisons of short modes against zero can be performed
11772 using the TST instruction with the appropriate bitmask. */
11778 /* Similarly, comparisons of zero_extends from shorter modes can
11779 be performed using an ANDS with an immediate mask. */
11786 /* Zero extracts support equality comparisons. */
11794 /* ANDS/BICS/TST support equality and all signed comparisons. */
11802 /* ADDS/SUBS correctly set N and Z flags. */
11809 /* A compare with a shifted operand. Because of canonicalization,
11810 the comparison will have to be swapped when we emit the assembly
11811 code. */
11819 /* Similarly for a negated operand, but we can only do this for
11820 equalities. */
11827 /* A test for unsigned overflow from an addition. */
11834 /* A test for unsigned overflow from an add with carry. */
11844 /* A test for signed overflow. */
11851 /* For everything else, return CCmode. */
11853 }
11855 static int
11858 int
11860 {
11867 }
11869 static int
11871 {
11873 {
11877 {
11891 }
11896 {
11908 }
11913 {
11925 }
11930 {
11940 }
11945 {
11953 }
11958 {
11964 }
11969 {
11973 }
11978 {
11982 }
11987 {
11991 }
11996 {
12000 }
12005 }
12008 }
12010 /* Return true if X is a CONST_INT, CONST_WIDE_INT or a constant vector
12011 duplicate of such constants. If so, store in RET_WI the wide_int
12012 representation of the constant paired with the inner mode of the vector mode
12013 or MODE for scalar X constants. If MODE is not provided then TImode is
12014 used. */
12016 static bool
12019 {
12023 scalar_mode smode
12027 }
12029 /* Return true if X is a scalar or a constant vector of integer
12030 immediates that represent the rounding constant used in the fixed-point
12031 arithmetic instructions.
12032 The accepted form of the constant is (1 << (C - 1)) where C is in the range
12033 [1, MODE_WIDTH/2]. */
12035 bool
12037 {
12038 wide_int rnd_cst;
12045 }
12047 /* Return true if RND is a constant vector of integer rounding constants
12048 corresponding to a constant vector of shifts, SHIFT.
12049 The relationship should be RND == (1 << (SHIFT - 1)). */
12051 bool
12053 {
12060 }
12062 bool
12064 HOST_WIDE_INT minval,
12065 HOST_WIDE_INT maxval)
12066 {
12067 rtx elt;
12071 }
12073 /* Some constants can't be made using normal mov instructions in Advanced SIMD
12074 but we can still create them in various ways. If the constant in VAL can be
12075 created using alternate methods then if possible then return true and
12076 additionally set TARGET to the rtx for the sequence if TARGET is not NULL.
12077 Otherwise return false if sequence is not possible. */
12079 bool
12081 {
12082 wide_int wval;
12087 /* For Advanced SIMD we can create an integer with only the top bit set
12088 using fneg (0.0f). */
12090 && !TARGET_SVE
12093 {
12097 /* Use the same base type as aarch64_gen_shareable_zero. */
12103 }
12106 }
12108 /* Check if the value in VAL with mode MODE can be created using special
12109 instruction sequences. */
12112 {
12114 }
12116 bool
12118 {
12120 }
12122 /* Return true if VEC is a constant in which every element is in the range
12123 [MINVAL, MAXVAL]. The elements do not need to have the same value. */
12125 static bool
12127 HOST_WIDE_INT minval,
12128 HOST_WIDE_INT maxval)
12129 {
12141 {
12146 }
12148 }
12150 /* N Z C V. */
12151 #define AARCH64_CC_V 1
12152 #define AARCH64_CC_C (1 << 1)
12153 #define AARCH64_CC_Z (1 << 2)
12154 #define AARCH64_CC_N (1 << 3)
12156 /* N Z C V flags for ccmp. Indexed by AARCH64_COND_CODE. */
12158 {
12175 };
12177 /* Print floating-point vector immediate operand X to F, negating it
12178 first if NEGATE is true. Return true on success, false if it isn't
12179 a constant we can handle. */
12181 static bool
12183 {
12184 rtx elt;
12193 /* Handle the SVE single-bit immediates specially, since they have a
12194 fixed form in the assembly syntax. */
12203 else
12204 {
12210 }
12213 }
12215 /* Return the equivalent letter for size. */
12216 static char
12218 {
12220 {
12226 }
12227 }
12229 /* Print operand X to file F in a target specific manner according to CODE.
12230 The acceptable formatting commands given by CODE are:
12231 'c': An integer or symbol address without a preceding #
12232 sign.
12233 'C': Take the duplicated element in a vector constant
12234 and print it in hex.
12235 'D': Take the duplicated element in a vector constant
12236 and print it as an unsigned integer, in decimal.
12237 'e': Print the sign/zero-extend size as a character 8->b,
12238 16->h, 32->w. Can also be used for masks:
12239 0xff->b, 0xffff->h, 0xffffffff->w.
12240 'I': If the operand is a duplicated vector constant,
12241 replace it with the duplicated scalar. If the
12242 operand is then a floating-point constant, replace
12243 it with the integer bit representation. Print the
12244 transformed constant as a signed decimal number.
12245 'p': Prints N such that 2^N == X (X must be power of 2 and
12246 const int).
12247 'P': Print the number of non-zero bits in X (a const_int).
12248 'H': Print the higher numbered register of a pair (TImode)
12249 of regs.
12250 'm': Print a condition (eq, ne, etc).
12251 'M': Same as 'm', but invert condition.
12252 'N': Take the duplicated element in a vector constant
12253 and print the negative of it in decimal.
12254 'b/h/s/d/q': Print a scalar FP/SIMD register name.
12255 'Z': Same for SVE registers. ('z' was already taken.)
12256 Note that it is not necessary to use %Z for operands
12257 that have SVE modes. The convention is to use %Z
12258 only for non-SVE (or potentially non-SVE) modes.
12259 'S/T/U/V': Print a FP/SIMD register name for a register list.
12260 The register printed is the FP/SIMD register name
12261 of X + 0/1/2/3 for S/T/U/V.
12262 'R': Print a scalar Integer/FP/SIMD register name + 1.
12263 'X': Print bottom 16 bits of integer constant in hex.
12264 'w/x': Print a general register name or the zero register
12265 (32-bit or 64-bit).
12266 '0': Print a normal operand, if it's a general register,
12267 then we assume DImode.
12268 'k': Print NZCV for conditional compare instructions.
12269 'K': Print a predicate register as pn<N> rather than p<N>
12270 'A': Output address constant representing the first
12271 argument of X, specifying a relocation offset
12272 if appropriate.
12273 'L': Output constant address specified by X
12274 with a relocation offset if appropriate.
12275 'G': Prints address of X, specifying a PC relative
12276 relocation mode if appropriate.
12277 'y': Output address of LDP or STP - this is used for
12278 some LDP/STPs which don't use a PARALLEL in their
12279 pattern (so the mode needs to be adjusted).
12280 'z': Output address of a typical LDP or STP. */
12282 static void
12284 {
12285 rtx elt;
12287 {
12291 else
12292 {
12293 poly_int64 offset;
12297 else
12299 }
12303 {
12306 {
12309 }
12318 else
12319 {
12322 }
12323 }
12327 {
12331 {
12334 }
12337 }
12342 {
12345 }
12352 {
12355 }
12358 {
12361 }
12367 {
12371 else
12372 {
12375 }
12377 }
12381 {
12383 /* CONST_TRUE_RTX means al/nv (al is the default, don't print it). */
12385 {
12389 }
12392 {
12395 }
12403 else
12405 }
12410 {
12413 }
12419 ;
12420 else
12421 {
12424 }
12435 {
12436 output_operand_lossage ("incompatible floating point / vector register operand for '%%%c'", code);
12438 }
12447 {
12450 }
12453 else
12467 else
12469 code);
12474 {
12477 }
12482 {
12483 /* Print a replicated constant in hex. */
12485 {
12488 }
12491 }
12495 {
12496 /* Print a replicated constant in decimal, treating it as
12497 unsigned. */
12499 {
12502 }
12505 }
12511 {
12514 }
12517 {
12520 }
12523 {
12526 }
12528 /* Fall through */
12532 {
12535 }
12538 {
12540 {
12543 }
12546 {
12549 else
12550 {
12551 char suffix
12556 }
12557 }
12558 else
12577 {
12580 }
12581 /* fall through */
12585 {
12588 }
12594 ;
12595 else
12596 {
12599 }
12603 /* Since we define TARGET_SUPPORTS_WIDE_INT we shouldn't ever
12604 be getting CONST_DOUBLEs holding integers. */
12607 {
12610 }
12612 {
12613 #define buf_size 20
12621 #undef buf_size
12622 }
12628 }
12636 {
12663 }
12669 {
12704 }
12710 {
12716 }
12721 {
12722 HOST_WIDE_INT cond_code;
12725 {
12728 }
12733 }
12738 {
12741 }
12747 {
12755 {
12758 }
12762 ? ADDR_QUERY_LDP_STP_N
12765 }
12771 }
12772 }
12774 /* Print address 'x' of a memory access with mode 'mode'.
12775 'op' is the context required by aarch64_classify_address. It can either be
12776 MEM for a normal memory access or PARALLEL for LDP/STP. */
12777 static bool
12779 aarch64_addr_query_type type)
12780 {
12784 /* Check all addresses are Pmode - including ILP32. */
12788 {
12791 }
12798 {
12801 {
12804 }
12808 {
12809 HOST_WIDE_INT vnum
12815 }
12828 else
12837 else
12846 else
12852 /* Writeback is only supported for fixed-width modes. */
12855 {
12878 }
12890 }
12893 }
12895 /* Print address 'x' of a memory access with mode 'mode'. */
12896 static void
12898 {
12901 }
12903 /* Implement TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */
12905 static bool
12907 {
12909 {
12912 }
12914 }
12916 bool
12918 {
12925 /* UNSPEC_TLS entries for a symbol include a LABEL_REF for the
12926 referencing instruction, but they are constant offsets, not
12927 symbols. */
12933 {
12935 {
12941 }
12944 }
12947 }
12949 /* Implement REGNO_REG_CLASS. */
12951 enum reg_class
12953 {
12990 }
12992 /* OFFSET is an address offset for mode MODE, which has SIZE bytes.
12993 If OFFSET is out of range, return an offset of an anchor point
12994 that is in range. Return 0 otherwise. */
12996 static HOST_WIDE_INT
12998 machine_mode mode)
12999 {
13000 /* Does it look like we'll need a 16-byte load/store-pair operation? */
13004 /* For offsets that aren't a multiple of the access size, the limit is
13005 -256...255. */
13007 {
13008 /* BLKmode typically uses LDP of X-registers. */
13012 }
13014 /* Small negative offsets are supported. */
13021 /* Use 12-bit offset by access size. */
13023 }
13025 static rtx
13027 {
13028 #if TARGET_PECOFF
13032 #endif
13034 /* Try to split X+CONST into Y=X+(CONST & ~mask), Y+(CONST&mask),
13035 where mask is selected by alignment and size of the offset.
13036 We try to pick as large a range for the offset as possible to
13037 maximize the chance of a CSE. However, for aligned addresses
13038 we limit the range to 4k so that structures with different sized
13039 elements are likely to use the same base. We need to be careful
13040 not to split a CONST for some forms of address expression, otherwise
13041 it will generate sub-optimal code. */
13043 /* First split X + CONST (base, offset) into (base + X) + offset. */
13045 {
13046 poly_int64 offset;
13052 }
13055 {
13061 {
13065 /* Force any scaling into a temp for CSE. */
13069 /* Let the pointer register be in op0. */
13073 /* If the pointer is virtual or frame related, then we know that
13074 virtual register instantiation or register elimination is going
13075 to apply a second constant. We want the two constants folded
13076 together easily. Therefore, emit as (OP0 + CONST) + OP1. */
13078 {
13082 }
13084 /* Otherwise, in order to encourage CSE (and thence loop strength
13085 reduce) scaled addresses, emit as (OP0 + OP1) + CONST. */
13089 }
13091 HOST_WIDE_INT size;
13093 {
13095 mode);
13097 {
13101 }
13102 }
13103 }
13106 }
13108 static reg_class_t
13110 reg_class_t rclass,
13111 machine_mode mode,
13113 {
13114 /* Use aarch64_sve_reload_mem for SVE memory reloads that cannot use
13115 LDR and STR. See the comment at the head of aarch64-sve.md for
13116 more details about the big-endian handling. */
13124 {
13127 }
13129 /* If we have to disable direct literal pool loads and stores because the
13130 function is too big, then we need a scratch register. */
13135 {
13138 }
13140 /* Without the TARGET_SIMD or TARGET_SVE instructions we cannot move a
13141 Q register to a Q register directly. We need a scratch. */
13148 && !TARGET_SIMD
13151 {
13154 }
13156 /* A TFmode, TImode or TDmode memory access should be handled via an FP_REGS
13157 because AArch64 has richer addressing modes for LDR/STR instructions
13158 than LDP/STP instructions. */
13169 }
13171 /* Implement TARGET_SECONDARY_MEMORY_NEEDED. */
13173 static bool
13175 reg_class_t class2)
13176 {
13180 {
13181 /* We can't do a 128-bit FPR-to-FPR move without TARGET_SIMD,
13182 so we can't easily split a move involving tuples of 128-bit
13183 vectors. Force the copy through memory instead.
13185 (Tuples of 64-bit vectors are fine.) */
13189 }
13191 }
13193 /* Implement TARGET_FRAME_POINTER_REQUIRED. */
13195 static bool
13197 {
13198 /* If the function needs to record the incoming value of PSTATE.SM,
13199 make sure that the slot is accessible from the frame pointer. */
13201 }
13203 static bool
13205 {
13208 /* If we need a frame pointer, ARG_POINTER_REGNUM and FRAME_POINTER_REGNUM
13209 can only eliminate to HARD_FRAME_POINTER_REGNUM. */
13213 }
13215 poly_int64
13217 {
13221 {
13227 }
13230 {
13233 }
13236 }
13239 /* Get return address without mangling. */
13241 rtx
13243 {
13245 /* Note: aarch64_return_address_signing_enabled only
13246 works after cfun->machine->frame.laid_out is set,
13247 so here we don't know if the return address will
13248 be signed or not. */
13253 }
13256 /* Implement RETURN_ADDR_RTX. We do not support moving back to a
13257 previous frame. */
13259 rtx
13261 {
13265 }
13267 static void
13269 {
13270 /* Even if the current function doesn't have branch protection, some
13271 later function might, so since this template is only generated once
13272 we have to add a BTI just in case. */
13276 {
13279 }
13280 else
13281 {
13284 }
13287 /* We always emit a speculation barrier.
13288 This is because the same trampoline template is used for every nested
13289 function. Since nested functions are not particularly common or
13290 performant we don't worry too much about the extra instructions to copy
13291 around.
13292 This is not yet a problem, since we have not yet implemented function
13293 specific attributes to choose between hardening against straight line
13294 speculation or not, but such function specific attributes are likely to
13295 happen in the future. */
13300 }
13302 static void
13304 {
13308 /* Don't need to copy the trailing D-words, we fill those in below. */
13309 /* We create our own memory address in Pmode so that `emit_block_move` can
13310 use parts of the backend which expect Pmode addresses. */
13324 /* XXX We should really define a "clear_cache" pattern and use
13325 gen_clear_cache(). */
13329 a_tramp,
13330 TRAMPOLINE_SIZE));
13331 }
13333 static unsigned char
13335 {
13336 /* ??? Logically we should only need to provide a value when
13337 HARD_REGNO_MODE_OK says that at least one register in REGCLASS
13338 can hold MODE, but at the moment we need to handle all modes.
13339 Just ignore any runtime parts for registers that can't store them. */
13343 {
13383 }
13385 }
13387 static reg_class_t
13389 {
13394 {
13400 }
13402 /* Register eliminiation can result in a request for
13403 SP+constant->FP_REGS. We cannot support such operations which
13404 use SP as source and an FP_REG as destination, so reject out
13405 right now. */
13407 {
13410 /* Look through a possible SUBREG introduced by ILP32. */
13416 POINTER_REGS));
13418 }
13421 }
13423 void
13425 {
13427 }
13429 static void
13431 {
13434 else
13435 {
13437 /* While priority is known to be in range [0, 65535], so 18 bytes
13438 would be enough, the compiler might not know that. To avoid
13439 -Wformat-truncation false positive, use a larger size. */
13446 }
13447 }
13449 static void
13451 {
13454 else
13455 {
13457 /* While priority is known to be in range [0, 65535], so 18 bytes
13458 would be enough, the compiler might not know that. To avoid
13459 -Wformat-truncation false positive, use a larger size. */
13466 }
13467 }
13471 {
13477 {
13478 {
13481 },
13482 {
13485 },
13486 {
13489 },
13490 /* We assume that DImode is only generated when not optimizing and
13491 that we don't really need 64-bit address offsets. That would
13492 imply an object file with 8GB of code in a single function! */
13493 {
13496 }
13497 };
13506 /* Need to implement table size reduction, by chaning the code below. */
13515 operands);
13518 }
13520 /* Return the asm string for an SME ZERO instruction whose 8-bit mask
13521 operand is MASK. */
13524 {
13537 };
13538 /* The last entry in the list has the form "za7.d }", but that's the
13539 same length as "za7.d, ". */
13542 {
13549 {
13551 {
13556 }
13559 }
13561 {
13565 }
13566 }
13568 }
13570 /* Return size in bits of an arithmetic operand which is shifted/scaled and
13571 masked such that it is suitable for a UXTB, UXTH, or UXTW extend
13572 operator. */
13574 int
13576 {
13578 {
13581 {
13585 }
13586 }
13588 }
13590 /* Constant pools are per function only when PC relative
13591 literal loads are true or we are in the large memory
13592 model. */
13596 {
13599 }
13601 static bool
13603 {
13604 /* We can't use blocks for constants when we're using a per-function
13605 constant pool. */
13607 }
13609 /* Select appropriate section for constants depending
13610 on where we place literal pools. */
13614 rtx x,
13616 {
13621 }
13623 /* Implement ASM_OUTPUT_POOL_EPILOGUE. */
13624 void
13626 HOST_WIDE_INT offset)
13627 {
13628 /* When using per-function literal pools, we must ensure that any code
13629 section is aligned to the minimal instruction length, lest we get
13630 errors from the assembler re "unaligned instructions". */
13633 }
13635 /* Costs. */
13637 /* Helper function for rtx cost calculation. Strip a shift expression
13638 from X. Returns the inner operand if successful, or the original
13639 expression on failure. */
13640 static rtx
13642 {
13645 /* We accept both ROTATERT and ROTATE: since the RHS must be a constant
13646 we can convert both to ROR during final output. */
13661 }
13663 /* Helper function for rtx cost calculation. Strip an extend
13664 expression from X. Returns the inner operand if successful, or the
13665 original expression on failure. We deal with a number of possible
13666 canonicalization variations here. If STRIP_SHIFT is true, then
13667 we can strip off a shift also. */
13668 static rtx
13670 {
13671 scalar_int_mode mode;
13685 /* Now handle extended register, as this may also have an optional
13686 left shift by 1..4. */
13701 }
13703 /* Helper function for rtx cost calculation. Strip extension as well as any
13704 inner VEC_SELECT high-half from X. Returns the inner vector operand if
13705 successful, or the original expression on failure. */
13706 static rtx
13708 {
13710 {
13716 }
13718 }
13720 /* Helper function for rtx cost calculation. Strip VEC_DUPLICATE as well as
13721 any subsequent extend and VEC_SELECT from X. Returns the inner scalar
13722 operand if successful, or the original expression on failure. */
13723 static rtx
13725 {
13728 {
13735 }
13737 }
13739 /* Return true iff CODE is a shift supported in combination
13740 with arithmetic instructions. */
13742 static bool
13744 {
13746 }
13749 /* Return true iff X is a cheap shift without a sign extend. */
13751 static bool
13753 {
13779 }
13781 /* Helper function for rtx cost calculation. Calculate the cost of
13782 a MULT or ASHIFT, which may be part of a compound PLUS/MINUS rtx.
13783 Return the calculated cost of the expression, recursing manually in to
13784 operands where needed. */
13786 static int
13788 {
13802 {
13805 {
13806 /* The select-operand-high-half versions of the instruction have the
13807 same cost as the three vector version - don't add the costs of the
13808 extension or selection into the costs of the multiply. */
13811 /* The by-element versions of the instruction have the same costs as
13812 the normal 3-vector version. We make an assumption that the input
13813 to the VEC_DUPLICATE is already on the FP & SIMD side. This means
13814 costing of a MUL by element pre RA is a bit optimistic. */
13817 }
13821 {
13824 /* This is to catch the SSRA costing currently flowing here. */
13825 else
13827 }
13829 }
13831 /* Integer multiply/fma. */
13833 {
13834 /* The multiply will be canonicalized as a shift, cost it as such. */
13838 {
13842 {
13844 {
13845 /* If the shift is considered cheap,
13846 then don't add any cost. */
13848 ;
13850 /* ARITH + shift-by-register. */
13853 /* ARITH + extended register. We don't have a cost field
13854 for ARITH+EXTEND+SHIFT, so use extend_arith here. */
13856 else
13857 /* ARITH + shift-by-immediate. */
13859 }
13860 else
13861 /* LSL (immediate). */
13864 }
13865 /* Strip extends as we will have costed them in the case above. */
13872 }
13874 /* MNEG or [US]MNEGL. Extract the NEG operand and indicate that it's a
13875 compound and let the below cases handle it. After all, MNEG is a
13876 special-case alias of MSUB. */
13878 {
13881 }
13883 /* Integer multiplies or FMAs have zero/sign extending variants. */
13888 {
13893 {
13895 /* SMADDL/UMADDL/UMSUBL/SMSUBL. */
13897 else
13898 /* MUL/SMULL/UMULL. */
13900 }
13903 }
13905 /* This is either an integer multiply or a MADD. In both cases
13906 we want to recurse and cost the operands. */
13911 {
13913 /* MADD/MSUB. */
13915 else
13916 /* MUL. */
13918 }
13921 }
13922 else
13923 {
13925 {
13926 /* Floating-point FMA/FMUL can also support negations of the
13927 operands, unless the rounding mode is upward or downward in
13928 which case FNMUL is different than FMUL with operand negation. */
13932 {
13937 }
13940 /* FMADD/FNMADD/FNMSUB/FMSUB. */
13942 else
13943 /* FMUL/FNMUL. */
13945 }
13950 }
13951 }
13953 static int
13955 machine_mode mode,
13956 addr_space_t as ATTRIBUTE_UNUSED,
13958 {
13966 {
13968 {
13969 /* This is a CONST or SYMBOL ref which will be split
13970 in a different way depending on the code model in use.
13971 Cost it through the generic infrastructure. */
13973 /* Divide through by the cost of one instruction to
13974 bring it to the same units as the address costs. */
13976 /* The cost is then the cost of preparing the address,
13977 followed by an immediate (possibly 0) offset. */
13979 }
13980 else
13981 {
13982 /* This is most likely a jump table from a case
13983 statement. */
13985 }
13986 }
13989 {
14000 {
14006 else
14008 }
14009 else
14028 }
14032 {
14033 /* For the sake of calculating the cost of the shifted register
14034 component, we can treat same sized modes in the same way. */
14041 else
14042 /* We can't tell, or this is a 128-bit vector. */
14044 }
14047 }
14049 /* Return the cost of a branch. If SPEED_P is true then the compiler is
14050 optimizing for speed. If PREDICTABLE_P is true then the branch is predicted
14051 to be taken. */
14053 int
14055 {
14056 /* When optimizing for speed, use the cost of unpredictable branches. */
14062 else
14064 }
14066 /* Return true if X is a zero or sign extract
14067 usable in an ADD or SUB (extended register) instruction. */
14068 static bool
14070 {
14071 /* The simple case <ARITH>, XD, XN, XM, [us]xt.
14072 No shift. */
14078 }
14080 static bool
14082 {
14084 {
14096 }
14097 }
14099 /* Return true iff X is an rtx that will match an extr instruction
14100 i.e. as described in the *extr<mode>5_insn family of patterns.
14101 OP0 and OP1 will be set to the operands of the shifts involved
14102 on success and will be NULL_RTX otherwise. */
14104 static bool
14106 {
14108 scalar_int_mode mode;
14123 {
14124 /* Canonicalise locally to ashift in op0, lshiftrt in op1. */
14136 {
14140 }
14141 }
14144 }
14146 /* Calculate the cost of calculating (if_then_else (OP0) (OP1) (OP2)),
14147 storing it in *COST. Result is true if the total cost of the operation
14148 has now been calculated. */
14149 static bool
14151 {
14152 rtx inner;
14153 rtx comparator;
14159 {
14163 }
14164 else
14165 {
14169 }
14172 {
14173 /* Conditional branch. */
14176 else
14177 {
14179 {
14181 {
14182 /* TBZ/TBNZ/CBZ/CBNZ. */
14184 /* TBZ/TBNZ. */
14187 else
14188 /* CBZ/CBNZ. */
14192 }
14195 {
14196 /* SUB and SUBS. */
14201 }
14202 }
14204 {
14205 /* TBZ/TBNZ. */
14208 }
14209 }
14210 }
14212 {
14213 /* CCMP. */
14215 {
14216 /* Increase cost of CCMP reg, 0, imm, CC to prefer CMP reg, 0. */
14220 {
14225 else
14227 }
14229 }
14231 /* It's a conditional operation based on the status flags,
14232 so it must be some flavor of CSEL. */
14234 /* CSNEG, CSINV, and CSINC are handled for free as part of CSEL. */
14240 {
14241 /* CSEL with zero-extension (*cmovdi_insn_uxtw). */
14244 }
14246 {
14249 /* CSINV/NEG with zero extend + const 0 (*csinv3_uxtw_insn3). */
14251 }
14253 {
14254 /* Use CSINV or CSINC. */
14257 }
14259 {
14260 /* Use CSINV or CSINC. */
14263 }
14268 }
14270 /* We don't know what this is, cost all operands. */
14272 }
14274 /* Check whether X is a bitfield operation of the form shift + extend that
14275 maps down to a UBFIZ/SBFIZ/UBFX/SBFX instruction. If so, return the
14276 operand to which the bitfield operation is applied. Otherwise return
14277 NULL_RTX. */
14279 static rtx
14281 {
14295 {
14313 }
14316 }
14318 /* Return true if the mask and a shift amount from an RTX of the form
14319 (x << SHFT_AMNT) & MASK are valid to combine into a UBFIZ instruction of
14320 mode MODE. See the *andim_ashift<mode>_bfiz pattern. */
14322 bool
14324 rtx shft_amnt)
14325 {
14332 }
14334 /* Return true if the masks and a shift amount from an RTX of the form
14335 ((x & MASK1) | ((y << SHIFT_AMNT) & MASK2)) are valid to combine into
14336 a BFI instruction of mode MODE. See *arch64_bfi patterns. */
14338 bool
14343 {
14346 /* Verify that there is no overlap in what bits are set in the two masks. */
14350 /* Verify that mask2 is not all zeros or ones. */
14354 /* The shift amount should always be less than the mode size. */
14357 /* Verify that the mask being shifted is contiguous and would be in the
14358 least significant bits after shifting by shft_amnt. */
14361 }
14363 /* Return true if X is an RTX representing an operation in the ABD family
14364 of instructions. */
14366 static bool
14368 {
14386 }
14388 /* Calculate the cost of calculating X, storing it in *COST. Result
14389 is true if the total cost of the operation has now been calculated. */
14390 static bool
14393 {
14398 scalar_int_mode int_mode;
14400 /* By default, assume that everything has equivalent cost to the
14401 cheapest instruction. Any additional costs are applied as a delta
14402 above this default. */
14406 {
14408 /* The cost depends entirely on the operands to SET. */
14414 {
14417 {
14431 }
14440 /* Fall through. */
14442 /* The cost is one per vector-register copied. */
14444 {
14447 }
14448 /* const0_rtx is in general free, but we will use an
14449 instruction to set a register to 0. */
14451 {
14452 /* The cost is 1 per register copied. */
14455 }
14456 else
14457 /* Cost is just the cost of the RHS of the set. */
14463 /* Bit-field insertion. Strip any redundant widening of
14464 the RHS to meet the width of the target. */
14475 {
14476 /* MOV immediate is assumed to always be cheap. */
14478 }
14479 else
14480 {
14481 /* BFM. */
14485 }
14490 /* We can't make sense of this, assume default cost. */
14493 }
14497 /* If an instruction can incorporate a constant within the
14498 instruction, the instruction's expression avoids calling
14499 rtx_cost() on the constant. If rtx_cost() is called on a
14500 constant, then it is usually because the constant must be
14501 moved into a register by one or more instructions.
14503 The exception is constant 0, which can be expressed
14504 as XZR/WZR and is therefore free. The exception to this is
14505 if we have (set (reg) (const0_rtx)) in which case we must cost
14506 the move. However, we can catch that when we cost the SET, so
14507 we don't need to consider that here. */
14510 else
14511 {
14512 /* To an approximation, building any other constant is
14513 proportionally expensive to the number of instructions
14514 required to build that constant. This is true whether we
14515 are compiling for SPEED or otherwise. */
14520 }
14525 /* First determine number of instructions to do the move
14526 as an integer constant. */
14530 {
14541 }
14544 {
14545 /* mov[df,sf]_aarch64. */
14547 /* FMOV (scalar immediate). */
14550 {
14551 /* This will be a load from memory. */
14554 else
14556 }
14557 else
14558 /* Otherwise this is +0.0. We get this using MOVI d0, #0
14559 or MOV v0.s[0], wzr - neither of which are modeled by the
14560 cost tables. Just use the default cost. */
14561 {
14562 }
14563 }
14569 {
14570 /* For loads we want the base cost of a load, plus an
14571 approximation for the additional cost of the addressing
14572 mode. */
14586 }
14594 {
14595 /* Many vector comparison operations are represented as NEG
14596 of a comparison. */
14598 {
14602 /* FACGE/FACGT. */
14606 {
14609 }
14615 }
14617 {
14618 /* FNEG. */
14620 }
14622 }
14625 {
14628 {
14629 /* CSETM. */
14632 }
14634 /* Cost this as SUB wzr, X. */
14638 }
14641 {
14642 /* Support (neg(fma...)) as a single instruction only if
14643 sign of zeros is unimportant. This matches the decision
14644 making in aarch64.md. */
14646 {
14647 /* FNMADD. */
14650 }
14652 {
14653 /* FNMUL. */
14656 }
14658 /* FNEG. */
14661 }
14668 {
14671 else
14673 }
14679 {
14683 }
14685 {
14689 }
14690 else
14691 {
14695 }
14704 {
14708 }
14711 {
14712 /* TODO: A write to the CC flags possibly costs extra, this
14713 needs encoding in the cost tables. */
14716 /* ANDS. */
14718 {
14721 }
14724 {
14725 /* ADDS (and CMN alias). */
14728 }
14731 {
14732 /* SUBS. */
14735 }
14740 {
14741 /* COMPARE of ZERO_EXTRACT form of TST-immediate.
14742 Handle it here directly rather than going to cost_logic
14743 since we know the immediate generated for the TST is valid
14744 so we can avoid creating an intermediate rtx for it only
14745 for costing purposes. */
14752 }
14755 {
14756 /* CMN. */
14763 }
14765 /* CMP.
14767 Compare can freely swap the order of operands, and
14768 canonicalization puts the more complex operation first.
14769 But the integer MINUS logic expects the shift/extend
14770 operation in op1. */
14773 {
14776 }
14778 }
14781 {
14782 /* FCMP. */
14787 {
14789 /* FCMP supports constant 0.0 for no extra cost. */
14791 }
14793 }
14796 {
14797 /* Vector compare. */
14802 {
14803 /* Vector cm (eq|ge|gt|lt|le) supports constant 0.0 for no extra
14804 cost. */
14806 }
14808 }
14812 {
14816 cost_minus:
14818 {
14821 {
14822 /* Recognise the SABD and UABD operation here.
14823 Recursion from the PLUS case will catch the accumulating
14824 forms. */
14826 {
14830 }
14831 /* SUBL2 and SUBW2.
14832 The select-operand-high-half versions of the sub instruction
14833 have the same cost as the regular three vector version -
14834 don't add the costs of the select into the costs of the sub.
14835 */
14838 }
14839 }
14843 /* Detect valid immediates. */
14849 {
14851 /* SUB(S) (immediate). */
14854 }
14856 /* Look for SUB (extended register). */
14859 {
14866 }
14870 /* Cost this as an FMA-alike operation. */
14874 {
14877 }
14882 {
14884 {
14885 /* Vector SUB. */
14887 }
14889 {
14890 /* SUB(S). */
14892 }
14894 {
14895 /* FSUB. */
14897 }
14898 }
14900 }
14903 {
14904 rtx new_op0;
14909 cost_plus:
14911 {
14912 /* ADDL2 and ADDW2. */
14915 {
14916 /* The select-operand-high-half versions of the add instruction
14917 have the same cost as the regular three vector version -
14918 don't add the costs of the select into the costs of the add.
14919 */
14922 }
14923 }
14927 {
14928 /* CSINC. */
14932 }
14937 {
14941 {
14942 /* ADD (immediate). */
14945 /* Some tunings prefer to not use the VL-based scalar ops.
14946 Increase the cost of the poly immediate to prevent their
14947 formation. */
14952 }
14954 }
14957 {
14958 /* 24-bit add in 2 instructions or 12-bit shifted add. */
14964 }
14968 /* Look for ADD (extended register). */
14971 {
14978 }
14980 /* Strip any extend, leave shifts behind as we will
14981 cost them through mult_cost. */
14986 {
14988 speed);
14990 }
14995 {
14997 {
14998 /* Vector ADD. */
15000 }
15002 {
15003 /* ADD. */
15005 }
15007 {
15008 /* FADD. */
15010 }
15011 }
15013 }
15020 {
15023 else
15025 }
15030 {
15034 {
15037 else
15039 }
15041 }
15044 {
15051 }
15052 /* Fall through. */
15055 cost_logic:
15060 {
15064 }
15072 {
15073 /* This is a UBFM/SBFM. */
15078 }
15081 {
15083 {
15084 /* We have a mask + shift version of a UBFIZ
15085 i.e. the *andim_ashift<mode>_bfiz pattern. */
15089 {
15095 }
15097 {
15098 /* We possibly get the immediate for free, this is not
15099 modelled. */
15105 }
15106 }
15107 else
15108 {
15111 /* Handle ORN, EON, or BIC. */
15117 /* If we had a shift on op0 then this is a logical-shift-
15118 by-register/immediate operation. Otherwise, this is just
15119 a logical operation. */
15121 {
15123 {
15124 /* Shift by immediate. */
15127 else
15129 }
15130 else
15132 }
15134 /* In both cases we want to cost both operands. */
15139 }
15140 }
15148 {
15149 /* Vector NOT. */
15152 }
15154 /* MVN-shifted-reg. */
15156 {
15163 }
15164 /* EON can have two forms: (xor (not a) b) but also (not (xor a b)).
15165 Handle the second form here taking care that 'a' in the above can
15166 be a shift. */
15168 {
15177 {
15180 else
15182 }
15185 }
15186 /* MVN. */
15195 /* If a value is written in SI mode, then zero extended to DI
15196 mode, the operation will in general be free as a write to
15197 a 'w' register implicitly zeroes the upper bits of an 'x'
15198 register. However, if this is
15200 (set (reg) (zero_extend (reg)))
15202 we must cost the explicit register move. */
15205 {
15208 /* If OP_COST is non-zero, then the cost of the zero extend
15209 is effectively the cost of the inner operation. Otherwise
15210 we have a MOV instruction and we take the cost from the MOV
15211 itself. This is true independently of whether we are
15212 optimizing for space or time. */
15217 }
15219 {
15220 /* All loads can zero extend to any size for free. */
15223 }
15227 {
15232 }
15235 {
15237 {
15238 /* UMOV. */
15240 }
15241 else
15242 {
15243 /* We generate an AND instead of UXTB/UXTH. */
15245 }
15246 }
15251 {
15252 /* LDRSH. */
15254 {
15261 }
15263 }
15267 {
15272 }
15275 {
15278 else
15280 }
15292 {
15294 {
15296 {
15297 /* Vector shift (immediate). */
15299 }
15300 else
15301 {
15302 /* LSL (immediate), ASR (immediate), UBMF, UBFIZ and friends.
15303 These are all aliases. */
15305 }
15306 }
15308 /* We can incorporate zero/sign extend for free. */
15315 }
15316 else
15317 {
15319 {
15321 /* Vector shift (register). */
15323 }
15324 else
15325 {
15327 /* LSLV, ASRV. */
15330 /* The register shift amount may be in a shorter mode expressed
15331 as a lowpart SUBREG. For costing purposes just look inside. */
15338 {
15340 /* We already demanded XEXP (op1, 0) to be REG_P, so
15341 don't recurse into it. */
15343 }
15344 }
15346 }
15352 {
15353 /* LDR. */
15356 }
15359 {
15360 /* ADRP, followed by ADD. */
15364 }
15367 {
15368 /* ADR. */
15371 }
15374 {
15375 /* One extra load instruction, after accessing the GOT. */
15379 }
15384 /* ADRP/ADD (immediate). */
15391 /* UBFX/SBFX. */
15393 {
15396 else
15398 }
15400 /* We can trust that the immediates used will be correct (there
15401 are no by-register forms), so we need only cost op0. */
15407 /* aarch64_rtx_mult_cost always handles recursion to its
15408 operands. */
15412 /* We can expand signed mod by power of 2 using a NEGS, two parallel
15413 ANDs and a CSNEG. Assume here that CSNEG is the same as the cost of
15414 an unconditional negate. This case should only ever be reached through
15415 the set_smod_pow2_cheap check in expmed.cc. */
15419 {
15420 /* We expand to 4 instructions. Reset the baseline. */
15428 }
15430 /* Fall-through. */
15433 {
15434 /* Slighly prefer UMOD over SMOD. */
15441 }
15448 {
15452 /* There is no integer SQRT, so only DIV and UDIV can get
15453 here. */
15455 /* Slighly prefer UDIV over SDIV. */
15457 else
15459 }
15485 {
15488 else
15490 }
15492 /* FMSUB, FNMADD, and FNMSUB are free. */
15499 /* aarch64_fnma4_elt_to_64v2df has the NEG as operand 1,
15500 and the by-element operand as operand 0. */
15504 /* Catch vector-by-element operations. The by-element operand can
15505 either be (vec_duplicate (vec_select (x))) or just
15506 (vec_select (x)), depending on whether we are multiplying by
15507 a vector or a scalar.
15509 Canonicalization is not very good in these cases, FMA4 will put the
15510 by-element operand as operand 0, FNMA4 will have it as operand 1. */
15521 /* If the remaining parameters are not registers,
15522 get the cost to put them into registers. */
15536 {
15538 {
15539 /*Vector truncate. */
15541 }
15542 else
15544 }
15549 {
15551 {
15552 /*Vector conversion. */
15554 }
15555 else
15557 }
15563 /* Strip the rounding part. They will all be implemented
15564 by the fcvt* family of instructions anyway. */
15566 {
15575 }
15578 {
15581 else
15583 }
15585 /* We can combine fmul by a power of 2 followed by a fcvt into a single
15586 fixed-point fcvt. */
15591 {
15594 }
15601 {
15602 /* ABS (vector). */
15605 }
15607 {
15610 /* FABD, which is analogous to FADD. */
15612 {
15619 }
15620 /* Simple FABS is analogous to FNEG. */
15623 }
15624 else
15625 {
15626 /* Integer ABS will either be split to
15627 two arithmetic instructions, or will be an ABS
15628 (scalar), which we don't model. */
15632 }
15638 {
15641 else
15642 {
15643 /* FMAXNM/FMINNM/FMAX/FMIN.
15644 TODO: This may not be accurate for all implementations, but
15645 we do not model this in the cost tables. */
15647 }
15648 }
15652 /* The floating point round to integer frint* instructions. */
15654 {
15659 }
15664 /* Decompose <su>muldi3_highpart. */
15666 mode == DImode
15667 /* (lshiftrt:TI */
15670 /* (mult:TI */
15672 /* (ANY_EXTEND:TI (reg:DI))
15673 (ANY_EXTEND:TI (reg:DI))) */
15680 /* (const_int 64) */
15683 {
15684 /* UMULH/SMULH. */
15692 }
15695 {
15696 /* Load using MOVI/MVNI. */
15702 }
15704 /* depending on the operation, either DUP or INS.
15705 For now, keep default costing. */
15708 /* Load using a DUP. */
15712 {
15716 /* cost subreg of 0 as free, otherwise as DUP */
15719 ;
15722 else
15725 }
15728 }
15736 }
15738 /* Wrapper around aarch64_rtx_costs, dumps the partial, or total cost
15739 calculated for X. This cost is stored in *COST. Returns true
15740 if the total cost of X was calculated. */
15741 static bool
15744 {
15749 {
15754 }
15757 }
15759 static int
15762 {
15768 /* Trest any subset of POINTER_REGS as though it were GENERAL_REGS. */
15775 /* Make RDFFR very expensive. In particular, if we know that the FFR
15776 contains a PTRUE (e.g. after a SETFFR), we must never use RDFFR
15777 as a way of obtaining a PTRUE. */
15783 /* Moves to/from sysregs are expensive, and must go via GPR. */
15789 /* Moving between GPR and stack cost is the same as GP2GP. */
15794 /* To/From the stack register, we move via the gprs. */
15802 {
15803 /* 128-bit operations on general registers require 2 instructions. */
15811 /* When AdvSIMD instructions are disabled it is not possible to move
15812 a 128-bit value directly between Q registers. This is handled in
15813 secondary reload. A general register is used as a scratch to move
15814 the upper DI value and the lower DI value is moved directly,
15815 hence the cost is the sum of three moves. */
15820 }
15830 {
15831 /* Needs a round-trip through memory, which can use LDP/STP for pairs.
15832 The cost must be greater than 2 units to indicate that direct
15833 moves aren't possible. */
15837 }
15840 }
15842 /* Implements TARGET_MEMORY_MOVE_COST. */
15843 static int
15845 {
15861 /* If the move needs to go through GPRs, add the cost of doing that. */
15864 base += (in
15871 }
15873 /* Implement TARGET_INSN_COST. We have the opportunity to do something
15874 much more productive here, such as using insn attributes to cost things.
15875 But we don't, not yet.
15877 The main point of this current definition is to make calling insn_cost
15878 on one instruction equivalent to calling seq_cost on a sequence that
15879 contains only that instruction. The default definition would instead
15880 only look at SET_SRCs, ignoring SET_DESTs.
15882 This ensures that, for example, storing a 128-bit zero vector is more
15883 expensive than storing a 128-bit vector register. A move of zero
15884 into a 128-bit vector register followed by multiple stores of that
15885 register is then cheaper than multiple stores of zero (which would
15886 use STP of XZR). This in turn allows STP Qs to be formed. */
15887 static int
15889 {
15893 }
15895 /* Implement TARGET_INIT_BUILTINS. */
15896 static void
15898 {
15901 #ifdef SUBTARGET_INIT_BUILTINS
15902 SUBTARGET_INIT_BUILTINS;
15903 #endif
15904 }
15906 /* Implement TARGET_FOLD_BUILTIN. */
15907 static tree
15909 {
15914 {
15920 }
15922 }
15924 /* Implement TARGET_GIMPLE_FOLD_BUILTIN. */
15925 static bool
15927 {
15934 {
15942 }
15949 }
15951 /* Implement TARGET_EXPAND_BUILTIN. */
15952 static rtx
15954 {
15959 {
15965 }
15967 }
15969 /* Implement TARGET_BUILTIN_DECL. */
15970 static tree
15972 {
15975 {
15981 }
15983 }
15985 /* Return true if it is safe and beneficial to use the approximate rsqrt optabs
15986 to optimize 1.0/sqrt. */
15988 static bool
15990 {
15992 && flag_unsafe_math_optimizations
15996 }
15998 /* Function to decide when to use the approximate reciprocal square root
15999 builtin. */
16001 static tree
16003 {
16011 {
16017 }
16019 }
16021 /* Emit code to perform the floating-point operation:
16023 DST = SRC1 * SRC2
16025 where all three operands are already known to be registers.
16026 If the operation is an SVE one, PTRUE is a suitable all-true
16027 predicate. */
16029 static void
16031 {
16036 else
16038 }
16040 /* Emit instruction sequence to compute either the approximate square root
16041 or its approximate reciprocal, depending on the flag RECP, and return
16042 whether the sequence was emitted or not. */
16044 bool
16046 {
16050 {
16053 }
16056 {
16063 || flag_trapping_math
16064 || !flag_unsafe_math_optimizations
16067 }
16068 else
16069 /* Caller assumes we cannot fail. */
16080 {
16081 /* When calculating the approximate square root, compare the
16082 argument with 0.0 and create a mask. */
16085 {
16090 }
16091 else
16092 {
16097 }
16098 }
16100 /* Estimate the approximate reciprocal square root. */
16104 /* Iterate over the series twice for SF and thrice for DF. */
16107 /* Optionally iterate over the series once less for faster performance
16108 while sacrificing the accuracy. */
16111 iterations--;
16113 /* Iterate over the series to calculate the approximate reciprocal square
16114 root. */
16117 {
16125 }
16128 {
16130 /* Multiply nonzero source values by the corresponding intermediate
16131 result elements, so that the final calculation is the approximate
16132 square root rather than its reciprocal. Select a zero result for
16133 zero source values, to avoid the Inf * 0 -> NaN that we'd get
16134 otherwise. */
16137 else
16138 {
16139 /* Qualify the approximate reciprocal square root when the
16140 argument is 0.0 by squashing the intermediary result to 0.0. */
16146 /* Calculate the approximate square root. */
16148 }
16149 }
16151 /* Finalize the approximation. */
16155 }
16157 /* Emit the instruction sequence to compute the approximation for the division
16158 of NUM by DEN in QUO and return whether the sequence was emitted or not. */
16160 bool
16162 {
16173 || flag_trapping_math
16174 || !flag_unsafe_math_optimizations
16186 /* Estimate the approximate reciprocal. */
16190 /* Iterate over the series twice for SF and thrice for DF. */
16193 /* Optionally iterate over the series less for faster performance,
16194 while sacrificing the accuracy. The default is 2 for DF and 1 for SF. */
16197 ? aarch64_double_recp_precision
16200 /* Iterate over the series to calculate the approximate reciprocal. */
16203 {
16208 }
16211 {
16212 /* As the approximate reciprocal of DEN is already calculated, only
16213 calculate the approximate division when NUM is not 1.0. */
16216 }
16218 /* Finalize the approximation. */
16221 }
16223 /* Emit an optimized sequence to perform a vector rotate
16224 of REG by the vector constant amount AMNT_VEC and place the result
16225 in DST. Return true iff successful. */
16227 bool
16229 {
16234 /* Don't end up here after reload. */
16236 /* Rotates by half the element width map down to REV* instructions and should
16237 always be preferred when possible. */
16241 /* 64 and 128-bit vector modes can use the XAR instruction
16242 when available. */
16244 || (TARGET_SVE2
16247 {
16249 rtx xar_op
16251 amnt_vec);
16254 }
16255 /* If none of the above, try to expand rotates by any byte amount as
16256 permutes. */
16260 }
16262 /* Return the number of instructions that can be issued per cycle. */
16263 static int
16265 {
16267 }
16269 /* Implement TARGET_SCHED_VARIABLE_ISSUE. */
16270 static int
16272 {
16284 }
16286 static int
16288 {
16292 }
16295 /* Implement TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD as
16296 autopref_multipass_dfa_lookahead_guard from haifa-sched.cc. It only
16297 has an effect if PARAM_SCHED_AUTOPREF_QUEUE_DEPTH > 0. */
16299 static int
16302 {
16304 }
16307 /* Vectorizer cost model target hooks. */
16309 /* If a vld1 from address ADDR should be recorded in vector_load_decls,
16310 return the decl that should be recorded. Return null otherwise. */
16311 tree
16313 {
16320 }
16322 /* Return true if STMT_INFO accesses a decl that is known to be the
16323 argument to a vld1 in the same function. */
16324 static bool
16326 {
16334 }
16336 /* Information about how the CPU would issue the scalar, Advanced SIMD
16337 or SVE version of a vector loop, using the scheme defined by the
16338 aarch64_base_vec_issue_info hierarchy of structures. */
16339 class aarch64_vec_op_count
16340 {
16360 /* The number of individual "general" operations. See the comments
16361 in aarch64_base_vec_issue_info for details. */
16364 /* The number of load and store operations, under the same scheme
16365 as above. */
16369 /* The minimum number of cycles needed to execute all loop-carried
16370 operations, which in the vector code become associated with
16371 reductions. */
16374 /* The number of individual predicate operations. See the comments
16375 in aarch64_sve_vec_issue_info for details. */
16379 /* The issue information for the core. */
16382 /* - If M_VEC_FLAGS is zero then this structure describes scalar code
16383 - If M_VEC_FLAGS & VEC_ADVSIMD is nonzero then this structure describes
16384 Advanced SIMD code.
16385 - If M_VEC_FLAGS & VEC_ANY_SVE is nonzero then this structure describes
16386 SVE code. */
16389 /* Assume that, when the code is executing on the core described
16390 by M_ISSUE_INFO, one iteration of the loop will handle M_VF_FACTOR
16391 times more data than the vectorizer anticipates.
16393 This is only ever different from 1 for SVE. It allows us to consider
16394 what would happen on a 256-bit SVE target even when the -mtune
16395 parameters say that the “likely” SVE length is 128 bits. */
16397 };
16405 {
16406 }
16408 /* Return the base issue information (i.e. the parts that make sense
16409 for both scalar and vector code). Return null if we have no issue
16410 information. */
16413 {
16417 }
16419 /* If the structure describes vector code and we have associated issue
16420 information, return that issue information, otherwise return null. */
16423 {
16429 }
16431 /* If the structure describes SVE code and we have associated issue
16432 information, return that issue information, otherwise return null. */
16435 {
16439 }
16441 /* Estimate the minimum number of cycles per iteration needed to rename
16442 the instructions.
16444 ??? For now this is done inline rather than via cost tables, since it
16445 isn't clear how it should be parameterized for the general case. */
16446 fractional_cost
16448 {
16452 /* + 1 for an addition. We've already counted a general op for each
16453 store, so we don't need to account for stores separately. The branch
16454 reads no registers and so does not need to be counted either.
16456 ??? This value is very much on the pessimistic side, but seems to work
16457 pretty well in practice. */
16461 }
16463 /* Like min_cycles_per_iter, but excluding predicate operations. */
16464 fractional_cost
16466 {
16477 }
16479 /* Like min_cycles_per_iter, but including only the predicate operations. */
16480 fractional_cost
16482 {
16486 }
16488 /* Estimate the minimum number of cycles needed to issue the operations.
16489 This is a very simplistic model! */
16490 fractional_cost
16492 {
16495 }
16497 /* Dump information about the structure. */
16498 void
16500 {
16517 {
16519 " estimated min cycles per iteration"
16523 " estimated min cycles per iteration"
16525 }
16530 }
16532 /* Information about vector code that we're in the process of costing. */
16534 {
16549 aarch64_vec_op_count *);
16558 /* True if we have performed one-time initialization based on the
16559 vec_info. */
16562 /* This loop uses an average operation that is not supported by SVE, but is
16563 supported by Advanced SIMD and SVE2. */
16566 /* Additional initialization costs for using gather or scatter operation in
16567 the current loop. */
16570 /* True if the vector body contains a store to a decl and if the
16571 function is known to have a vld1 from the same decl.
16573 In the Advanced SIMD ACLE, the recommended endian-agnostic way of
16574 initializing a vector is:
16576 float f[4] = { elts };
16577 float32x4_t x = vld1q_f32(f);
16579 We should strongly prefer vectorization of the initialization of f,
16580 so that the store to f and the load back can be optimized away,
16581 leaving a vectorization of { elts }. */
16584 /* Non-zero if the last operation we costed is a vector promotion or demotion.
16585 In this case the value is the number of insns in the last operation.
16587 On AArch64 vector promotion and demotions require us to first widen or
16588 narrow the input and only after that emit conversion instructions. For
16589 costing this means we need to emit the cost of the final conversions as
16590 well. */
16593 /* - If M_VEC_FLAGS is zero then we're costing the original scalar code.
16594 - If M_VEC_FLAGS & VEC_ADVSIMD is nonzero then we're costing Advanced
16595 SIMD code.
16596 - If M_VEC_FLAGS & VEC_ANY_SVE is nonzero then we're costing SVE code. */
16599 /* At the moment, we do not model LDP and STP in the vector and scalar costs.
16600 This means that code such as:
16602 a[0] = x;
16603 a[1] = x;
16605 will be costed as two scalar instructions and two vector instructions
16606 (a scalar_to_vec and an unaligned_store). For SLP, the vector form
16607 wins if the costs are equal, because of the fact that the vector costs
16608 include constant initializations whereas the scalar costs don't.
16609 We would therefore tend to vectorize the code above, even though
16610 the scalar version can use a single STP.
16612 We should eventually fix this and model LDP and STP in the main costs;
16613 see the comment in aarch64_sve_adjust_stmt_cost for some of the problems.
16614 Until then, we look specifically for code that does nothing more than
16615 STP-like operations. We cost them on that basis in addition to the
16616 normal latency-based costs.
16618 If the scalar or vector code could be a sequence of STPs +
16619 initialization, this variable counts the cost of the sequence,
16620 with 2 units per instruction. The variable is ~0U for other
16621 kinds of code. */
16624 /* On some CPUs, SVE and Advanced SIMD provide the same theoretical vector
16625 throughput, such as 4x128 Advanced SIMD vs. 2x256 SVE. In those
16626 situations, we try to predict whether an Advanced SIMD implementation
16627 of the loop could be completely unrolled and become straight-line code.
16628 If so, it is generally better to use the Advanced SIMD version rather
16629 than length-agnostic SVE, since the SVE loop would execute an unknown
16630 number of times and so could not be completely unrolled in the same way.
16632 If we're applying this heuristic, M_UNROLLED_ADVSIMD_NITERS is the
16633 number of Advanced SIMD loop iterations that would be unrolled and
16634 M_UNROLLED_ADVSIMD_STMTS estimates the total number of statements
16635 in the unrolled loop. Both values are zero if we're not applying
16636 the heuristic. */
16640 /* If we're vectorizing a loop that executes a constant number of times,
16641 this variable gives the number of times that the vector loop would
16642 iterate, otherwise it is zero. */
16645 /* Used only when vectorizing loops. Estimates the number and kind of
16646 operations that would be needed by one iteration of the scalar
16647 or vector loop. There is one entry for each tuning option of
16648 interest. */
16650 };
16657 {
16659 {
16662 {
16665 vf_factor });
16666 }
16667 }
16668 }
16670 /* Implement TARGET_VECTORIZE_CREATE_COSTS. */
16671 vector_costs *
16673 {
16675 }
16677 /* Return the appropriate SIMD costs for vectors of type VECTYPE. */
16680 {
16687 }
16689 /* Return the appropriate SIMD costs for vectors with VEC_* flags FLAGS. */
16692 {
16697 }
16699 /* If STMT_INFO is a memory reference, return the scalar memory type,
16700 otherwise return null. */
16701 static tree
16703 {
16707 }
16709 /* Decide whether to use the unrolling heuristic described above
16710 m_unrolled_advsimd_niters, updating that field if so. LOOP_VINFO
16711 describes the loop that we're vectorizing. */
16712 void
16715 {
16716 /* The heuristic only makes sense on targets that have the same
16717 vector throughput for SVE and Advanced SIMD. */
16722 /* We only want to apply the heuristic if LOOP_VINFO is being
16723 vectorized for SVE. */
16727 /* Check whether it is possible in principle to use Advanced SIMD
16728 instead. */
16732 /* We don't want to apply the heuristic to outer loops, since it's
16733 harder to track two levels of unrolling. */
16737 /* Only handle cases in which the number of Advanced SIMD iterations
16738 would be known at compile time but the number of SVE iterations
16739 would not. */
16744 /* Guess how many times the Advanced SIMD loop would iterate and make
16745 sure that it is within the complete unrolling limit. Even if the
16746 number of iterations is small enough, the number of statements might
16747 not be, which is why we need to estimate the number of statements too. */
16750 unsigned HOST_WIDE_INT unrolled_advsimd_niters
16755 /* Record that we're applying the heuristic and should try to estimate
16756 the number of statements in the Advanced SIMD loop. */
16758 }
16760 /* Do one-time initialization of the aarch64_vector_costs given that we're
16761 costing the loop vectorization described by LOOP_VINFO. */
16762 void
16764 {
16765 /* Record the number of times that the vector loop would execute,
16766 if known. */
16770 {
16774 else
16776 }
16778 /* Detect whether we're vectorizing for SVE and should apply the unrolling
16779 heuristic described above m_unrolled_advsimd_niters. */
16781 }
16783 /* Implement targetm.vectorize.builtin_vectorization_cost. */
16784 static int
16786 tree vectype,
16788 {
16799 {
16852 }
16853 }
16855 /* Return true if an access of kind KIND for STMT_INFO (or NODE if SLP)
16856 represents one vector of an LD[234] or ST[234] operation. Return the total
16857 number of vectors (2, 3 or 4) if so, otherwise return a value outside that
16858 range. */
16859 static int
16861 slp_tree node)
16862 {
16868 {
16873 }
16875 }
16877 /* Return true if creating multiple copies of STMT_INFO for Advanced SIMD
16878 vectors would produce a series of LDP or STP operations. KIND is the
16879 kind of statement that STMT_INFO represents. */
16880 static bool
16882 stmt_vec_info stmt_info)
16883 {
16885 {
16894 }
16897 }
16899 /* Return true if STMT_INFO is the second part of a two-statement multiply-add
16900 or multiply-subtract sequence that might be suitable for fusing into a
16901 single instruction. If VEC_FLAGS is zero, analyze the operation as
16902 a scalar one, otherwise analyze it as an operation on vectors with those
16903 VEC_* flags. */
16904 static bool
16907 {
16916 {
16918 /* ??? Should we try to check for a single use as well? */
16931 {
16932 /* Scalar and SVE code can tie the result to any FMLA input (or none,
16933 although that requires a MOVPRFX for SVE). However, Advanced SIMD
16934 only supports MLA forms, so will require a move if the result
16935 cannot be tied to the accumulator. The most important case in
16936 which this is true is when the accumulator input is invariant. */
16945 }
16948 };
16951 /* Advanced SIMD doesn't have FNMADD/FNMSUB/FNMLA/FNMLS, so the
16952 multiplication must be on the second operand (to form an FMLS).
16953 But if both operands are multiplications and the second operand
16954 is used more than once, we'll instead negate the second operand
16955 and use it as an accumulator for the first operand. */
16961 }
16963 /* Return true if STMT_INFO is the second part of a two-statement boolean AND
16964 expression sequence that might be suitable for fusing into a
16965 single instruction. If VEC_FLAGS is zero, analyze the operation as
16966 a scalar one, otherwise analyze it as an operation on vectors with those
16967 VEC_* flags. */
16969 static bool
16972 {
16981 {
17002 }
17004 }
17006 /* We are considering implementing STMT_INFO using SVE. If STMT_INFO is an
17007 in-loop reduction that SVE supports directly, return its latency in cycles,
17008 otherwise return zero. SVE_COSTS specifies the latencies of the relevant
17009 instructions. */
17010 static unsigned int
17012 stmt_vec_info stmt_info,
17014 {
17016 {
17022 {
17035 }
17037 }
17040 }
17042 /* STMT_INFO describes a loop-carried operation in the original scalar code
17043 that we are considering implementing as a reduction. Return one of the
17044 following values, depending on VEC_FLAGS:
17046 - If VEC_FLAGS is zero, return the loop carry latency of the original
17047 scalar operation.
17049 - If VEC_FLAGS & VEC_ADVSIMD, return the loop carry latency of the
17050 Advanced SIMD implementation.
17052 - If VEC_FLAGS & VEC_ANY_SVE, return the loop carry latency of the
17053 SVE implementation. */
17054 static unsigned int
17057 {
17063 /* If the caller is asking for the SVE latency, check for forms of reduction
17064 that only SVE can handle directly. */
17066 {
17067 unsigned int latency
17071 }
17073 /* Handle scalar costs. */
17076 {
17080 }
17082 /* Otherwise, the loop body just contains normal integer or FP operations,
17083 with a vector reduction outside the loop. */
17089 }
17091 /* STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost
17092 for STMT_INFO, which has cost kind KIND. If this is a scalar operation,
17093 try to subdivide the target-independent categorization provided by KIND
17094 to get a more accurate cost. */
17095 static fractional_cost
17097 stmt_vec_info stmt_info,
17098 fractional_cost stmt_cost)
17099 {
17100 /* Detect an extension of a loaded value. In general, we'll be able to fuse
17101 the extension with the load. */
17106 }
17108 /* STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost
17109 for the vectorized form of STMT_INFO possibly using SLP node NODE, which has
17110 cost kind KIND and which when vectorized would operate on vector type
17111 VECTYPE. Try to subdivide the target-independent categorization provided by
17112 KIND to get a more accurate cost. WHERE specifies where the cost associated
17113 with KIND occurs. */
17114 static fractional_cost
17117 tree vectype,
17119 fractional_cost stmt_cost)
17120 {
17126 /* It's generally better to avoid costing inductions, since the induction
17127 will usually be hidden by other operations. This is particularly true
17128 for things like COND_REDUCTIONS. */
17132 /* Detect cases in which vec_to_scalar is describing the extraction of a
17133 vector element in preparation for a scalar store. The store itself is
17134 costed separately. */
17138 /* Detect SVE gather loads, which are costed as a single scalar_load
17139 for each element. We therefore need to divide the full-instruction
17140 cost by the number of elements in the vector. */
17142 && sve_costs
17144 {
17146 /* Test for VNx2 modes, which have 64-bit containers. */
17150 }
17152 /* Detect cases in which a scalar_store is really storing one element
17153 in a scatter operation. */
17155 && sve_costs
17159 /* Detect cases in which vec_to_scalar represents an in-loop reduction. */
17163 {
17164 unsigned int latency
17168 }
17170 /* Detect cases in which vec_to_scalar represents a single reduction
17171 instruction like FADDP or MAXV. */
17176 {
17201 }
17203 /* Otherwise stick with the original categorization. */
17205 }
17207 /* STMT_COST is the cost calculated by aarch64_builtin_vectorization_cost
17208 for STMT_INFO, which has cost kind KIND and which when vectorized would
17209 operate on vector type VECTYPE. Adjust the cost as necessary for SVE
17210 targets. */
17211 static fractional_cost
17214 fractional_cost stmt_cost)
17215 {
17216 /* Unlike vec_promote_demote, vector_stmt conversions do not change the
17217 vector register size or number of units. Integer promotions of this
17218 type therefore map to SXT[BHW] or UXT[BHW].
17220 Most loads have extending forms that can do the sign or zero extension
17221 on the fly. Optimistically assume that a load followed by an extension
17222 will fold to this form during combine, and that the extension therefore
17223 comes for free. */
17227 /* For similar reasons, vector_stmt integer truncations are a no-op,
17228 because we can just ignore the unused upper bits of the source. */
17232 /* Advanced SIMD can load and store pairs of registers using LDP and STP,
17233 but there are no equivalent instructions for SVE. This means that
17234 (all other things being equal) 128-bit SVE needs twice as many load
17235 and store instructions as Advanced SIMD in order to process vector pairs.
17237 Also, scalar code can often use LDP and STP to access pairs of values,
17238 so it is too simplistic to say that one SVE load or store replaces
17239 VF scalar loads and stores.
17241 Ideally we would account for this in the scalar and Advanced SIMD
17242 costs by making suitable load/store pairs as cheap as a single
17243 load/store. However, that would be a very invasive change and in
17244 practice it tends to stress other parts of the cost model too much.
17245 E.g. stores of scalar constants currently count just a store,
17246 whereas stores of vector constants count a store and a vec_init.
17247 This is an artificial distinction for AArch64, where stores of
17248 nonzero scalar constants need the same kind of register invariant
17249 as vector stores.
17251 An alternative would be to double the cost of any SVE loads and stores
17252 that could be paired in Advanced SIMD (and possibly also paired in
17253 scalar code). But this tends to stress other parts of the cost model
17254 in the same way. It also means that we can fall back to Advanced SIMD
17255 even if full-loop predication would have been useful.
17257 Here we go for a more conservative version: double the costs of SVE
17258 loads and stores if one iteration of the scalar loop processes enough
17259 elements for it to use a whole number of Advanced SIMD LDP or STP
17260 instructions. This makes it very likely that the VF would be 1 for
17261 Advanced SIMD, and so no epilogue should be needed. */
17263 {
17270 }
17273 }
17275 /* STMT_COST is the cost calculated for STMT_INFO, which has cost kind KIND
17276 and which when vectorized would operate on vector type VECTYPE. Add the
17277 cost of any embedded operations. */
17278 static fractional_cost
17282 {
17284 {
17287 /* Detect cases in which a vector load or store represents an
17288 LD[234] or ST[234] instruction. */
17290 {
17302 }
17306 {
17307 /* For MLA we need to reduce the cost since MLA is 1 instruction. */
17312 /* For vector boolean ANDs with a compare operand we just need
17313 one insn. */
17316 }
17320 {
17323 else
17325 }
17326 }
17330 {
17333 else
17335 }
17338 }
17340 /* Return true if STMT_INFO is part of a reduction that has the form:
17342 r = r op ...;
17343 r = r op ...;
17345 with the single accumulator being read and written multiple times. */
17346 static bool
17348 {
17354 }
17356 /* COUNT, KIND and STMT_INFO are the same as for vector_costs::add_stmt_cost
17357 and they describe an operation in the body of a vector loop. Record issue
17358 information relating to the vector operation in OPS. */
17359 void
17363 {
17370 /* Calculate the minimum cycles per iteration imposed by a reduction
17371 operation. */
17374 {
17375 unsigned int base
17378 /* ??? Ideally we'd use a tree to reduce the copies down to 1 vector,
17379 and then accumulate that, but at the moment the loop-carried
17380 dependency includes all copies. */
17382 else
17384 }
17387 {
17388 /* Assume that multiply-adds will become a single operation. */
17392 /* Assume that bool AND with compare operands will become a single
17393 operation. */
17396 }
17398 /* Detect the case where we are using an emulated gather/scatter. When a
17399 target does not support gathers and scatters directly the vectorizer
17400 emulates these by constructing an index vector and then issuing an
17401 extraction for every lane in the vector. If the index vector is loaded
17402 from memory, the vector load and extractions are subsequently lowered by
17403 veclower into a series of scalar index loads. After the final loads are
17404 done it issues a vec_construct to recreate the vector from the scalar. For
17405 costing when we see a vec_to_scalar on a stmt with VMAT_GATHER_SCATTER we
17406 are dealing with an emulated instruction and should adjust costing
17407 properly. */
17411 {
17415 {
17417 {
17420 {
17422 {
17425 else
17426 /* Stores want to count both the index to array and data to
17427 array using vec_to_scalar. However we have index stores
17428 in Adv.SIMD and so we only want to adjust the index
17429 loads. */
17432 }
17434 }
17435 }
17437 }
17438 }
17440 /* Count the basic operation cost associated with KIND. */
17442 {
17447 /* We currently don't expect these to be used in a loop body. */
17475 }
17477 /* Add any embedded comparison operations. */
17482 /* COND_REDUCTIONS need two sets of VEC_COND_EXPRs, whereas so far we
17483 have only accounted for one. */
17488 /* Count the predicate operations needed by an SVE comparison. */
17491 {
17496 }
17498 /* Add any extra overhead associated with LD[234] and ST[234] operations. */
17501 {
17513 }
17515 /* Add any overhead associated with gather loads and scatter stores. */
17519 {
17523 }
17524 }
17526 /* Return true if STMT_INFO contains a memory access and if the constant
17527 component of the memory address is aligned to SIZE bytes. */
17528 static bool
17530 poly_uint64 size)
17531 {
17538 /* Needed for gathers & scatters, for example. */
17543 }
17545 /* Check if a scalar or vector stmt could be part of a region of code
17546 that does nothing more than store values to memory, in the scalar
17547 case using STP. Return the cost of the stmt if so, counting 2 for
17548 one instruction. Return ~0U otherwise.
17550 The arguments are a subset of those passed to add_stmt_cost. */
17551 unsigned int
17554 {
17555 /* Code that stores vector constants uses a vector_load to create
17556 the constant. We don't apply the heuristic to that case for two
17557 main reasons:
17559 - At the moment, STPs are only formed via peephole2, and the
17560 constant scalar moves would often come between STRs and so
17561 prevent STP formation.
17563 - The scalar code also has to load the constant somehow, and that
17564 isn't costed. */
17566 {
17568 /* Count 2 insns for a GPR->SIMD dup and 1 insn for a FPR->SIMD dup. */
17573 /* Count 1 insn for the maximum number of FP->SIMD INS
17574 instructions. */
17577 /* Count 2 insns for a GPR->SIMD move and 2 insns for the
17578 maximum number of GPR->SIMD INS instructions. */
17583 /* Count 1 insn per vector if we can't form STP Q pairs. */
17588 {
17589 /* Assume we won't be able to use STP if the constant offset
17590 component of the address is misaligned. ??? This could be
17591 removed if we formed STP pairs earlier, rather than relying
17592 on peephole2. */
17596 }
17601 {
17602 /* Check for a mode in which STP pairs can be formed. */
17607 /* Assume we won't be able to use STP if the constant offset
17608 component of the address is misaligned. ??? This could be
17609 removed if we formed STP pairs earlier, rather than relying
17610 on peephole2. */
17613 }
17618 }
17619 }
17621 unsigned
17625 vect_cost_model_location where)
17626 {
17627 fractional_cost stmt_cost
17631 && stmt_info
17634 /* Do one-time initialization based on the vinfo. */
17637 {
17642 }
17644 /* Apply the heuristic described above m_stp_sequence_cost. */
17646 {
17650 }
17652 /* Try to get a more accurate cost by looking at STMT_INFO instead
17653 of just looking at KIND. */
17655 {
17656 /* If we scalarize a strided store, the vectorizer costs one
17657 vec_to_scalar for each element. However, we can store the first
17658 element using an FP store without a separate extract step. */
17669 stmt_cost);
17671 /* Check if we've seen an SVE gather/scatter operation and which size. */
17675 {
17678 {
17679 /* Test for VNx2 modes, which have 64-bit containers. */
17681 aarch64_sve_vg))
17682 m_sve_gather_scatter_init_cost
17684 else
17685 m_sve_gather_scatter_init_cost
17687 }
17688 }
17689 }
17691 /* Do any SVE-specific adjustments to the cost. */
17696 /* Vector promotion and demotion requires us to widen the operation first
17697 and only after that perform the conversion. Unfortunately the mid-end
17698 expects this to be doable as a single operation and doesn't pass on
17699 enough context here for us to tell which operation is happening. To
17700 account for this we count every promote-demote operation twice and if
17701 the previously costed operation was also a promote-demote we reduce
17702 the cost of the currently being costed operation to simulate the final
17703 conversion cost. Note that for SVE we can do better here if the converted
17704 value comes from a load since the widening load would consume the widening
17705 operations. However since we're in stage 3 we can't change the helper
17706 vect_is_extending_load and duplicating the code seems not useful. */
17711 {
17715 }
17716 else
17720 {
17721 /* Account for any extra "embedded" costs that apply additively
17722 to the base cost calculated above. */
17726 /* If we're recording a nonzero vector loop body cost for the
17727 innermost loop, also estimate the operations that would need
17728 to be issued by all relevant implementations of the loop. */
17736 /* If we're applying the SVE vs. Advanced SIMD unrolling heuristic,
17737 estimate the number of statements in the unrolled Advanced SIMD
17738 loop. For simplicitly, we assume that one iteration of the
17739 Advanced SIMD loop would need the same number of statements
17740 as one iteration of the SVE loop. */
17744 /* Detect the use of an averaging operation. */
17748 {
17750 {
17756 }
17757 }
17758 }
17760 /* If the statement stores to a decl that is known to be the argument
17761 to a vld1 in the same function, ignore the store for costing purposes.
17762 See the comment above m_stores_to_vector_load_decl for more details. */
17766 {
17769 }
17772 }
17774 /* Return true if (a) we're applying the Advanced SIMD vs. SVE unrolling
17775 heuristic described above m_unrolled_advsimd_niters and (b) the heuristic
17776 says that we should prefer the Advanced SIMD loop. */
17777 bool
17779 {
17785 " unrolled Advanced SIMD loop = "
17787 m_unrolled_advsimd_stmts);
17789 /* The balance here is tricky. On the one hand, we can't be sure whether
17790 the code is vectorizable with Advanced SIMD or not. However, even if
17791 it isn't vectorizable with Advanced SIMD, there's a possibility that
17792 the scalar code could also be unrolled. Some of the code might then
17793 benefit from SLP, or from using LDP and STP. We therefore apply
17794 the heuristic regardless of can_use_advsimd_p. */
17796 && (m_unrolled_advsimd_stmts
17798 }
17800 /* Subroutine of adjust_body_cost for handling SVE. Use ISSUE_INFO to work out
17801 how fast the SVE code can be issued and compare it to the equivalent value
17802 for scalar code (SCALAR_CYCLES_PER_ITER). If COULD_USE_ADVSIMD is true,
17803 also compare it to the issue rate of Advanced SIMD code
17804 (ADVSIMD_CYCLES_PER_ITER).
17806 ORIG_BODY_COST is the cost originally passed to adjust_body_cost and
17807 *BODY_COST is the current value of the adjusted cost. *SHOULD_DISPARAGE
17808 is true if we think the loop body is too expensive. */
17810 fractional_cost
17813 fractional_cost scalar_cycles_per_iter,
17816 {
17823 /* If the scalar version of the loop could issue at least as
17824 quickly as the predicate parts of the SVE loop, make the SVE loop
17825 prohibitively expensive. In this case vectorization is adding an
17826 overhead that the original scalar code didn't have.
17828 This is mostly intended to detect cases in which WHILELOs dominate
17829 for very tight loops, which is something that normal latency-based
17830 costs would not model. Adding this kind of cliffedge would be
17831 too drastic for scalar_cycles_per_iter vs. sve_cycles_per_iter;
17832 code in the caller handles that case in a more conservative way. */
17835 {
17836 unsigned int min_cost
17839 {
17842 "Increasing body cost to %d because the"
17843 " scalar code could issue within the limit"
17845 min_cost);
17848 }
17849 }
17852 }
17854 unsigned int
17856 {
17858 /* If we are trying to unroll an Advanced SIMD main loop that contains
17859 an averaging operation that we do not support with SVE and we might use a
17860 predicated epilogue, we need to be conservative and block unrolling as
17861 this might lead to a less optimal loop for the first and only epilogue
17862 using the original loop's vectorization factor.
17863 TODO: Remove this constraint when we add support for multiple epilogue
17864 vectorization. */
17870 {
17875 /* Limit unroll factor to a value adjustable by the user, the default
17876 value is 4. */
17878 unsigned int factor
17882 /* Sanity check, this should never happen. */
17886 /* Check stores. */
17888 {
17892 }
17894 /* Check loads + stores. */
17896 {
17900 }
17902 /* Check general ops. */
17904 {
17908 }
17910 }
17912 /* Make sure unroll factor is power of 2. */
17914 }
17916 /* BODY_COST is the cost of a vector loop body. Adjust the cost as necessary
17917 and return the new cost. */
17918 unsigned int
17923 {
17937 /* If we know we have a single partial vector iteration, cap the VF
17938 to the number of scalar iterations for costing purposes. */
17940 {
17944 "Scalar loop iterates at most %wd times. Capping VF "
17948 }
17950 fractional_cost scalar_cycles_per_iter
17956 {
17960 m_num_vector_iterations);
17964 " estimated cycles per vector iteration"
17967 }
17970 {
17973 vector_cycles_per_iter
17978 {
17979 /* Also take Neoverse V1 tuning into account, doubling the
17980 scalar and Advanced SIMD estimates to account for the
17981 doubling in SVE vector length. */
17988 }
17989 }
17990 else
17991 {
17993 {
17997 }
17998 }
18000 /* Decide whether to stick to latency-based costs or whether to try to
18001 take issue rates into account. */
18008 {
18011 "Low iteration count, so using pure latency"
18013 }
18014 /* Increase the cost of the vector code if it looks like the scalar code
18015 could issue more quickly. These values are only rough estimates,
18016 so minor differences should only result in minor changes. */
18018 {
18020 scalar_cycles_per_iter);
18023 "Increasing body cost to %d because scalar code"
18025 }
18026 /* In general, it's expected that the proposed vector code would be able
18027 to issue more quickly than the original scalar code. This should
18028 already be reflected to some extent in the latency-based costs.
18030 However, the latency-based costs effectively assume that the scalar
18031 code and the vector code execute serially, which tends to underplay
18032 one important case: if the real (non-serialized) execution time of
18033 a scalar iteration is dominated by loop-carried dependencies,
18034 and if the vector code is able to reduce both the length of
18035 the loop-carried dependencies *and* the number of cycles needed
18036 to issue the code in general, we can be more confident that the
18037 vector code is an improvement, even if adding the other (non-loop-carried)
18038 latencies tends to hide this saving. We therefore reduce the cost of the
18039 vector loop body in proportion to the saving. */
18044 {
18046 scalar_cycles_per_iter);
18049 "Decreasing body cost to %d account for smaller"
18051 }
18054 }
18056 void
18058 {
18059 /* Record the issue information for any SVE WHILE instructions that the
18060 loop needs. */
18063 && loop_vinfo
18065 {
18076 }
18081 {
18086 /* For gather and scatters there's an additional overhead for the first
18087 iteration. For low count loops they're not beneficial so model the
18088 overhead as loop prologue costs. */
18090 }
18092 /* Apply the heuristic described above m_stp_sequence_cost. Prefer
18093 the scalar code in the event of a tie, since there is more chance
18094 of scalar code being optimized with surrounding operations.
18096 In addition, if the vector body is a simple store to a decl that
18097 is elsewhere loaded using vld1, strongly prefer the vector form,
18098 to the extent of giving the prologue a zero cost. See the comment
18099 above m_stores_to_vector_load_decl for details. */
18101 && scalar_costs
18103 {
18108 }
18111 }
18113 bool
18116 {
18130 /* Apply the unrolling heuristic described above
18131 m_unrolled_advsimd_niters. */
18134 {
18138 {
18141 "Preferring Advanced SIMD loop because"
18144 }
18145 }
18148 {
18150 {
18162 }
18169 /* If it appears that one loop could process the same amount of data
18170 in fewer cycles, prefer that loop over the other one. */
18171 fractional_cost this_cost
18173 fractional_cost other_cost
18176 {
18185 }
18187 {
18190 "Preferring loop with lower cycles"
18193 }
18195 /* If the issue rate of SVE code is limited by predicate operations
18196 (i.e. if sve_pred_cycles_per_iter > sve_nonpred_cycles_per_iter),
18197 and if Advanced SIMD code could issue within the limit imposed
18198 by the predicate operations, the predicate operations are adding an
18199 overhead that the original code didn't have and so we should prefer
18200 the Advanced SIMD version. */
18203 {
18207 {
18210 "Preferring Advanced SIMD loop since"
18213 }
18215 };
18220 }
18223 }
18227 /* Parse TOKEN, which has length LENGTH to see if it is an option
18228 described in FLAG. If it is, return the index bit for that fusion type.
18229 If not, error (printing OPTION_NAME) and return zero. */
18231 static unsigned int
18236 {
18238 {
18242 }
18246 }
18248 /* Parse OPTION which is a comma-separated list of flags to enable.
18249 FLAGS gives the list of flags we understand, INITIAL_STATE gives any
18250 default state we inherit from the CPU tuning structures. OPTION_NAME
18251 gives the top-level option we are parsing in the -moverride string,
18252 for use in error messages. */
18254 static unsigned int
18259 {
18266 {
18269 token_length,
18270 flags,
18271 option_name);
18272 /* If we find "none" (or, for simplicity's sake, an error) anywhere
18273 in the token stream, reset the supported operations. So:
18275 adrp+add.cmp+branch.none.adrp+add
18277 would have the result of turning on only adrp+add fusion. */
18283 }
18285 /* We ended with a comma, print something. */
18287 {
18290 }
18292 /* We still have one more token to parse. */
18295 token_length,
18296 flags,
18297 option_name);
18303 }
18305 /* Support for overriding instruction fusion. */
18307 static void
18310 {
18312 aarch64_fusible_pairs,
18315 }
18317 /* Support for overriding other tuning flags. */
18319 static void
18322 {
18323 tune->extra_tuning_flags
18325 aarch64_tuning_flags,
18328 }
18330 /* Parse the sve_width tuning moverride string in TUNE_STRING.
18331 Accept the valid SVE vector widths allowed by
18332 aarch64_sve_vector_bits_enum and use it to override sve_width
18333 in TUNE. */
18335 static void
18338 {
18343 {
18346 }
18348 {
18357 }
18359 }
18361 /* Parse TOKEN, which has length LENGTH to see if it is a tuning option
18362 we understand. If it is, extract the option string and handoff to
18363 the appropriate function. */
18365 void
18369 {
18375 {
18378 }
18380 /* Get the length of the option name. */
18382 /* Skip the '=' to get to the option string. */
18383 option_part++;
18386 {
18388 {
18391 }
18392 }
18396 }
18398 /* A checking mechanism for the implementation of the tls size. */
18400 static void
18402 {
18407 {
18409 /* Both the default and maximum TLS size allowed under tiny is 1M which
18410 needs two instructions to address, so we clamp the size to 24. */
18415 /* The maximum TLS size allowed under small is 4G. */
18420 /* The maximum TLS size allowed under large is 16E.
18421 FIXME: 16E should be 64bit, we only support 48bit offset now. */
18427 }
18430 }
18432 /* Return the CPU corresponding to the enum CPU. */
18436 {
18440 }
18442 /* Return the architecture corresponding to the enum ARCH. */
18446 {
18450 }
18452 /* Parse STRING looking for options in the format:
18453 string :: option:string
18454 option :: name=substring
18455 name :: {a-z}
18456 substring :: defined by option. */
18458 static void
18461 {
18472 {
18474 /* Make this substring look like a string. */
18478 }
18480 /* One last option to parse. */
18483 }
18485 /* Adjust CURRENT_TUNE (a generic tuning struct) with settings that
18486 are best for a generic target with the currently-enabled architecture
18487 extensions. */
18488 static void
18490 {
18491 /* Neoverse V1 is the only core that is known to benefit from
18492 AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS. There is therefore no
18493 point enabling it for SVE2 and above. */
18495 current_tune.extra_tuning_flags
18497 }
18499 static void
18501 {
18502 /* PR 70044: We have to be careful about being called multiple times for the
18503 same function. This means all changes should be repeatable. */
18505 /* Set aarch64_use_frame_pointer based on -fno-omit-frame-pointer.
18506 Disable the frame pointer flag so the mid-end will not use a frame
18507 pointer in leaf functions in order to support -fomit-leaf-frame-pointer.
18508 Set x_flag_omit_frame_pointer to the special value 2 to differentiate
18509 between -fomit-frame-pointer (1) and -fno-omit-frame-pointer (2). */
18514 /* If not optimizing for size, set the default
18515 alignment to what the target wants. */
18517 {
18524 }
18526 /* We default to no pc-relative literal loads. */
18530 /* If -mpc-relative-literal-loads is set on the command line, this
18531 implies that the user asked for PC relative literal loads. */
18535 /* In the tiny memory model it makes no sense to disallow PC relative
18536 literal pool loads. */
18541 /* When enabling the lower precision Newton series for the square root, also
18542 enable it for the reciprocal square root, since the latter is an
18543 intermediary step for the former. */
18546 }
18548 /* 'Unpack' up the internal tuning structs and update the options
18549 in OPTS. The caller must have set up selected_tune and selected_arch
18550 as all the other target-specific codegen decisions are
18551 derived from them. */
18553 void
18555 {
18558 /* Make a copy of the tuning parameters attached to the core, which
18559 we may later overwrite. */
18574 /* This target defaults to strict volatile bitfields. */
18580 {
18583 aarch64_stack_protector_guard_offset_str);
18584 }
18589 {
18591 "%<-mstack-protector-guard-reg%> must be used "
18593 }
18596 {
18599 }
18602 {
18611 }
18620 {
18623 else
18627 " option %<-march%>, or by using the %<target%>"
18632 }
18640 {
18652 }
18654 /* We don't mind passing in global_options_set here as we don't use
18655 the *options_set structs anyway. */
18659 /* Set up parameters to be used in prefetching algorithm. Do not
18660 override the defaults unless we are tuning for a core we have
18661 researched values for. */
18664 param_simultaneous_prefetches,
18668 param_l1_cache_size,
18672 param_l1_cache_line_size,
18676 {
18678 param_destruct_interfere_size,
18681 param_construct_interfere_size,
18683 }
18684 else
18685 {
18686 /* For a generic AArch64 target, cover the current range of cache line
18687 sizes. */
18689 param_destruct_interfere_size,
18692 param_construct_interfere_size,
18694 }
18698 param_l2_cache_size,
18705 param_prefetch_minimum_stride,
18708 /* Use the alternative scheduling-pressure algorithm by default. */
18710 param_sched_pressure_algorithm,
18711 SCHED_PRESSURE_MODEL);
18713 /* Validate the guard size. */
18721 /* Enforce that interval is the same size as size so the mid-end does the
18722 right thing. */
18724 param_stack_clash_protection_probe_interval,
18725 guard_size);
18727 /* The maybe_set calls won't update the value if the user has explicitly set
18728 one. Which means we need to validate that probing interval and guard size
18729 are equal. */
18730 int probe_interval
18736 /* Enable sw prefetching at specified optimization level for
18737 CPUS that have prefetch. Lower optimization level threshold by 1
18738 when profiling is enabled. */
18745 /* Avoid loop-dependant FMA chains. */
18751 /* Consider fully pipelined FMA in reassociation. */
18758 }
18760 /* Straight line speculation indicators. */
18761 enum aarch64_sls_hardening_type
18762 {
18767 };
18770 /* Return whether we should mitigatate Straight Line Speculation for the RET
18771 and BR instructions. */
18772 bool
18774 {
18776 }
18778 /* Return whether we should mitigatate Straight Line Speculation for the BLR
18779 instruction. */
18780 bool
18782 {
18784 }
18786 /* As of yet we only allow setting these options globally, in the future we may
18787 allow setting them per function. */
18788 static void
18790 {
18795 {
18798 }
18800 {
18803 }
18812 {
18818 {
18821 }
18822 else
18823 {
18826 }
18828 }
18831 }
18833 /* Return the VG value associated with -msve-vector-bits= value VALUE. */
18835 static poly_uint16
18837 {
18838 /* 128-bit SVE and Advanced SIMD modes use different register layouts
18839 on big-endian targets, so we would need to forbid subregs that convert
18840 from one to the other. By default a reinterpret sequence would then
18841 involve a store to memory in one mode and a load back in the other.
18842 Even if we optimize that sequence using reverse instructions,
18843 it would still be a significant potential overhead.
18845 For now, it seems better to generate length-agnostic code for that
18846 case instead. */
18850 else
18852 }
18854 /* Set the global aarch64_asm_isa_flags to FLAGS and update
18855 aarch64_isa_flags accordingly. */
18857 void
18859 {
18861 }
18863 static void
18865 {
18869 }
18871 static void
18873 {
18878 }
18880 static void
18882 {
18885 }
18887 static void
18889 {
18891 }
18893 static void
18895 {
18897 }
18899 static void
18901 {
18903 }
18904 static void
18906 {
18908 }
18914 };
18917 {
18925 };
18927 /* Implement TARGET_OPTION_OVERRIDE. This is called once in the beginning
18928 and is used to parse the -m{cpu,tune,arch} strings and setup the initial
18929 tuning structs. In particular it must set selected_tune and
18930 aarch64_asm_isa_flags that define the available ISA features and tuning
18931 decisions. It must also set selected_arch as this will be used to
18932 output the .arch asm tags for each function. */
18934 static void
18936 {
18950 aarch64_branch_protection_string,
18953 /* -mcpu=CPU is shorthand for -march=ARCH_FOR_CPU, -mtune=CPU.
18954 If either of -march or -mtune is given, they override their
18955 respective component of -mcpu. */
18965 #ifdef SUBTARGET_OVERRIDE_OPTIONS
18966 SUBTARGET_OVERRIDE_OPTIONS;
18967 #endif
18970 {
18971 /* If both -mcpu and -march are specified, warn if they are not
18972 feature compatible. feature compatible means that the inclusion of the
18973 cpu features would end up disabling an achitecture feature. In
18974 otherwords the cpu features need to be a strict superset of the arch
18975 features and if so prefer the -march ISA flags. */
18977 {
18982 aarch64_cpu_string,
18983 aarch64_arch_string,
18985 }
18989 }
18991 {
18994 }
18996 {
19000 }
19001 else
19002 {
19003 /* No -mcpu or -march specified, so use the default CPU. */
19009 }
19014 {
19015 #ifdef TARGET_ENABLE_BTI
19017 #else
19019 #endif
19020 }
19023 {
19024 #ifdef TARGET_ENABLE_GCS
19026 #else
19028 #endif
19029 }
19031 /* Return address signing is currently not supported for ILP32 targets. For
19032 LP64 targets use the configured option in the absence of a command-line
19033 option for -mbranch-protection. */
19035 {
19036 #ifdef TARGET_ENABLE_PAC_RET
19038 #else
19040 #endif
19041 }
19043 #ifndef HAVE_AS_MABI_OPTION
19044 /* The compiler may have been configured with 2.23.* binutils, which does
19045 not have support for ILP32. */
19048 #endif
19052 /* Convert -msve-vector-bits to a VG count. */
19058 /* The pass to insert speculation tracking runs before
19059 shrink-wrapping and the latter does not know how to update the
19060 tracking status. So disable it in this case. */
19066 /* Save these options as the default ones in case we push and pop them later
19067 while processing functions with potential target attributes. */
19068 target_option_default_node = target_option_current_node
19070 }
19072 /* Implement targetm.override_options_after_change. */
19074 static void
19076 {
19078 }
19080 /* Implement the TARGET_OFFLOAD_OPTIONS hook. */
19083 {
19086 else
19088 }
19092 {
19096 }
19098 void
19100 {
19102 }
19104 /* A checking mechanism for the implementation of the various code models. */
19105 static void
19107 {
19110 {
19117 {
19118 #ifdef HAVE_AS_SMALL_PIC_RELOCS
19120 ? AARCH64_CMODEL_SMALL_PIC
19122 #else
19124 #endif
19125 }
19138 }
19139 }
19141 /* Implements TARGET_OPTION_RESTORE. Restore the backend codegen decisions
19142 using the information saved in PTR. */
19144 static void
19148 {
19150 }
19152 /* Implement TARGET_OPTION_PRINT. */
19154 static void
19156 {
19167 }
19171 void
19173 {
19175 }
19177 /* Restore or save the TREE_TARGET_GLOBALS from or to NEW_TREE.
19178 Used by aarch64_set_current_function and aarch64_pragma_target_parse to
19179 make sure optab availability predicates are recomputed when necessary. */
19181 void
19183 {
19188 else
19190 }
19192 /* Return the target_option_node for FNDECL, or the current options
19193 if FNDECL is null. */
19195 static tree
19197 {
19205 }
19207 /* Implement TARGET_SET_CURRENT_FUNCTION. Unpack the codegen decisions
19208 like tuning and ISA features from the DECL_FUNCTION_SPECIFIC_TARGET
19209 of the function, if such exists. This function may be called multiple
19210 times on a single function so use aarch64_previous_fndecl to avoid
19211 setting up identical state. */
19213 static void
19215 {
19227 && fndecl
19229 {
19233 " option %<-march%>, or by using the %<target%>"
19236 }
19238 /* If nothing to do, return. #pragma GCC reset or #pragma GCC pop to
19239 the default have been handled by aarch64_save_restore_target_globals from
19240 aarch64_pragma_target_parse. */
19244 {
19247 }
19251 /* First set the target options. */
19255 /* The ISA mode can vary based on function type attributes and
19256 function declaration attributes. Make sure that the target
19257 options correctly reflect these attributes. */
19259 {
19273 }
19278 }
19280 /* Enum describing the various ways we can handle attributes.
19281 In many cases we can reuse the generic option handling machinery. */
19283 enum aarch64_attr_opt_type
19284 {
19288 aarch64_attr_custom /* Attribute requires a custom handling function. */
19289 };
19291 /* All the information needed to handle a target attribute.
19292 NAME is the name of the attribute.
19293 ATTR_TYPE specifies the type of behavior of the attribute as described
19294 in the definition of enum aarch64_attr_opt_type.
19295 ALLOW_NEG is true if the attribute supports a "no-" form.
19296 HANDLER is the function that takes the attribute string as an argument
19297 It is needed only when the ATTR_TYPE is aarch64_attr_custom.
19298 OPT_NUM is the enum specifying the option that the attribute modifies.
19299 This is needed for attributes that mirror the behavior of a command-line
19300 option, that is it has ATTR_TYPE aarch64_attr_mask, aarch64_attr_bool or
19301 aarch64_attr_enum. */
19303 struct aarch64_attribute_info
19304 {
19310 };
19312 /* Handle the ARCH_STR argument to the arch= target attribute. */
19314 static bool
19316 {
19319 aarch64_feature_flags tmp_flags;
19320 enum aarch_parse_opt_result parse_res
19324 {
19330 }
19333 {
19348 }
19351 }
19353 /* Handle the argument CPU_STR to the cpu= target attribute. */
19355 static bool
19357 {
19360 aarch64_feature_flags tmp_flags;
19361 enum aarch_parse_opt_result parse_res
19365 {
19372 }
19375 {
19390 }
19393 }
19395 /* Handle the argument STR to the branch-protection= attribute. */
19397 static bool
19399 {
19402 }
19404 /* Handle the argument STR to the tune= target attribute. */
19406 static bool
19408 {
19410 enum aarch_parse_opt_result parse_res
19414 {
19418 }
19421 {
19428 }
19431 }
19433 /* Parse an architecture extensions target attribute string specified in STR.
19434 For example "+fp+nosimd". Show any errors if needed. Return TRUE
19435 if successful. Update aarch64_isa_flags to reflect the ISA features
19436 modified. */
19438 static bool
19440 {
19444 /* We allow "+nothing" in the beginning to clear out all architectural
19445 features if the user wants to handpick specific features. */
19447 {
19450 }
19456 {
19459 }
19462 {
19474 }
19477 }
19479 /* The target attributes that we support. On top of these we also support just
19480 ISA extensions, like __attribute__ ((target ("+crc"))), but that case is
19481 handled explicitly in aarch64_process_one_target_attr. */
19484 {
19486 OPT_mgeneral_regs_only },
19488 OPT_mfix_cortex_a53_835769 },
19490 OPT_mfix_cortex_a53_843419 },
19494 OPT_momit_leaf_frame_pointer },
19497 OPT_march_ },
19500 OPT_mtune_ },
19504 OPT_msign_return_address_ },
19506 OPT_moutline_atomics},
19508 };
19510 /* Parse ARG_STR which contains the definition of one target attribute.
19511 Show appropriate errors if any or return true if the attribute is valid. */
19513 static bool
19515 {
19521 {
19524 }
19531 /* We have something like __attribute__ ((target ("+fp+nosimd"))).
19532 It is easier to detect and handle it explicitly here rather than going
19533 through the machinery for the rest of the target attributes in this
19534 function. */
19539 {
19542 }
19545 /* If we found opt=foo then terminate STR_TO_CHECK at the '='
19546 and point ARG to "foo". */
19548 {
19550 arg++;
19551 }
19555 {
19556 /* If the names don't match up, or the user has given an argument
19557 to an attribute that doesn't accept one, or didn't give an argument
19558 to an attribute that expects one, fail to match. */
19567 {
19570 }
19572 /* If the name matches but the attribute does not allow "no-" versions
19573 then we can't match. */
19575 {
19578 }
19581 {
19582 /* Has a custom handler registered.
19583 For example, cpu=, arch=, tune=. */
19590 /* Either set or unset a boolean option. */
19592 {
19600 }
19601 /* Set or unset a bit in the target_flags. aarch64_handle_option
19602 should know what mask to apply given the option number. */
19604 {
19606 /* We only need to specify the option number.
19607 aarch64_handle_option will know which mask to apply. */
19613 }
19614 /* Use the option setting machinery to set an option to an enum. */
19616 {
19623 {
19626 global_dc);
19627 }
19628 else
19629 {
19631 }
19633 }
19636 }
19637 }
19639 /* If we reached here we either have found an attribute and validated
19640 it or didn't match any. If we matched an attribute but its arguments
19641 were malformed we will have returned false already. */
19643 }
19645 /* Count how many times the character C appears in
19646 NULL-terminated string STR. */
19648 static unsigned int
19650 {
19653 {
19655 res++;
19657 str++;
19658 }
19661 }
19663 /* Parse the tree in ARGS that contains the target attribute information
19664 and update the global target options space. */
19666 bool
19668 {
19670 {
19671 do
19672 {
19675 {
19678 }
19683 }
19686 {
19689 }
19698 {
19701 }
19703 /* Used to catch empty spaces between commas i.e.
19704 attribute ((target ("attr1,,attr2"))). */
19707 /* Handle multiple target attributes separated by ','. */
19712 {
19713 num_attrs++;
19715 {
19716 /* Check if token is possibly an arch extension without
19717 leading '+'. */
19720 enum aarch_parse_opt_result ext_res
19725 token);
19726 else
19729 }
19732 }
19735 {
19738 }
19741 }
19745 /* Implement TARGET_OPTION_VALID_ATTRIBUTE_P. This is used to
19746 process attribute ((target ("..."))). */
19748 static bool
19750 {
19753 tree old_optimize;
19757 /* If what we're processing is the current pragma string then the
19758 target option node is already stored in target_option_current_node
19759 by aarch64_pragma_target_parse in aarch64-c.cc. Use that to avoid
19760 having to re-parse the string. This is especially useful to keep
19761 arm_neon.h compile times down since that header contains a lot
19762 of intrinsics enclosed in pragmas. */
19764 {
19767 }
19770 old_optimize
19774 /* If the function changed the optimization levels as well as setting
19775 target options, start with the optimizations specified. */
19780 /* Save the current target options to restore at the end. */
19783 /* If fndecl already has some target attributes applied to it, unpack
19784 them so that we add this attribute on top of them, rather than
19785 overwriting them. */
19787 {
19793 existing_options);
19794 }
19795 else
19801 {
19805 {
19806 /* Reapply any target_version attribute after target attribute.
19807 This should be equivalent to applying the target_version once
19808 after processing all target attributes. */
19811 }
19812 }
19814 /* Set up any additional state. */
19816 {
19820 }
19821 else
19828 {
19833 }
19841 }
19846 {
19848 aarch64_fmv_feature_mask feature_mask;
19849 aarch64_feature_flags opt_flags;
19852 #define AARCH64_FMV_FEATURE(NAME, FEAT_NAME, C) \
19853 {NAME, 1ULL << FEAT_##FEAT_NAME, ::feature_deps::fmv_deps_##FEAT_NAME},
19855 /* The "rdma" alias uses a different FEAT_NAME to avoid a duplicate
19856 feature_deps name. */
19857 #define FEAT_RDMA FEAT_RDM
19859 /* FMV features are listed in priority order, to make it easier to sort target
19860 strings. */
19863 };
19865 /* Parse a function multiversioning feature string STR, as found in a
19866 target_version or target_clones attribute.
19868 If ISA_FLAGS is nonnull, then update it with the specified architecture
19869 features turned on. If FEATURE_MASK is nonnull, then assign to it a bitmask
19870 representing the set of features explicitly specified in the feature string.
19871 Return an aarch_parse_opt_result describing the result.
19873 When the STR string contains an invalid or duplicate extension, a copy of
19874 the extension string is created and stored to INVALID_EXTENSION. */
19876 static enum aarch_parse_opt_result
19880 {
19888 {
19896 else
19905 {
19908 {
19912 {
19916 {
19917 /* Duplicate feature. */
19921 }
19922 }
19924 }
19925 }
19928 {
19929 /* Feature not found in list. */
19933 }
19937 /* Skip over the next '+'. */
19938 str++;
19939 }
19942 }
19944 /* Parse the tree in ARGS that contains the target_version attribute
19945 information and update the global target options space. */
19947 static bool
19949 {
19952 {
19954 "Function Multi Versioning support is experimental, and the "
19957 }
19960 {
19962 {
19965 }
19967 }
19970 {
19973 }
19985 {
19988 }
19991 {
19999 str);
20005 str);
20010 }
20013 }
20015 /* Implement TARGET_OPTION_VALID_VERSION_ATTRIBUTE_P. This is used to
20016 process attribute ((target_version ("..."))). */
20018 static bool
20020 {
20023 tree new_target;
20026 /* Save the current target options to restore at the end. */
20029 /* If fndecl already has some target attributes applied to it, unpack
20030 them so that we add this attribute on top of them, rather than
20031 overwriting them. */
20033 {
20039 existing_options);
20040 }
20041 else
20047 /* Set up any additional state. */
20049 {
20053 }
20054 else
20063 }
20065 /* This parses the attribute arguments to target_version in DECL and the
20066 feature mask required to select those targets. No adjustments are made to
20067 add or remove redundant feature requirements. */
20069 static aarch64_fmv_feature_mask
20071 {
20080 aarch64_fmv_feature_mask feature_mask;
20083 NULL);
20085 /* We should have detected any errors before getting here. */
20089 }
20091 /* Compare priorities of two feature masks. Return:
20092 1: mask1 is higher priority
20093 -1: mask2 is higher priority
20094 0: masks are equal. */
20096 static int
20098 aarch64_fmv_feature_mask mask2)
20099 {
20112 {
20116 }
20118 }
20120 /* Compare priorities of two version decls. */
20122 int
20124 {
20129 }
20131 /* Build the struct __ifunc_arg_t type:
20133 struct __ifunc_arg_t
20134 {
20135 unsigned long _size; // Size of the struct, so it can grow.
20136 unsigned long _hwcap;
20137 unsigned long _hwcap2;
20138 }
20139 */
20141 static tree
20143 {
20147 long_unsigned_type_node);
20150 long_unsigned_type_node);
20153 long_unsigned_type_node);
20169 }
20171 /* Implement TARGET_MANGLE_DECL_ASSEMBLER_NAME, to add function multiversioning
20172 suffixes. */
20174 tree
20176 {
20177 /* For function version, add the target suffix to the assembler name. */
20180 {
20185 /* For the default version, append ".default". */
20187 {
20190 }
20196 {
20198 {
20201 }
20202 }
20208 }
20210 }
20212 /* Return an identifier for the base assembler name of a versioned function.
20213 This is computed by taking the default version's assembler name, and
20214 stripping off the ".default" suffix if it's already been appended. */
20216 static tree
20218 {
20226 }
20228 /* Make the resolver function decl to dispatch the versions of
20229 a multi-versioned function, DEFAULT_DECL. IFUNC_ALIAS_DECL is
20230 ifunc alias that will point to the created resolver. Create an
20231 empty basic block in the resolver and store the pointer in
20232 EMPTY_BB. Return the decl of the resolver function. */
20234 static tree
20238 {
20241 /* Create resolver function name based on default_decl. We need to remove an
20242 existing ".default" suffix if this has already been appended. */
20246 /* The resolver function should have signature
20247 (void *) resolver (uint64_t, const __ifunc_arg_t *) */
20249 uint64_type_node,
20251 NULL_TREE);
20263 /* Resolver is not external, body is generated. */
20273 {
20274 /* In this case, each translation unit with a call to this
20275 versioned function will put out a resolver. Ensure it
20276 is comdat to keep just one copy. */
20279 }
20280 else
20283 /* Build result decl and add to function_decl. */
20290 /* Build parameter decls and add to function_decl. */
20293 uint64_type_node);
20319 /* Mark ifunc_alias_decl as "ifunc" with resolver as resolver_name. */
20324 /* Create the alias for dispatch to resolver here. */
20327 }
20329 /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
20330 to return a pointer to VERSION_DECL if all feature bits specified in
20331 FEATURE_MASK are not set in MASK_VAR. This function will be called during
20332 version dispatch to decide which function version to execute. It returns
20333 the basic block at the end, to which more conditions can be added. */
20334 static basic_block
20336 aarch64_fmv_feature_mask feature_mask,
20338 {
20347 gimple_seq gseq;
20361 {
20362 /* Default version. */
20370 }
20374 long_long_unsigned_type_node,
20375 mask_var,
20377 feature_mask));
20414 }
20416 /* This function generates the dispatch function for
20417 multi-versioned functions. DISPATCH_DECL is the function which will
20418 contain the dispatch logic. FNDECLS are the function choices for
20419 dispatch, and is a tree chain. EMPTY_BB is the basic block pointer
20420 in DISPATCH_DECL in which the dispatch code is generated. */
20422 static int
20426 {
20428 gimple_seq gseq;
20438 /* Function version dispatch is via IFUNC. IFUNC resolvers fire before
20439 constructors, so explicity call __init_cpu_features_resolver here. */
20441 long_unsigned_type_node,
20443 NULL);
20457 /* Build the struct type for __aarch64_cpu_features. */
20461 long_long_unsigned_type_node);
20468 global_type);
20492 /* fndecls_p is actually a vector. */
20495 /* At least one more version other than the default. */
20499 struct function_version_info
20500 {
20501 tree version_decl;
20502 aarch64_fmv_feature_mask feature_mask;
20511 {
20512 aarch64_fmv_feature_mask feature_mask;
20513 /* Get attribute string, parse it and find the right features. */
20517 actual_versions++;
20518 }
20524 };
20526 /* Sort the versions according to descending order of dispatch priority. */
20534 mask_var,
20539 }
20541 /* Implement TARGET_GENERATE_VERSION_DISPATCHER_BODY. */
20543 tree
20545 {
20546 tree resolver_decl;
20547 basic_block empty_bb;
20548 tree default_ver_decl;
20564 /* The first version in the chain corresponds to the default version. */
20567 /* node is going to be an alias, so remove the finalized bit. */
20581 {
20583 /* Check for virtual functions here again, as by this time it should
20584 have been determined if this function needs a vtable index or
20585 not. This happens for methods in derived classes that override
20586 virtual methods in base classes but are not explicitly marked as
20587 virtual. */
20592 }
20598 /* Fix up symbol names. First we need to obtain the base name, which may
20599 have already been mangled. */
20602 /* We need to redo the version mangling on the non-default versions for the
20603 target_clones case. Redoing the mangling for the target_version case is
20604 redundant but does no harm. We need to skip the default version, because
20605 expand_clones will append ".default" later; fortunately that suffix is the
20606 one we want anyway. */
20609 {
20612 base_name);
20614 }
20616 /* We also need to use the base name for the ifunc declaration. */
20620 }
20622 /* Make a dispatcher declaration for the multi-versioned function DECL.
20623 Calls to DECL function will be replaced with calls to the dispatcher
20624 by the front-end. Returns the decl of the dispatcher function. */
20626 tree
20628 {
20650 /* Find the default version and make it the first node. */
20652 /* Go to the beginning of the chain. */
20657 {
20662 }
20664 /* If there is no default node, just return NULL. */
20668 /* Make default info the first node. */
20670 {
20677 }
20682 {
20687 /* Right now, the dispatching is done via ifunc. */
20694 dispatcher_version_info
20699 /* Set the dispatcher for all the versions. */
20702 {
20705 }
20706 }
20707 else
20708 {
20710 "multiversioning needs %<ifunc%> which is not supported "
20712 }
20715 }
20717 /* This function returns true if FN1 and FN2 are versions of the same function,
20718 that is, the target_version attributes of the function decls are different.
20719 This assumes that FN1 and FN2 have the same signature. */
20721 bool
20723 {
20729 }
20731 /* Implement TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P. Use an opt-out
20732 rather than an opt-in list. */
20734 static bool
20736 {
20737 /* A function that has local SME state cannot be inlined into its caller,
20738 since we only support managing PSTATE.ZA switches at function scope. */
20741 }
20743 /* Helper for aarch64_can_inline_p. In the case where CALLER and CALLEE are
20744 tri-bool options (yes, no, don't care) and the default value is
20745 DEF, determine whether to reject inlining. */
20747 static bool
20750 {
20751 /* If the callee doesn't care, always allow inlining. */
20755 /* If the caller doesn't care, always allow inlining. */
20759 /* Otherwise, allow inlining if either the callee and caller values
20760 agree, or if the callee is using the default value. */
20762 }
20764 /* Bit allocations for ipa_fn_summary::target_info. */
20766 /* Set if the function contains a stmt that relies on the function's
20767 choice of PSTATE.SM setting (0 for non-streaming, 1 for streaming).
20768 Not meaningful for streaming-compatible functions. */
20771 /* Set if the function clobbers ZA and ZT0. Not meaningful for functions that
20772 have ZA state. */
20776 /* Implement TARGET_NEED_IPA_FN_TARGET_INFO. */
20778 static bool
20780 {
20781 /* We could in principle skip this for streaming-compatible functions
20782 that have ZA state, but that's a rare combination. */
20784 }
20786 /* Implement TARGET_UPDATE_IPA_FN_TARGET_INFO. */
20788 static bool
20790 {
20792 {
20793 /* We don't know what the asm does, so conservatively assume that
20794 it requires the function's current SM mode. */
20797 {
20804 }
20805 }
20807 {
20809 {
20810 /* The attributes on AArch64 builtins are supposed to be accurate.
20811 If the function isn't marked streaming-compatible then it
20812 needs whichever SM mode it selects. */
20816 }
20817 }
20819 }
20821 /* Implement TARGET_CAN_INLINE_P. Decide whether it is valid
20822 to inline CALLEE into CALLER based on target-specific info.
20823 Make sure that the caller and callee have compatible architectural
20824 features. Then go through the other possible target attributes
20825 and see if they can block inlining. Try not to reject always_inline
20826 callees unless they are incompatible architecturally. */
20828 static bool
20830 {
20842 /* Callee's ISA flags should be a subset of the caller's. */
20857 /* Return true if the callee might have target_info property PROPERTY.
20858 The answer must be true unless we have positive proof to the contrary. */
20860 {
20866 };
20868 /* Streaming-compatible code can be inlined into functions with any
20869 PSTATE.SM mode. Otherwise the caller and callee must agree on
20870 PSTATE.SM mode, unless we can prove that the callee is naturally
20871 streaming-compatible. */
20879 /* aarch64_function_attribute_inlinable_p prevents new-ZA and new-ZT0
20880 functions from being inlined into others. We also need to prevent
20881 inlining of shared-ZA functions into functions without ZA state,
20882 since this is an error condition.
20884 The only other problematic case for ZA is inlining a function that
20885 directly clobbers ZA or ZT0 into a function that has ZA or ZT0 state. */
20899 /* Allow non-strict aligned functions inlining into strict
20900 aligned ones. */
20910 /* If the architectural features match up and the callee is always_inline
20911 then the other attributes don't matter. */
20923 /* Honour explicit requests to workaround errata. */
20936 /* If the user explicitly specified -momit-leaf-frame-pointer for the
20937 caller and calle and they don't match up, reject inlining. */
20944 /* If the callee has specific tuning overrides, respect them. */
20949 /* If the user specified tuning override strings for the
20950 caller and callee and they don't match up, reject inlining.
20951 We just do a string compare here, we don't analyze the meaning
20952 of the string, as it would be too costly for little gain. */
20960 }
20962 /* Return the ID of the TLDESC ABI, initializing the descriptor if hasn't
20963 been already. */
20965 arm_pcs
20967 {
20970 {
20971 HARD_REG_SET full_reg_clobbers;
20978 }
20980 }
20982 /* Return true if SYMBOL_REF X binds locally. */
20984 static bool
20986 {
20990 }
20992 /* Return true if SYMBOL_REF X is thread local */
20993 static bool
20995 {
21004 }
21006 /* Classify a TLS symbol into one of the TLS kinds. */
21007 enum aarch64_symbol_type
21009 {
21013 {
21020 {
21026 }
21037 else
21046 }
21047 }
21049 /* Return the correct method for accessing X + OFFSET, where X is either
21050 a SYMBOL_REF or LABEL_REF. */
21052 enum aarch64_symbol_type
21054 {
21058 {
21060 {
21075 }
21076 }
21079 {
21084 {
21087 /* With -fPIC non-local symbols use the GOT. For orthogonality
21088 always use the GOT for extern weak symbols. */
21094 /* When we retrieve symbol + offset address, we have to make sure
21095 the offset does not cause overflow of the final address. But
21096 we have no way of knowing the address of symbol at compile time
21097 so we can't accurately say if the distance between the PC and
21098 symbol + offset is outside the addressible range of +/-1MB in the
21099 TINY code model. So we limit the maximum offset to +/-64KB and
21100 assume the offset to the symbol is not larger than +/-(1MB - 64KB).
21101 If offset_within_block_p is true we allow larger offsets. */
21118 /* Same reasoning as the tiny code model, but the offset cap here is
21119 1MB, allowing +/-3.9GB for the offset to the symbol. */
21127 /* This is alright even in PIC code as the constant
21128 pool reference is always PC relative and within
21129 the same translation unit. */
21132 else
21137 }
21138 }
21140 /* By default push everything into the constant pool. */
21142 }
21144 bool
21146 {
21148 }
21150 bool
21152 {
21153 poly_int64 offset;
21159 }
21161 /* Implement TARGET_LEGITIMATE_CONSTANT_P hook. Return true for constants
21162 that should be rematerialized rather than spilled. */
21164 static bool
21166 {
21167 /* Support CSE and rematerialization of common constants. */
21172 /* Only accept variable-length vector constants if they can be
21173 handled directly.
21175 ??? It would be possible (but complex) to handle rematerialization
21176 of other constants via secondary reloads. */
21180 /* Otherwise, accept any CONST_VECTOR that, if all else fails, can at
21181 least be forced to memory and loaded from there. */
21185 /* Do not allow vector struct mode constants for Advanced SIMD.
21186 We could support 0 and -1 easily, but they need support in
21187 aarch64-simd.md. */
21195 /* Accept polynomial constants that can be calculated by using the
21196 destination of a move as the sole temporary. Constants that
21197 require a second temporary cannot be rematerialized (they can't be
21198 forced to memory and also aren't legitimate constants). */
21199 poly_int64 offset;
21203 /* If an offset is being added to something else, we need to allow the
21204 base to be moved into the destination register, meaning that there
21205 are no free temporaries for the offset. */
21210 /* Do not allow const (plus (anchor_symbol, const_int)). */
21214 /* Treat symbols as constants. Avoid TLS symbols as they are complex,
21215 so spilling them is better than rematerialization. */
21219 /* Label references are always constant. */
21224 }
21226 rtx
21228 {
21234 /* Can return in any reg. */
21237 }
21239 /* On AAPCS systems, this is the "struct __va_list". */
21242 /* Implement TARGET_BUILD_BUILTIN_VA_LIST.
21243 Return the type to use as __builtin_va_list.
21245 AAPCS64 \S 7.1.4 requires that va_list be a typedef for a type defined as:
21247 struct __va_list
21248 {
21249 void *__stack;
21250 void *__gr_top;
21251 void *__vr_top;
21252 int __gr_offs;
21253 int __vr_offs;
21254 }; */
21256 static tree
21258 {
21259 tree va_list_name;
21262 /* Create the type. */
21264 /* Give it the required name. */
21266 TYPE_DECL,
21268 va_list_type);
21274 /* Create the fields. */
21277 ptr_type_node);
21280 ptr_type_node);
21283 ptr_type_node);
21286 integer_type_node);
21289 integer_type_node);
21291 /* Tell tree-stdarg pass about our internal offset fields.
21292 NOTE: va_list_gpr/fpr_counter_field are only used for tree comparision
21293 purpose to identify whether the code is updating va_list internal
21294 offset fields through irregular way. */
21316 /* Compute its layout. */
21320 }
21322 /* Implement TARGET_EXPAND_BUILTIN_VA_START. */
21323 static void
21325 {
21329 tree t;
21343 {
21346 }
21355 NULL_TREE);
21357 NULL_TREE);
21359 NULL_TREE);
21361 NULL_TREE);
21363 NULL_TREE);
21365 /* Emit code to initialize STACK, which points to the next varargs stack
21366 argument. CUM->AAPCS_STACK_SIZE gives the number of stack words used
21367 by named arguments. STACK is 8-byte aligned. */
21374 /* Emit code to initialize GRTOP, the top of the GR save area.
21375 virtual_incoming_args_rtx should have been 16 byte aligned. */
21380 /* Emit code to initialize VRTOP, the top of the VR save area.
21381 This address is gr_save_area_bytes below GRTOP, rounded
21382 down to the next 16-byte boundary. */
21392 /* Emit code to initialize GROFF, the offset from GRTOP of the
21393 next GPR argument. */
21398 /* Likewise emit code to initialize VROFF, the offset from FTOP
21399 of the next VR argument. */
21403 }
21405 /* Implement TARGET_GIMPLIFY_VA_ARG_EXPR. */
21407 static tree
21410 {
21411 tree addr;
21417 machine_mode mode;
21443 align
21452 {
21453 /* No frontends can create types with variable-sized modes, so we
21454 shouldn't be asked to pass or return them. */
21457 /* TYPE passed in fp/simd registers. */
21469 {
21472 }
21475 {
21477 }
21478 }
21479 else
21480 {
21481 /* TYPE passed in general registers. */
21490 && abi_break_gcc_13
21491 && warn_psabi
21497 && abi_break_gcc_14
21504 {
21506 && warn_psabi
21511 }
21515 {
21517 }
21518 }
21520 /* Get a local temporary for the field value. */
21523 /* Emit code to branch if off >= 0. */
21529 {
21530 /* Emit: offs = (offs + 15) & -16. */
21536 }
21537 else
21540 /* Update ap.__[g|v]r_offs */
21545 /* String up. */
21549 /* [cond2] if (ap.__[g|v]r_offs > 0) */
21554 /* String up: make sure the assignment happens before the use. */
21558 /* Prepare the trees handling the argument that is passed on the stack;
21559 the top level node will store in ON_STACK. */
21562 {
21563 /* if (alignof(type) > 8) (arg = arg + 15) & -16; */
21568 }
21569 else
21571 /* Advance ap.__stack */
21576 /* String up roundup and advance. */
21579 /* String up with arg */
21581 /* Big-endianness related address adjustment. */
21584 {
21588 }
21593 /* Adjustment to OFFSET in the case of BIG_ENDIAN. */
21603 {
21604 /* type ha; // treat as "struct {ftype field[n];}"
21605 ... [computing offs]
21606 for (i = 0; i <nregs; ++i, offs += 16)
21607 ha.field[i] = *((ftype *)(ap.__vr_top + offs));
21608 return ha; */
21612 /* Declare a local variable. */
21616 /* Establish the base type. */
21618 {
21653 {
21657 }
21661 }
21663 /* *(field_ptr_t)&ha = *((field_ptr_t)vr_saved_area */
21672 /* ha.field[i] = *((field_ptr_t)vr_saved_area + i) */
21674 {
21684 }
21688 }
21698 }
21700 /* Implement TARGET_SETUP_INCOMING_VARARGS. */
21702 static void
21706 {
21708 CUMULATIVE_ARGS local_cum;
21712 /* The caller has advanced CUM up to, but not beyond, the last named
21713 argument. Advance a local copy of CUM past the last "real" named
21714 argument, to find out how many registers are left over. */
21719 /* Found out how many registers we need to save.
21720 Honor tree-stdvar analysis results. */
21729 {
21732 }
21735 {
21737 {
21740 /* virtual_incoming_args_rtx should have been 16-byte aligned. */
21748 }
21750 {
21751 /* We can't use move_block_from_reg, because it will use
21752 the wrong mode, storing D regs only. */
21756 /* Set OFF to the offset from virtual_incoming_args_rtx of
21757 the first vector register. The VR save area lies below
21758 the GR one, and is aligned to 16 bytes. */
21765 {
21773 }
21774 }
21775 }
21777 /* We don't save the size into *PRETEND_SIZE because we want to avoid
21778 any complication of having crtl->args.pretend_args_size changed. */
21783 }
21785 static void
21787 {
21790 {
21792 {
21796 }
21797 }
21800 {
21803 }
21805 /* Only allow these registers to be accessed via special patterns. */
21812 /* When tracking speculation, we need a couple of call-clobbered registers
21813 to track the speculation state. It would be nice to just use
21814 IP0 and IP1, but currently there are numerous places that just
21815 assume these registers are free for other uses (eg pointer
21816 authentication). */
21818 {
21823 }
21824 }
21826 /* Implement TARGET_MEMBER_TYPE_FORCES_BLK. */
21828 bool
21830 {
21831 /* For records we're passed a FIELD_DECL, for arrays we're passed
21832 an ARRAY_TYPE. In both cases we're interested in the TREE_TYPE. */
21835 /* Assign BLKmode to anything that contains more than 2 SVE predicates.
21836 For structures, the "multiple" case is indicated by MODE being
21837 VOIDmode. */
21840 {
21845 }
21848 }
21850 /* Bitmasks that indicate whether earlier versions of GCC would have
21851 taken a different path through the ABI logic. This should result in
21852 a -Wpsabi warning if the earlier path led to a different ABI decision.
21854 WARN_PSABI_EMPTY_CXX17_BASE
21855 Indicates that the type includes an artificial empty C++17 base field
21856 that, prior to GCC 10.1, would prevent the type from being treated as
21857 a HFA or HVA. See PR94383 for details.
21859 WARN_PSABI_NO_UNIQUE_ADDRESS
21860 Indicates that the type includes an empty [[no_unique_address]] field
21861 that, prior to GCC 10.1, would prevent the type from being treated as
21862 a HFA or HVA. */
21867 /* Walk down the type tree of TYPE counting consecutive base elements.
21868 If *MODEP is VOIDmode, then set it to the first valid floating point
21869 type. If a non-floating point type is found, or if a floating point
21870 type that doesn't match a non-VOIDmode *MODEP is found, then return -1,
21871 otherwise return the count in the sub-tree.
21873 The WARN_PSABI_FLAGS argument allows the caller to check whether this
21874 function has changed its behavior relative to earlier versions of GCC.
21875 Normally the argument should be nonnull and point to a zero-initialized
21876 variable. The function then records whether the ABI decision might
21877 be affected by a known fix to the ABI logic, setting the associated
21878 WARN_PSABI_* bits if so.
21880 When the argument is instead a null pointer, the function tries to
21881 simulate the behavior of GCC before all such ABI fixes were made.
21882 This is useful to check whether the function returns something
21883 different after the ABI fixes. */
21884 static int
21887 {
21888 machine_mode mode;
21889 HOST_WIDE_INT size;
21895 {
21926 /* Use V2SImode and V4SImode as representatives of all 64-bit
21927 and 128-bit vector types. */
21930 {
21939 }
21944 /* Vector modes are considered to be opaque: two vectors are
21945 equivalent for the purposes of being homogeneous aggregates
21946 if they are the same size. */
21953 {
21957 /* Can't handle incomplete types nor sizes that are not
21958 fixed. */
21964 warn_psabi_flags);
21966 || !index
21977 /* There must be no padding. */
21983 }
21986 {
21989 tree field;
21991 /* Can't handle incomplete types nor sizes that are not
21992 fixed. */
21998 {
22003 {
22004 /* See whether this is something that earlier versions of
22005 GCC failed to ignore. */
22012 else
22013 /* No compatibility problem. */
22016 /* Simulate the old behavior when WARN_PSABI_FLAGS is null. */
22018 {
22021 }
22022 }
22023 /* A zero-width bitfield may affect layout in some
22024 circumstances, but adds no members. The determination
22025 of whether or not a type is an HFA is performed after
22026 layout is complete, so if the type still looks like an
22027 HFA afterwards, it is still classed as one. This is
22028 potentially an ABI break for the hard-float ABI. */
22031 {
22032 /* Prior to GCC-12 these fields were striped early,
22033 hiding them from the back-end entirely and
22034 resulting in the correct behaviour for argument
22035 passing. Simulate that old behaviour without
22036 generating a warning. */
22040 {
22043 }
22044 }
22047 warn_psabi_flags);
22051 }
22053 /* There must be no padding. */
22059 }
22063 {
22064 /* These aren't very interesting except in a degenerate case. */
22067 tree field;
22069 /* Can't handle incomplete types nor sizes that are not
22070 fixed. */
22076 {
22081 warn_psabi_flags);
22085 }
22087 /* There must be no padding. */
22093 }
22097 }
22100 }
22102 /* Return TRUE if the type, as described by TYPE and MODE, is a short vector
22103 type as described in AAPCS64 \S 4.1.2.
22105 See the comment above aarch64_composite_type_p for the notes on MODE. */
22107 static bool
22109 machine_mode mode)
22110 {
22114 {
22118 }
22121 {
22122 /* The containing "else if" is too loose: it means that we look at TYPE
22123 if the type is a vector type (good), but that we otherwise ignore TYPE
22124 and look only at the mode. This is wrong because the type describes
22125 the language-level information whereas the mode is purely an internal
22126 GCC concept. We can therefore reach here for types that are not
22127 vectors in the AAPCS64 sense.
22129 We can't "fix" that for the traditional Advanced SIMD vector modes
22130 without breaking backwards compatibility. However, there's no such
22131 baggage for the structure modes, which were introduced in GCC 12. */
22135 /* For similar reasons, rely only on the type, not the mode, when
22136 processing SVE types. */
22138 /* Leave later code to report an error if SVE is disabled. */
22140 else
22142 }
22144 {
22145 /* 64-bit and 128-bit vectors should only acquire an SVE mode if
22146 they are being treated as scalable AAPCS64 types. */
22150 }
22152 }
22154 /* Return TRUE if the type, as described by TYPE and MODE, is a composite
22155 type as described in AAPCS64 \S 4.3. This includes aggregate, union and
22156 array types. The C99 floating-point complex types are also considered
22157 as composite types, according to AAPCS64 \S 7.1.1. The complex integer
22158 types, which are GCC extensions and out of the scope of AAPCS64, are
22159 treated as composite types here as well.
22161 Note that MODE itself is not sufficient in determining whether a type
22162 is such a composite type or not. This is because
22163 stor-layout.cc:compute_record_mode may have already changed the MODE
22164 (BLKmode) of a RECORD_TYPE TYPE to some other mode. For example, a
22165 structure with only one field may have its MODE set to the mode of the
22166 field. Also an integer mode whose size matches the size of the
22167 RECORD_TYPE type may be used to substitute the original mode
22168 (i.e. BLKmode) in certain circumstances. In other words, MODE cannot be
22169 solely relied on. */
22171 static bool
22173 machine_mode mode)
22174 {
22192 }
22194 /* Return TRUE if an argument, whose type is described by TYPE and MODE,
22195 shall be passed or returned in simd/fp register(s) (providing these
22196 parameter passing registers are available).
22198 Upon successful return, *COUNT returns the number of needed registers,
22199 *BASE_MODE returns the mode of the individual register and when IS_HA
22200 is not NULL, *IS_HA indicates whether or not the argument is a homogeneous
22201 floating-point aggregate or a homogeneous short-vector aggregate.
22203 SILENT_P is true if the function should refrain from reporting any
22204 diagnostics. This should only be used if the caller is certain that
22205 any ABI decisions would eventually come through this function with
22206 SILENT_P set to false. */
22208 static bool
22210 const_tree type,
22215 {
22226 {
22229 }
22231 {
22235 }
22237 {
22242 {
22247 && warn_psabi
22248 && warn_psabi_flags
22252 {
22259 /* Use TYPE_MAIN_VARIANT to strip any redundant const
22260 qualification. */
22263 "type %qT with %<[[no_unique_address]]%> members "
22268 "type %qT when C++17 is enabled changed to match "
22275 }
22279 }
22280 else
22282 }
22283 else
22289 }
22291 /* Implement TARGET_STRUCT_VALUE_RTX. */
22293 static rtx
22296 {
22298 }
22300 /* Implements target hook vector_mode_supported_p. */
22301 static bool
22303 {
22306 }
22308 /* Implements target hook vector_mode_supported_any_target_p. */
22309 static bool
22311 {
22314 }
22316 /* Return the full-width SVE vector mode for element mode MODE, if one
22317 exists. */
22318 opt_machine_mode
22320 {
22322 {
22341 }
22342 }
22344 /* Return the 64-bit Advanced SIMD vector mode for element mode MODE,
22345 if it exists. */
22346 opt_machine_mode
22348 {
22350 {
22365 }
22366 }
22368 /* Return the 128-bit Advanced SIMD vector mode for element mode MODE,
22369 if it exists. */
22370 opt_machine_mode
22372 {
22374 {
22393 }
22394 }
22396 /* Return appropriate SIMD container
22397 for MODE within a vector of WIDTH bits. */
22398 static machine_mode
22400 {
22408 {
22411 else
22413 }
22415 }
22417 /* Compare an SVE mode SVE_M and an Advanced SIMD mode ASIMD_M
22418 and return whether the SVE mode should be preferred over the
22419 Advanced SIMD one in aarch64_autovectorize_vector_modes. */
22420 static bool
22422 {
22423 /* Take into account the aarch64-autovec-preference param if non-zero. */
22432 /* The preference in case of a tie in costs. */
22438 /* If the CPU information does not have an SVE width registered use the
22439 generic poly_int comparison that prefers SVE. If a preference is
22440 explicitly requested avoid this path. */
22442 && !prefer_asimd
22446 /* Otherwise estimate the runtime width of the modes involved. */
22450 /* Preferring SVE means picking it first unless the Advanced SIMD mode
22451 is clearly wider. */
22454 /* Conversely, preferring Advanced SIMD means picking SVE only if SVE
22455 is clearly wider. */
22459 /* In the default case prefer Advanced SIMD over SVE in case of a tie. */
22461 }
22463 /* Return 128-bit container as the preferred SIMD mode for MODE. */
22464 static machine_mode
22466 {
22467 /* Take into account explicit auto-vectorization ISA preferences through
22468 aarch64_cmp_autovec_modes. */
22474 }
22476 /* Return a list of possible vector sizes for the vectorizer
22477 to iterate over. */
22478 static unsigned int
22480 {
22482 /* Try using full vectors for all element types. */
22483 VNx16QImode,
22485 /* Try using 16-bit containers for 8-bit elements and full vectors
22486 for wider elements. */
22487 VNx8QImode,
22489 /* Try using 32-bit containers for 8-bit and 16-bit elements and
22490 full vectors for wider elements. */
22491 VNx4QImode,
22493 /* Try using 64-bit containers for all element types. */
22494 VNx2QImode
22495 };
22498 /* Try using 128-bit vectors for all element types. */
22499 V16QImode,
22501 /* Try using 64-bit vectors for 8-bit elements and 128-bit vectors
22502 for wider elements. */
22503 V8QImode,
22505 /* Try using 64-bit vectors for 16-bit elements and 128-bit vectors
22506 for wider elements.
22508 TODO: We could support a limited form of V4QImode too, so that
22509 we use 32-bit vectors for 8-bit elements. */
22510 V4HImode,
22512 /* Try using 64-bit vectors for 32-bit elements and 128-bit vectors
22513 for 64-bit elements.
22515 TODO: We could similarly support limited forms of V2QImode and V2HImode
22516 for this case. */
22517 V2SImode
22518 };
22520 /* Try using N-byte SVE modes only after trying N-byte Advanced SIMD mode.
22521 This is because:
22523 - If we can't use N-byte Advanced SIMD vectors then the placement
22524 doesn't matter; we'll just continue as though the Advanced SIMD
22525 entry didn't exist.
22527 - If an SVE main loop with N bytes ends up being cheaper than an
22528 Advanced SIMD main loop with N bytes then by default we'll replace
22529 the Advanced SIMD version with the SVE one.
22531 - If an Advanced SIMD main loop with N bytes ends up being cheaper
22532 than an SVE main loop with N bytes then by default we'll try to
22533 use the SVE loop to vectorize the epilogue instead. */
22542 {
22547 else
22549 }
22557 }
22559 /* Implement TARGET_MANGLE_TYPE. */
22563 {
22564 /* The AArch64 ABI documents say that "__va_list" has to be
22565 mangled as if it is in the "std" namespace. */
22569 /* Half-precision floating point types. */
22571 {
22576 else
22578 }
22580 /* Modal 8 bit floating point types. */
22584 /* Mangle AArch64-specific internal types. TYPE_NAME is non-NULL_TREE for
22585 builtin types. */
22587 {
22592 }
22594 /* Use the default mangling. */
22596 }
22598 /* Implement TARGET_INVALID_CONVERSION. */
22602 {
22603 /* Do not allow conversions to/from FP8. But do allow conversions between
22604 volatile and const variants of __mfp8. */
22605 bool fromtype_is_fp8
22617 /* Conversion allowed. */
22619 }
22621 /* Implement TARGET_VERIFY_TYPE_CONTEXT. */
22623 static bool
22626 {
22628 }
22630 /* Find the first rtx_insn before insn that will generate an assembly
22631 instruction. */
22635 {
22639 do
22640 {
22642 }
22646 }
22648 static bool
22650 {
22652 /* A number of these may be AArch32 only. */
22657 };
22660 {
22663 }
22666 }
22668 /* Check if there is a register dependency between a load and the insn
22669 for which we hold recog_data. */
22671 static bool
22673 {
22674 rtx load_reg;
22684 {
22690 }
22692 }
22695 /* When working around the Cortex-A53 erratum 835769,
22696 given rtx_insn INSN, return true if it is a 64-bit multiply-accumulate
22697 instruction and has a preceding memory instruction such that a NOP
22698 should be inserted between them. */
22700 bool
22702 {
22705 rtx body;
22718 /* aarch64_prev_real_insn can call recog_memoized on insns other than INSN.
22719 Restore recog state to INSN to avoid state corruption. */
22727 /* If the previous insn is a memory op and there is no dependency between
22728 it and the DImode madd, emit a NOP between them. If body is NULL then we
22729 have a complex memory operation, probably a load/store pair.
22730 Be conservative for now and emit a NOP. */
22737 }
22740 /* Implement FINAL_PRESCAN_INSN. */
22742 void
22744 {
22747 }
22750 /* Return true if BASE_OR_STEP is a valid immediate operand for an SVE INDEX
22751 instruction. */
22753 bool
22755 {
22758 }
22760 /* Return true if X is a valid immediate for the SVE ADD and SUB instructions
22761 when applied to mode MODE. Negate X first if NEGATE_P is true. */
22763 bool
22765 {
22778 }
22780 /* Return true if X is a valid immediate for the SVE SQADD and SQSUB
22781 instructions when applied to mode MODE. Negate X first if NEGATE_P
22782 is true. */
22784 bool
22786 {
22790 /* After the optional negation, the immediate must be nonnegative.
22791 E.g. a saturating add of -127 must be done via SQSUB Zn.B, Zn.B, #127
22792 instead of SQADD Zn.B, Zn.B, #129. */
22795 }
22797 /* Return true if X is a valid immediate operand for an SVE logical
22798 instruction such as AND. */
22800 bool
22802 {
22803 rtx elt;
22809 }
22811 /* Return true if X is a valid immediate for the SVE DUP and CPY
22812 instructions. */
22814 bool
22816 {
22825 }
22827 /* Return true if X is a valid immediate operand for an SVE CMP instruction.
22828 SIGNED_P says whether the operand is signed rather than unsigned. */
22830 bool
22832 {
22835 && (signed_p
22838 }
22840 /* Return true if X is a valid immediate operand for an SVE FADD or FSUB
22841 instruction. Negate X first if NEGATE_P is true. */
22843 bool
22845 {
22846 rtx elt;
22847 REAL_VALUE_TYPE r;
22864 }
22866 /* Return true if X is a valid immediate operand for an SVE FMUL
22867 instruction. */
22869 bool
22871 {
22872 rtx elt;
22879 }
22881 /* Return true if replicating VAL32 is a valid 2-byte or 4-byte immediate
22882 for the Advanced SIMD operation described by WHICH and INSN. If INFO
22883 is nonnull, use it to describe valid immediates. */
22884 static bool
22889 {
22890 /* Try a 4-byte immediate with LSL. */
22893 {
22898 }
22900 /* Try a 2-byte immediate with LSL. */
22905 {
22910 }
22912 /* Try a 4-byte immediate with MSL, except for cases that MVN
22913 can handle. */
22916 {
22919 {
22924 }
22925 }
22928 }
22930 /* Return true if replicating VAL64 with mode MODE is a valid immediate for the
22931 Advanced SIMD operation described by WHICH. If INFO is nonnull,
22932 use it to describe valid immediates. */
22933 static bool
22935 scalar_int_mode mode,
22938 {
22943 {
22954 /* Try using a replicated byte. */
22956 {
22960 }
22961 }
22963 /* Try using a bit-to-bytemask. */
22965 {
22968 {
22972 }
22974 {
22978 }
22979 }
22981 }
22983 /* Return true if replicating IVAL with MODE gives a valid immediate for an SVE
22984 MOV instruction. If INFO is nonnull, use it to describe valid
22985 immediates. */
22987 static bool
22991 {
22995 {
22997 {
22998 /* DUP with no shift. */
23003 }
23005 {
23006 /* DUP with LSL #8. */
23011 }
23012 }
23014 {
23015 /* DUPM. */
23019 }
23021 }
23023 /* Return true if X is an UNSPEC_PTRUE constant of the form:
23025 (const (unspec [PATTERN ZERO] UNSPEC_PTRUE))
23027 where PATTERN is the svpattern as a CONST_INT and where ZERO
23028 is a zero constant of the required PTRUE mode (which can have
23029 fewer elements than X's mode, if zero bits are significant).
23031 If so, and if INFO is nonnull, describe the immediate in INFO. */
23032 bool
23034 {
23043 {
23044 aarch64_svpattern pattern
23049 }
23051 }
23053 /* Return true if X is a valid SVE predicate. If INFO is nonnull, use
23054 it to describe valid immediates. */
23056 static bool
23058 {
23063 {
23067 }
23069 /* Analyze the value as a VNx16BImode. This should be relatively
23070 efficient, since rtx_vector_builder has enough built-in capacity
23071 to store all VLA predicate constants without needing the heap. */
23072 rtx_vector_builder builder;
23078 {
23082 {
23084 {
23087 }
23089 }
23090 }
23092 }
23094 /* We can only represent floating point constants which will fit in
23095 "quarter-precision" values. These values are characterised by
23096 a sign bit, a 4-bit mantissa and a 3-bit exponent. And are given
23097 by:
23099 (-1)^s * (n/16) * 2^r
23101 Where:
23102 's' is the sign bit.
23103 'n' is an integer in the range 16 <= n <= 31.
23104 'r' is an integer in the range -3 <= r <= 4.
23106 Return true iff R represents a vale encodable into an AArch64 floating point
23107 move instruction as an immediate. Othewise false. */
23109 static bool
23111 {
23112 /* This represents our current view of how many bits
23113 make up the mantissa. */
23117 REAL_VALUE_TYPE m;
23120 /* We cannot represent infinities, NaNs or +/-zero. We won't
23121 know if we have +zero until we analyse the mantissa, but we
23122 can reject the other invalid values. */
23127 /* Extract exponent. */
23131 /* For the mantissa, we expand into two HOST_WIDE_INTS, apart from the
23132 highest (sign) bit, with a fixed binary point at bit point_pos.
23133 m1 holds the low part of the mantissa, m2 the high part.
23134 WARNING: If we ever have a representation using more than 2 * H_W_I - 1
23135 bits for the mantissa, this can fail (low bits will be lost). */
23139 /* If the low part of the mantissa has bits set we cannot represent
23140 the value. */
23144 /* We have rejected the lower HOST_WIDE_INT, so update our
23145 understanding of how many bits lie in the mantissa and
23146 look only at the high HOST_WIDE_INT. */
23150 /* We can only represent values with a mantissa of the form 1.xxxx. */
23155 /* Having filtered unrepresentable values, we may now remove all
23156 but the highest 5 bits. */
23159 /* We cannot represent the value 0.0, so reject it. This is handled
23160 elsewhere. */
23164 /* Then, as bit 4 is always set, we can mask it off, leaving
23165 the mantissa in the range [0, 15]. */
23169 /* GCC internally does not use IEEE754-like encoding (where normalized
23170 significands are in the range [1, 2). GCC uses [0.5, 1) (see real.cc).
23171 Our mantissa values are shifted 4 places to the left relative to
23172 normalized IEEE754 so we must modify the exponent returned by REAL_EXP
23173 by 5 places to correct for GCC's representation. */
23177 }
23179 /* Return true if OP is a valid SIMD immediate for the operation
23180 described by WHICH. If INFO is nonnull, use it to describe valid
23181 immediates. */
23182 static bool
23185 {
23207 && TARGET_SVE
23209 {
23216 {
23217 /* Get the corresponding container mode. E.g. an INDEX on V2SI
23218 should yield two integer values per 128-bit block, meaning
23219 that we need to treat it in the same way as V2DI and then
23220 ignore the upper 32 bits of each element. */
23223 }
23225 }
23229 else
23232 /* If all elements in an SVE vector have the same value, we have a free
23233 choice between using the element mode and using the container mode.
23234 Using the element mode means that unused parts of the vector are
23235 duplicates of the used elements, while using the container mode means
23236 that the unused parts are an extension of the used elements. Using the
23237 element mode is better for (say) VNx4HI 0x101, since 0x01010101 is valid
23238 for its container mode VNx4SI while 0x00000101 isn't.
23240 If not all elements in an SVE vector have the same value, we need the
23241 transition from one element to the next to occur at container boundaries.
23242 E.g. a fixed-length VNx4HI containing { 1, 2, 3, 4 } should be treated
23243 in the same way as a VNx4SI containing { 1, 2, 3, 4 }. */
23244 scalar_int_mode elt_int_mode;
23247 else
23254 /* Expand the vector constant out into a byte vector, with the least
23255 significant byte of the register first. */
23259 {
23260 /* The vector is provided in gcc endian-neutral fashion.
23261 For aarch64_be Advanced SIMD, it must be laid out in the vector
23262 register in reverse order. */
23274 {
23277 }
23278 }
23280 /* The immediate must repeat every eight bytes. */
23286 /* Get the repeating 8-byte value as an integer. No endian correction
23287 is needed here because bytes is already in lsb-first order. */
23293 /* Try encoding the integer immediate as a floating point value if it's an
23294 exact value. */
23300 {
23306 {
23312 {
23315 }
23316 }
23317 }
23322 {
23327 REAL_VALUE_TYPE r;
23330 {
23332 {
23335 }
23337 }
23338 }
23349 }
23351 /* Return true if OP is a valid SIMD move immediate for SVE or AdvSIMD. */
23352 bool
23354 {
23356 }
23358 /* Return true if OP is a valid SIMD orr immediate for SVE or AdvSIMD. */
23359 bool
23361 {
23363 }
23365 /* Return true if OP is a valid SIMD and immediate for SVE or AdvSIMD. */
23366 bool
23368 {
23370 }
23372 /* Return true if OP is a valid SIMD xor immediate for SVE. */
23373 bool
23375 {
23377 }
23379 /* Check whether X is a VEC_SERIES-like constant that starts at 0 and
23380 has a step in the range of INDEX. Return the index expression if so,
23381 otherwise return null. */
23382 rtx
23384 {
23391 }
23393 /* Check of immediate shift constants are within range. */
23394 bool
23396 {
23403 else
23405 }
23407 /* Return the bitmask CONST_INT to select the bits required by a zero extract
23408 operation of width WIDTH at bit position POS. */
23410 rtx
23412 {
23416 unsigned HOST_WIDE_INT mask
23419 }
23421 bool
23423 {
23432 {
23433 /* Require predicate constants to be VNx16BI before RA, so that we
23434 force everything to have a canonical form. */
23436 && !reload_completed
23443 }
23445 /* Remove UNSPEC_SALT_ADDR before checking symbol reference. */
23448 /* GOT accesses are valid moves. */
23466 }
23468 /* Return a function-invariant register that contains VALUE. *CACHED_INSN
23469 caches instructions that set up such registers, so that they can be
23470 reused by future calls. */
23472 static rtx
23474 {
23477 {
23480 {
23486 }
23487 }
23490 function_beg_insn);
23492 }
23494 /* Create a 0 constant that is based on V4SI to allow CSE to optimally share
23495 the constant creation. */
23497 rtx
23499 {
23503 }
23505 /* INSN is some form of extension or shift that can be split into a
23506 permutation involving a shared zero. Return true if we should
23507 perform such a split.
23509 ??? For now, make sure that the split instruction executes more
23510 frequently than the zero that feeds it. In future it would be good
23511 to split without that restriction and instead recombine shared zeros
23512 if they turn out not to be worthwhile. This would allow splits in
23513 single-block functions and would also cope more naturally with
23514 rematerialization. The downside of not doing this is that we lose the
23515 optimizations for vector epilogues as well. */
23517 bool
23519 {
23524 }
23526 /* Return a const_int vector of VAL. */
23527 rtx
23529 {
23532 }
23534 /* Check OP is a legal scalar immediate for the MOVI instruction. */
23536 bool
23538 {
23539 machine_mode vmode;
23544 }
23546 /* Construct and return a PARALLEL RTX vector with elements numbering the
23547 lanes of either the high (HIGH == TRUE) or low (HIGH == FALSE) half of
23548 the vector - from the perspective of the architecture. This does not
23549 line up with GCC's perspective on lane numbers, so we end up with
23550 different masks depending on our target endian-ness. The diagram
23551 below may help. We must draw the distinction when building masks
23552 which select one half of the vector. An instruction selecting
23553 architectural low-lanes for a big-endian target, must be described using
23554 a mask selecting GCC high-lanes.
23556 Big-Endian Little-Endian
23558 GCC 0 1 2 3 3 2 1 0
23559 | x | x | x | x | | x | x | x | x |
23560 Architecture 3 2 1 0 3 2 1 0
23562 Low Mask: { 2, 3 } { 0, 1 }
23563 High Mask: { 0, 1 } { 2, 3 }
23565 MODE Is the mode of the vector and NUNITS is the number of units in it. */
23567 rtx
23569 {
23574 rtx t1;
23579 else
23587 }
23589 /* Check OP for validity as a PARALLEL RTX vector with elements
23590 numbering the lanes of either the high (HIGH == TRUE) or low lanes,
23591 from the perspective of the architecture. See the diagram above
23592 aarch64_simd_vect_par_cnst_half for more details. */
23594 bool
23597 {
23611 {
23618 }
23620 }
23622 /* Return a PARALLEL containing NELTS elements, with element I equal
23623 to BASE + I * STEP. */
23625 rtx
23627 {
23632 }
23634 /* Return true if OP is a PARALLEL of CONST_INTs that form a linear
23635 series with step STEP. */
23637 bool
23639 {
23650 }
23652 /* Return true if OPERANDS[0] to OPERANDS[NUM_OPERANDS - 1] form a
23653 sequence of strided registers, with the stride being equal STRIDE.
23654 The operands are already known to be FPRs. */
23655 bool
23658 {
23663 }
23665 /* Bounds-check lanes. Ensure OPERAND lies between LOW (inclusive) and
23666 HIGH (exclusive). */
23667 void
23669 const_tree exp)
23670 {
23671 HOST_WIDE_INT lane;
23676 {
23680 else
23682 }
23683 }
23685 /* Peform endian correction on lane number N, which indexes a vector
23686 of mode MODE, and return the result as an SImode rtx. */
23688 rtx
23690 {
23692 }
23694 /* Return TRUE if OP is a valid vector addressing mode. */
23696 bool
23698 {
23702 }
23704 /* Return true if OP is a valid MEM operand for an SVE LD1R instruction. */
23706 bool
23708 {
23710 scalar_mode mode;
23717 }
23719 /* Return true if OP is a valid MEM operand for an SVE LD1R{Q,O} instruction
23720 where the size of the read data is specified by `mode` and the size of the
23721 vector elements are specified by `elem_mode`. */
23722 bool
23724 scalar_mode elem_mode)
23725 {
23738 }
23740 /* Return true if OP is a valid MEM operand for an SVE LD1RQ instruction. */
23741 bool
23743 {
23746 }
23748 /* Return true if OP is a valid MEM operand for an SVE LD1RO instruction for
23749 accessing a vector where the element size is specified by `elem_mode`. */
23750 bool
23752 {
23754 }
23756 /* Return true if OP is a valid MEM operand for an SVE LDFF1 instruction. */
23757 bool
23759 {
23771 }
23773 /* Return true if OP is a valid MEM operand for an SVE LDNF1 instruction. */
23774 bool
23776 {
23783 }
23785 /* Return true if OP is a valid MEM operand for an SVE LDR instruction.
23786 The conditions for STR are the same. */
23787 bool
23789 {
23796 }
23798 /* Return true if OP is a valid address for an SVE PRF[BHWD] instruction,
23799 addressing memory of mode MODE. */
23800 bool
23802 {
23811 }
23813 /* Return true if OP is a valid MEM operand for an SVE_STRUCT mode.
23814 We need to be able to access the individual pieces, so the range
23815 is different from LD[234] and ST[234]. */
23816 bool
23818 {
23825 ADDR_QUERY_ANY)
23833 }
23835 /* Return true if OFFSET is a constant integer and if VNUM is
23836 OFFSET * the number of bytes in an SVE vector. This is the requirement
23837 that exists in SME LDR and STR instructions, where the VL offset must
23838 equal the ZA slice offset. */
23839 bool
23841 {
23846 {
23847 poly_int64 const_vnum;
23851 }
23852 else
23853 {
23854 HOST_WIDE_INT factor;
23857 }
23858 }
23860 /* Emit a register copy from operand to operand, taking care not to
23861 early-clobber source registers in the process.
23863 COUNT is the number of components into which the copy needs to be
23864 decomposed. */
23865 void
23868 {
23878 else
23882 }
23884 /* Compute and return the length of aarch64_simd_reglist<mode>, where <mode> is
23885 one of VSTRUCT modes: OI, CI, or XI. */
23886 int
23888 {
23889 /* This is only used (and only meaningful) for Advanced SIMD, not SVE. */
23891 }
23893 /* Implement target hook TARGET_VECTOR_ALIGNMENT. The AAPCS64 sets the maximum
23894 alignment of a vector to 128 bits. SVE predicates have an alignment of
23895 16 bits. */
23896 static HOST_WIDE_INT
23898 {
23899 /* ??? Checking the mode isn't ideal, but VECTOR_BOOLEAN_TYPE_P can
23900 be set for non-predicate vectors of booleans. Modes are the most
23901 direct way we have of identifying real SVE predicate types. */
23904 widest_int min_size
23907 }
23909 /* Implement target hook TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT. */
23910 static poly_uint64
23912 {
23914 {
23915 /* If the length of the vector is a fixed power of 2, try to align
23916 to that length, otherwise don't try to align at all. */
23917 HOST_WIDE_INT result;
23922 }
23924 }
23926 /* Implement target hook TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE. */
23927 static bool
23929 {
23933 /* For fixed-length vectors, check that the vectorizer will aim for
23934 full-vector alignment. This isn't true for generic GCC vectors
23935 that are wider than the ABI maximum of 128 bits. */
23936 poly_uint64 preferred_alignment =
23940 preferred_alignment))
23943 /* Vectors whose size is <= BIGGEST_ALIGNMENT are naturally aligned. */
23945 }
23947 /* Return true if the vector misalignment factor is supported by the
23948 target. */
23949 static bool
23953 {
23955 {
23956 /* Return if movmisalign pattern is not supported for this mode. */
23960 /* Misalignment factor is unknown at compile time. */
23963 }
23965 is_packed);
23966 }
23968 /* If VALS is a vector constant that can be loaded into a register
23969 using DUP, generate instructions to do so and return an RTX to
23970 assign to the register. Otherwise return NULL_RTX. */
23971 static rtx
23973 {
23976 rtx x;
23981 /* We can load this constant by using DUP and a constant in a
23982 single ARM register. This will be cheaper than a vector
23983 load. */
23986 }
23989 /* Generate code to load VALS, which is a PARALLEL containing only
23990 constants (for vec_init) or CONST_VECTOR, efficiently into a
23991 register. Returns an RTX to copy into the register, or NULL_RTX
23992 for a PARALLEL that cannot be converted into a CONST_VECTOR. */
23993 static rtx
23995 {
23997 rtx const_dup;
24005 {
24006 /* A CONST_VECTOR must contain only CONST_INTs and
24007 CONST_DOUBLEs, but CONSTANT_P allows more (e.g. SYMBOL_REF).
24008 Only store valid constants in a CONST_VECTOR. */
24011 {
24014 n_const++;
24015 }
24018 }
24019 else
24024 /* Load using MOVI/MVNI. */
24027 /* Loaded using DUP. */
24030 /* Load from constant pool. We cannot take advantage of single-cycle
24031 LD1 because we need a PC-relative addressing mode. */
24033 else
24034 /* A PARALLEL containing something not valid inside CONST_VECTOR.
24035 We cannot construct an initializer. */
24037 }
24039 /* A subroutine of aarch64_expand_vector_init, with the same interface.
24040 The caller has already tried a divide-and-conquer approach, so do
24041 not consider that case here. */
24043 void
24045 {
24048 /* The number of vector elements. */
24050 /* The number of vector elements which are not constant. */
24053 /* The first element of vals. */
24057 /* This is a special vec_init<M><N> where N is not an element mode but a
24058 vector mode with half the elements of M. We expect to find two entries
24059 of mode N in VALS and we must put their concatentation into TARGET. */
24061 {
24070 }
24072 /* Count the number of variable elements to initialise. */
24074 {
24078 else
24082 }
24084 /* No variable elements, hand off to aarch64_simd_make_constant which knows
24085 how best to handle this. */
24087 {
24090 {
24093 }
24094 }
24096 /* Splat a single non-constant element if we can. */
24098 {
24102 }
24107 /* If there are only variable elements, try to optimize
24108 the insertion using dup for the most common element
24109 followed by insertions. */
24111 /* The algorithm will fill matches[*][0] with the earliest matching element,
24112 and matches[X][1] with the count of duplicate elements (if X is the
24113 earliest element which has duplicates). */
24116 {
24119 {
24121 {
24123 {
24127 }
24128 }
24129 }
24136 {
24138 {
24141 }
24144 {
24147 }
24148 }
24150 /* Create a duplicate of the most common element, unless all elements
24151 are equally useless to us, in which case just immediately set the
24152 vector register using the first element. */
24155 {
24156 /* For vectors of two 64-bit elements, we can do even better. */
24161 {
24164 /* Combine can pick up this case, but handling it directly
24165 here leaves clearer RTL.
24167 This is load_pair_lanes<mode>, and also gives us a clean-up
24168 for store_pair_lanes<mode>. */
24172 {
24173 rtx t;
24176 else
24180 }
24181 }
24182 /* The subreg-move sequence below will move into lane zero of the
24183 vector register. For big-endian we want that position to hold
24184 the last element of VALS. */
24187 /* If we have a single constant element, use that for duplicating
24188 instead. */
24190 {
24193 }
24194 else
24195 {
24198 }
24199 }
24200 else
24201 {
24204 }
24206 /* Insert the rest. */
24208 {
24214 }
24216 }
24218 /* Initialise a vector which is part-variable. We want to first try
24219 to build those lanes which are constant in the most efficient way we
24220 can. */
24222 {
24225 /* Load constant part of vector. We really don't care what goes into the
24226 parts we will overwrite, but we're more likely to be able to load the
24227 constant efficiently if it has fewer, larger, repeating parts
24228 (see aarch64_simd_valid_imm). */
24230 {
24236 {
24237 /* Look in the copied vector, as more elements are const. */
24240 {
24243 }
24244 }
24246 }
24248 }
24250 /* Insert the variable lanes directly. */
24252 {
24258 }
24259 }
24261 /* Return even or odd half of VALS depending on EVEN_P. */
24263 static rtx
24265 {
24267 machine_mode new_mode
24275 }
24277 /* Return true if SET is a scalar move. */
24279 static bool
24281 {
24286 }
24288 /* Similar to seq_cost, but ignore cost for scalar moves. */
24290 static unsigned
24292 {
24297 {
24299 {
24302 }
24303 else
24304 {
24308 else
24309 cost++;
24310 }
24311 }
24314 }
24316 /* Expand a vector initialization sequence, such that TARGET is
24317 initialized to contain VALS. */
24319 void
24321 {
24322 /* Try decomposing the initializer into even and odd halves and
24323 then ZIP them together. Use the resulting sequence if it is
24324 strictly cheaper than loading VALS directly.
24326 Prefer the fallback sequence in the event of a tie, since it
24327 will tend to use fewer registers. */
24334 {
24337 }
24343 {
24353 }
24356 rtx_insn *zip1_insn
24358 unsigned seq_total_cost
24368 unsigned fallback_seq_cost
24373 }
24375 /* Emit RTL corresponding to:
24376 insr TARGET, ELEM. */
24378 static void
24380 {
24388 }
24390 /* Subroutine of aarch64_sve_expand_vector_init for handling
24391 trailing constants.
24392 This function works as follows:
24393 (a) Create a new vector consisting of trailing constants.
24394 (b) Initialize TARGET with the constant vector using emit_move_insn.
24395 (c) Insert remaining elements in TARGET using insr.
24396 NELTS is the total number of elements in original vector while
24397 while NELTS_REQD is the number of elements that are actually
24398 significant.
24400 ??? The heuristic used is to do above only if number of constants
24401 is at least half the total number of elements. May need fine tuning. */
24403 static bool
24404 aarch64_sve_expand_vector_init_handle_trailing_constants
24406 {
24413 i--)
24414 n_trailing_constants++;
24417 {
24418 /* Try to use the natural pattern of BUILDER to extend the trailing
24419 constant elements to a full vector. Replace any variables in the
24420 extra elements with zeros.
24422 ??? It would be better if the builders supported "don't care"
24423 elements, with the builder filling in whichever elements
24424 give the most compact encoding. */
24427 {
24432 }
24440 }
24443 }
24445 /* Subroutine of aarch64_sve_expand_vector_init.
24446 Works as follows:
24447 (a) Initialize TARGET by broadcasting element NELTS_REQD - 1 of BUILDER.
24448 (b) Skip trailing elements from BUILDER, which are the same as
24449 element NELTS_REQD - 1.
24450 (c) Insert earlier elements in reverse order in TARGET using insr. */
24452 static void
24456 {
24471 }
24473 /* Subroutine of aarch64_sve_expand_vector_init to handle case
24474 when all trailing elements of builder are same.
24475 This works as follows:
24476 (a) Use expand_insn interface to broadcast last vector element in TARGET.
24477 (b) Insert remaining elements in TARGET using insr.
24479 ??? The heuristic used is to do above if number of same trailing elements
24480 is at least 3/4 of total number of elements, loosely based on
24481 heuristic from mostly_zeros_p. May need fine-tuning. */
24483 static bool
24484 aarch64_sve_expand_vector_init_handle_trailing_same_elem
24486 {
24489 {
24493 }
24496 }
24498 /* Initialize register TARGET from BUILDER. NELTS is the constant number
24499 of elements in BUILDER.
24501 The function tries to initialize TARGET from BUILDER if it fits one
24502 of the special cases outlined below.
24504 Failing that, the function divides BUILDER into two sub-vectors:
24505 v_even = even elements of BUILDER;
24506 v_odd = odd elements of BUILDER;
24508 and recursively calls itself with v_even and v_odd.
24510 if (recursive call succeeded for v_even or v_odd)
24511 TARGET = zip (v_even, v_odd)
24513 The function returns true if it managed to build TARGET from BUILDER
24514 with one of the special cases, false otherwise.
24516 Example: {a, 1, b, 2, c, 3, d, 4}
24518 The vector gets divided into:
24519 v_even = {a, b, c, d}
24520 v_odd = {1, 2, 3, 4}
24522 aarch64_sve_expand_vector_init(v_odd) hits case 1 and
24523 initialize tmp2 from constant vector v_odd using emit_move_insn.
24525 aarch64_sve_expand_vector_init(v_even) fails since v_even contains
24526 4 elements, so we construct tmp1 from v_even using insr:
24527 tmp1 = dup(d)
24528 insr tmp1, c
24529 insr tmp1, b
24530 insr tmp1, a
24532 And finally:
24533 TARGET = zip (tmp1, tmp2)
24534 which sets TARGET to {a, 1, b, 2, c, 3, d, 4}. */
24536 static bool
24539 {
24542 /* Case 1: Vector contains trailing constants. */
24548 /* Case 2: Vector contains leading constants. */
24557 {
24560 }
24562 /* Case 3: Vector contains trailing same element. */
24568 /* Case 4: Vector contains leading same element. */
24572 {
24575 }
24577 /* Avoid recursing below 4-elements.
24578 ??? The threshold 4 may need fine-tuning. */
24587 {
24590 }
24606 /* Initialize v_even and v_odd using INSR if it didn't match any of the
24607 special cases and zip v_even, v_odd. */
24618 }
24620 /* Initialize register TARGET from the elements in PARALLEL rtx VALS. */
24622 void
24624 {
24633 /* If neither sub-vectors of v could be initialized specially,
24634 then use INSR to insert all elements from v into TARGET.
24635 ??? This might not be optimal for vectors with large
24636 initializers like 16-element or above.
24637 For nelts < 4, it probably isn't useful to handle specially. */
24642 }
24644 /* Initialize register TARGET from the two vector subelements in PARALLEL
24645 rtx VALS. */
24647 void
24649 {
24655 /* We have to be concatting vector. */
24665 /* Keep widening pairwise to have maximum throughput. */
24667 {
24668 wider_mode
24673 {
24681 }
24684 }
24690 }
24692 /* Check whether VALUE is a vector constant in which every element
24693 is either a power of 2 or a negated power of 2. If so, return
24694 a constant vector of log2s, and flip CODE between PLUS and MINUS
24695 if VALUE contains negated powers of 2. Return NULL_RTX otherwise. */
24697 static rtx
24699 {
24703 rtx_vector_builder builder;
24708 /* 1 if the result of the multiplication must be negated,
24709 0 if it mustn't, or -1 if we don't yet care. */
24713 {
24720 {
24721 /* It matters whether we negate or not. Make that choice,
24722 and make sure that it's consistent with previous elements. */
24728 }
24729 /* POW2 is now the value that we want to be a power of 2. */
24734 }
24736 /* PLUS and MINUS are equivalent; canonicalize on PLUS. */
24741 }
24743 /* Prepare for an integer SVE multiply-add or multiply-subtract pattern;
24744 CODE is PLUS for the former and MINUS for the latter. OPERANDS is the
24745 operands array, in the same order as for fma_optab. Return true if
24746 the function emitted all the necessary instructions, false if the caller
24747 should generate the pattern normally with the new OPERANDS array. */
24749 bool
24751 {
24754 {
24759 OPTAB_DIRECT);
24761 }
24764 }
24766 /* Likewise, but for a conditional pattern. */
24768 bool
24770 {
24773 {
24779 }
24782 }
24784 static unsigned HOST_WIDE_INT
24786 {
24790 }
24792 /* Select a format to encode pointers in exception handling data. */
24793 int
24795 {
24798 {
24804 /* text+got+data < 4Gb. 4-byte signed relocs are sufficient
24805 for everything. */
24809 /* No assumptions here. 8-byte relocs required. */
24812 }
24814 }
24816 /* Output .variant_pcs for aarch64_vector_pcs function symbols. */
24818 static void
24820 {
24822 {
24825 {
24829 }
24830 }
24831 }
24833 /* The last .arch and .tune assembly strings that we printed. */
24837 /* Implement ASM_DECLARE_FUNCTION_NAME. Output the ISA features used
24838 by the function fndecl. */
24840 void
24842 tree fndecl)
24843 {
24849 else
24857 /* Only update the assembler .arch string if it is distinct from the last
24858 such string we printed. */
24860 {
24863 }
24865 /* Print the cpu name we're tuning for in the comments, might be
24866 useful to readers of the generated asm. Do it only when it changes
24867 from function to function and verbose assembly is requested. */
24872 {
24876 }
24880 /* Don't forget the type directive for ELF. */
24885 }
24887 /* Implement PRINT_PATCHABLE_FUNCTION_ENTRY. */
24889 void
24893 {
24895 {
24896 /* Emit the patching area before the entry label, if any. */
24898 record_p);
24900 }
24908 {
24909 /* Emit the patchable_area at the beginning of the function. */
24913 }
24920 {
24921 /* Emit a BTI_C. */
24923 }
24925 /* Emit the patchable_area after BTI_C. */
24928 }
24930 /* Output patchable area. */
24932 void
24934 {
24936 record_p);
24937 }
24939 /* Implement ASM_OUTPUT_DEF_FROM_DECLS. Output .variant_pcs for aliases. */
24941 void
24943 {
24948 }
24950 /* Implement ASM_OUTPUT_EXTERNAL. Output .variant_pcs for undefined
24951 function symbol references. */
24953 void
24955 {
24958 }
24960 /* Triggered after a .cfi_startproc directive is emitted into the assembly file.
24961 Used to output the .cfi_b_key_frame directive when signing the current
24962 function with the B key. */
24964 void
24966 {
24970 }
24972 /* Implements TARGET_ASM_FILE_START. Output the assembly header. */
24974 static void
24976 {
24990 }
24992 /* Emit load exclusive. */
24994 static void
24997 {
25002 else
25004 }
25006 /* Emit store exclusive. */
25008 static void
25011 {
25016 else
25018 }
25020 /* Mark the previous jump instruction as unlikely. */
25022 static void
25024 {
25027 }
25029 /* We store the names of the various atomic helpers in a 5x5 array.
25030 Return the libcall function given MODE, MODEL and NAMES. */
25032 rtx
25035 {
25040 {
25058 }
25061 {
25083 }
25086 VISIBILITY_HIDDEN);
25087 }
25089 #define DEF0(B, N) \
25096 #define DEF4(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8), \
25097 { NULL, NULL, NULL, NULL }
25098 #define DEF5(B) DEF0(B, 1), DEF0(B, 2), DEF0(B, 4), DEF0(B, 8), DEF0(B, 16)
25107 #undef DEF0
25108 #undef DEF4
25109 #undef DEF5
25111 /* Expand a compare and swap pattern. */
25113 void
25115 {
25129 /* Normally the succ memory model must be stronger than fail, but in the
25130 unlikely event of fail being ACQUIRE and succ being RELEASE we need to
25131 promote succ to ACQ_REL so that we don't lose the acquire semantics. */
25138 {
25141 }
25144 {
25145 /* The CAS insn requires oldval and rval overlap, but we need to
25146 have a copy of oldval saved across the operation to tell if
25147 the operation is successful. */
25150 else
25158 }
25160 {
25161 /* Oldval must satisfy compare afterward. */
25169 }
25170 else
25171 {
25172 /* The oldval predicate varies by mode. Test it and force to reg. */
25180 }
25188 }
25190 /* Emit a barrier, that is appropriate for memory model MODEL, at the end of a
25191 sequence implementing an atomic operation. */
25193 static void
25195 {
25202 {
25204 }
25205 }
25207 /* Split a compare and swap pattern. */
25209 void
25211 {
25212 /* Split after prolog/epilog to avoid interactions with shrinkwrapping. */
25216 machine_mode mode;
25231 /* When OLDVAL is zero and we want the strong version we can emit a tighter
25232 loop:
25233 .label1:
25234 LD[A]XR rval, [mem]
25235 CBNZ rval, .label2
25236 ST[L]XR scratch, newval, [mem]
25237 CBNZ scratch, .label1
25238 .label2:
25239 CMP rval, 0. */
25245 {
25248 }
25251 /* The initial load can be relaxed for a __sync operation since a final
25252 barrier will be emitted to stop code hoisting. */
25255 else
25260 else
25261 {
25264 }
25272 {
25275 }
25276 else
25279 /* 128-bit LDAXP is not atomic unless STLXP succeeds. So for a mismatch,
25280 store the returned value and loop if the STLXP fails. */
25282 {
25294 }
25298 /* If we used a CBNZ in the exchange loop emit an explicit compare with RVAL
25299 to set the condition flags. If this is not used it will be removed by
25300 later passes. */
25304 /* Emit any final barrier needed for a __sync operation. */
25307 }
25309 /* Split an atomic operation. */
25311 void
25314 {
25315 /* Split after prolog/epilog to avoid interactions with shrinkwrapping. */
25323 rtx x;
25325 /* Split the atomic operation into a sequence. */
25333 else
25337 /* The initial load can be relaxed for a __sync operation since a final
25338 barrier will be emitted to stop code hoisting. */
25342 else
25346 {
25360 {
25363 }
25364 /* Fall through. */
25370 }
25378 /* Emit any final barrier needed for a __sync operation. */
25381 }
25383 static void
25385 {
25386 /* Half-precision float operations. The compiler handles all operations
25387 with NULL libfuncs by converting to SFmode. */
25389 /* Conversions. */
25393 /* Arithmetic. */
25400 /* Comparisons. */
25408 }
25410 /* Target hook for c_mode_for_suffix. */
25411 static machine_mode
25413 {
25418 }
25420 /* Return true iff X with mode MODE can be represented by a quarter-precision
25421 floating point immediate operand X. Note, we cannot represent 0.0. */
25423 bool
25425 {
25438 }
25440 /* Returns the string with the instruction for the SIMD immediate
25441 * CONST_VECTOR of MODE and WIDTH. WHICH selects a move, and(bic) or orr. */
25445 {
25462 {
25465 /* For FP zero change it to a CONST_INT 0 and use the integer SIMD
25466 move immediate path. */
25469 else
25470 {
25479 else
25483 }
25484 }
25489 {
25491 {
25498 }
25501 {
25506 }
25519 else
25523 }
25524 else
25525 {
25526 /* AARCH64_CHECK_ORR, AARCH64_CHECK_AND or AARCH64_CHECK_XOR. */
25534 {
25539 }
25545 else
25549 }
25551 }
25553 /* Returns the string with the ORR instruction for the SIMD immediate
25554 CONST_VECTOR of WIDTH bits. */
25557 {
25559 }
25561 /* Returns the string with the AND/BIC instruction for the SIMD immediate
25562 CONST_VECTOR of WIDTH bits. */
25565 {
25567 }
25569 /* Returns the string with the EOR instruction for the SIMD immediate
25570 CONST_VECTOR of WIDTH bits. */
25573 {
25575 }
25577 /* Returns the string with the MOV instruction for the SIMD immediate
25578 CONST_VECTOR of WIDTH bits. */
25581 {
25583 }
25587 {
25589 /* If a floating point number was passed and we desire to use it in an
25590 integer mode do the conversion to integer. */
25592 {
25597 }
25599 machine_mode vmode;
25600 /* use a 64 bit mode for everything except for DI/DF/DD mode, where we use
25601 a 128 bit vector mode. */
25607 }
25609 /* Return the output string to use for moving immediate CONST_VECTOR
25610 into an SVE register. */
25614 {
25627 {
25630 {
25633 }
25634 else
25635 {
25642 else
25645 }
25647 }
25650 {
25656 }
25659 {
25662 else
25663 {
25673 }
25674 }
25678 else
25682 }
25684 /* Return the asm template for a PTRUES. CONST_UNSPEC is the
25685 aarch64_sve_ptrue_svpattern_immediate that describes the predicate
25686 pattern. */
25690 {
25702 }
25704 /* Split operands into moves from op[1] + op[2] into op[0]. */
25706 void
25708 {
25723 {
25724 /* No-op move. Can't split to nothing; emit something. */
25727 }
25729 /* Special case of reversed high/low parts. */
25732 {
25736 }
25738 {
25739 /* Try to avoid unnecessary moves if part of the result
25740 is in the right place already. */
25745 }
25746 else
25747 {
25752 }
25753 }
25755 /* vec_perm support. */
25757 struct expand_vec_perm_d
25758 {
25760 vec_perm_indices perm;
25761 machine_mode vmode;
25762 machine_mode op_mode;
25768 };
25772 /* Generate a variable permutation. */
25774 static void
25776 {
25787 {
25789 {
25790 /* Expand the argument to a V16QI mode by duplicating it. */
25794 }
25795 else
25796 {
25798 }
25799 }
25800 else
25801 {
25802 rtx pair;
25805 {
25809 }
25810 else
25811 {
25815 }
25816 }
25817 }
25819 /* Expand a vec_perm with the operands given by TARGET, OP0, OP1 and SEL.
25820 NELT is the number of elements in the vector. */
25822 void
25825 {
25828 rtx mask;
25830 /* The TBL instruction does not use a modulo index, so we must take care
25831 of that ourselves. */
25836 /* For big-endian, we also need to reverse the index within the vector
25837 (but not which vector). */
25839 {
25840 /* If one_vector_p, mask is a vector of (nelt - 1)'s already. */
25845 }
25847 }
25849 /* Generate (set TARGET (unspec [OP0 OP1] CODE)). */
25851 static void
25853 {
25857 }
25859 /* Expand an SVE vec_perm with the given operands. */
25861 void
25863 {
25866 /* Enforced by the pattern condition. */
25869 /* Note: vec_perm indices are supposed to wrap when they go beyond the
25870 size of the two value vectors, i.e. the upper bits of the indices
25871 are effectively ignored. SVE TBL instead produces 0 for any
25872 out-of-range indices, so we need to modulo all the vec_perm indices
25873 to ensure they are all in range. */
25876 /* Check if the sel only references the first values vector. */
25879 {
25882 }
25884 /* Check if the two values vectors are the same. */
25886 {
25892 }
25894 /* Run TBL on for each value vector and combine the results. */
25901 {
25906 }
25913 else
25915 }
25917 /* Recognize patterns suitable for the TRN instructions. */
25918 static bool
25920 {
25921 HOST_WIDE_INT odd;
25929 /* Note that these are little-endian tests.
25930 We correct for big-endian later. */
25937 /* Success! */
25943 /* We don't need a big-endian lane correction for SVE; see the comment
25944 at the head of aarch64-sve.md for details. */
25946 {
25949 }
25955 }
25957 /* Try to re-encode the PERM constant so it combines odd and even elements.
25958 This rewrites constants such as {0, 1, 4, 5}/V4SF to {0, 2}/V2DI.
25959 We retry with this new constant with the full suite of patterns. */
25960 static bool
25962 {
25963 expand_vec_perm_d newd;
25965 /* The subregs that we'd create are not supported for big-endian SVE;
25966 see aarch64_modes_compatible_p for details. */
25970 /* Get the new mode. Always twice the size of the inner
25971 and half the elements. */
25972 machine_mode new_mode;
25976 vec_perm_indices newpermindices;
25993 }
25995 /* Recognize patterns suitable for the UZP instructions. */
25996 static bool
25998 {
25999 HOST_WIDE_INT odd;
26006 /* Note that these are little-endian tests.
26007 We correct for big-endian later. */
26013 /* Success! */
26019 /* We don't need a big-endian lane correction for SVE; see the comment
26020 at the head of aarch64-sve.md for details. */
26022 {
26025 }
26031 }
26033 /* Recognize patterns suitable for the ZIP instructions. */
26034 static bool
26036 {
26045 /* Note that these are little-endian tests.
26046 We correct for big-endian later. */
26054 /* Success! */
26060 /* We don't need a big-endian lane correction for SVE; see the comment
26061 at the head of aarch64-sve.md for details. */
26063 {
26066 }
26072 }
26074 /* Recognize patterns for the EXT insn. */
26076 static bool
26078 {
26079 HOST_WIDE_INT location;
26080 rtx offset;
26082 /* The first element always refers to the first vector.
26083 Check if the extracted indices are increasing by one. */
26089 /* Success! */
26093 /* The case where (location == 0) is a no-op for both big- and little-endian,
26094 and is removed by the mid-end at optimization levels -O1 and higher.
26096 We don't need a big-endian lane correction for SVE; see the comment
26097 at the head of aarch64-sve.md for details. */
26099 {
26100 /* After setup, we want the high elements of the first vector (stored
26101 at the LSB end of the register), and the low elements of the second
26102 vector (stored at the MSB end of the register). So swap. */
26104 /* location != 0 (above), so safe to assume (nelt - location) < nelt.
26105 to_constant () is safe since this is restricted to Advanced SIMD
26106 vectors. */
26108 }
26114 UNSPEC_EXT));
26116 }
26118 /* Recognize patterns for the REV{64,32,16} insns, which reverse elements
26119 within each 64-bit, 32-bit or 16-bit granule. */
26121 static bool
26123 {
26124 HOST_WIDE_INT diff;
26126 machine_mode pred_mode;
26136 else
26139 {
26142 }
26144 {
26147 }
26149 {
26152 }
26153 else
26161 /* Success! */
26166 {
26171 }
26175 }
26177 /* Recognize patterns for the REV insn, which reverses elements within
26178 a full vector. */
26180 static bool
26182 {
26191 /* Success! */
26198 }
26200 static bool
26202 {
26204 rtx in0;
26205 HOST_WIDE_INT elt;
26207 rtx lane;
26218 /* Success! */
26222 /* The generic preparation in aarch64_expand_vec_perm_const_1
26223 swaps the operand order and the permute indices if it finds
26224 d->perm[0] to be in the second operand. Thus, we can always
26225 use d->op0 and need not do any extra arithmetic to get the
26226 correct lane number. */
26234 }
26236 /* Recognize things that can be done using the SVE2p1 Hybrid-VLA
26237 permutations, which apply Advanced-SIMD-style permutations to each
26238 individual 128-bit block. */
26240 static bool
26242 {
26245 || !TARGET_NON_STREAMING
26246 || BYTES_BIG_ENDIAN
26251 /* Set SUBELTS to the number of elements in an Advanced SIMD vector
26252 and make sure that adding SUBELTS to each block of SUBELTS indices
26253 gives the next block of SUBELTS indices. That is, it must be possible
26254 to interpret the index vector as SUBELTS interleaved linear series in
26255 which each series has step SUBELTS. */
26262 /* Used once we have verified that we can use UNSPEC to do the operation. */
26264 {
26266 {
26269 }
26271 };
26273 /* Now check whether the first SUBELTS elements match a supported
26274 Advanced-SIMD-style operation. */
26278 {
26286 };
26288 {
26296 };
26298 {
26299 HOST_WIDE_INT start;
26303 {
26309 }
26311 {
26315 }
26317 };
26319 {
26320 HOST_WIDE_INT start;
26327 {
26330 }
26332 };
26335 }
26337 static bool
26339 {
26343 /* Make sure that the indices are constant. */
26352 /* Generic code will try constant permutation twice. Once with the
26353 original mode and again with the elements lowered to QImode.
26354 So wait and don't do the selector expansion ourselves. */
26358 /* to_constant is safe since this routine is specific to Advanced SIMD
26359 vectors. */
26362 /* If one register is the constant vector of 0 then we only need
26363 a one reg TBL and we map any accesses to the vector of 0 to -1. We can't
26364 do this earlier since vec_perm_indices clamps elements to within range so
26365 we can only do it during codegen. */
26372 {
26375 /* If we're selecting from a 0 vector, we can just use an out of range
26376 index instead. */
26379 else
26380 {
26381 /* If we are remapping a zero register as the first parameter we need
26382 to adjust the indices of the non-zero register. */
26386 /* If big-endian and two vectors we end up with a weird mixed-endian
26387 mode on NEON. Reverse the index within each word but not the word
26388 itself. to_constant is safe because we checked is_constant
26389 above. */
26391 }
26392 }
26399 }
26401 /* Try to implement D using an SVE TBL instruction. */
26403 static bool
26405 {
26408 /* Permuting two variable-length vectors could overflow the
26409 index range. */
26420 else
26423 }
26425 /* Try to implement D using SVE dup instruction. */
26427 static bool
26429 {
26450 }
26452 /* Try to implement D using SVE SEL instruction. */
26454 static bool
26456 {
26482 /* Build a predicate that is true when op0 elements should be used. */
26485 {
26489 }
26493 /* TARGET = PRED ? OP0 : OP1. */
26496 }
26498 /* Recognize patterns suitable for the INS instructions. */
26499 static bool
26501 {
26508 /* to_constant is safe since this routine is specific to Advanced SIMD
26509 vectors. */
26516 {
26517 HOST_WIDE_INT elt;
26523 {
26526 }
26528 }
26531 {
26534 {
26540 }
26544 }
26554 {
26557 }
26570 }
26572 static bool
26574 {
26577 /* The pattern matching functions above are written to look for a small
26578 number to begin the sequence (0, 1, N/2). If we begin with an index
26579 from the second operand, we can swap the operands. */
26582 {
26585 }
26592 {
26594 {
26622 }
26623 else
26624 {
26627 }
26628 }
26630 }
26632 /* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */
26634 static bool
26638 {
26641 /* Check whether the mask can be applied to a single vector. */
26646 {
26649 }
26651 {
26654 }
26655 else
26670 else
26682 }
26683 /* Generate a byte permute mask for a register of mode MODE,
26684 which has NUNITS units. */
26686 rtx
26688 {
26689 /* We have to reverse each vector because we dont have
26690 a permuted load that can reverse-load according to ABI rules. */
26691 rtx mask;
26704 }
26706 /* Expand an SVE integer comparison using the SVE equivalent of:
26708 (set TARGET (CODE OP0 OP1)). */
26710 void
26712 {
26719 }
26721 /* Return the UNSPEC_COND_* code for comparison CODE. */
26723 static unsigned int
26725 {
26727 {
26744 }
26745 }
26747 /* Emit:
26749 (set TARGET (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X>))
26751 where <X> is the operation associated with comparison CODE.
26752 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
26754 static void
26757 {
26763 }
26765 /* Emit the SVE equivalent of:
26767 (set TMP1 (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X1>))
26768 (set TMP2 (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X2>))
26769 (set TARGET (ior:PRED_MODE TMP1 TMP2))
26771 where <Xi> is the operation associated with comparison CODEi.
26772 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
26774 static void
26777 {
26784 }
26786 /* Emit the SVE equivalent of:
26788 (set TMP (unspec [PRED KNOWN_PTRUE_P OP0 OP1] UNSPEC_COND_<X>))
26789 (set TARGET (not TMP))
26791 where <X> is the operation associated with comparison CODE.
26792 KNOWN_PTRUE_P is true if PRED is known to be a PTRUE. */
26794 static void
26797 {
26802 }
26804 /* Expand an SVE floating-point comparison using the SVE equivalent of:
26806 (set TARGET (CODE OP0 OP1))
26808 If CAN_INVERT_P is true, the caller can also handle inverted results;
26809 return true if the result is in fact inverted. */
26811 bool
26814 {
26820 {
26822 /* UNORDERED has no immediate form. */
26824 /* fall through */
26831 {
26832 /* There is native support for the comparison. */
26835 }
26838 /* This is a trapping operation (LT or GT). */
26844 {
26845 /* This would trap for signaling NaNs. */
26850 }
26851 /* fall through */
26857 {
26858 /* Work out which elements are ordered. */
26864 /* Test the opposite condition for the ordered elements,
26865 then invert the result. */
26868 else
26871 {
26875 }
26879 }
26883 /* ORDERED has no immediate form. */
26889 }
26891 /* There is native support for the inverse comparison. */
26894 {
26897 }
26900 }
26902 /* Return true if:
26904 (a) MODE1 and MODE2 use the same layout for bytes that are common
26905 to both modes;
26907 (b) subregs involving the two modes behave as the target-independent
26908 subreg rules require; and
26910 (c) there is at least one register that can hold both modes.
26912 Return false otherwise. */
26914 static bool
26916 {
26934 /* Don't allow changes between predicate modes and other modes.
26935 Only predicate registers can hold predicate modes and only
26936 non-predicate registers can hold non-predicate modes, so any
26937 attempt to mix them would require a round trip through memory. */
26941 /* The contents of partial SVE modes are distributed evenly across
26942 the register, whereas GCC expects them to be clustered together.
26943 We therefore need to be careful about mode changes involving them. */
26945 {
26946 /* Reject changes between partial SVE modes that have different
26947 patterns of significant and insignificant bits. */
26952 }
26954 {
26955 /* The first lane of MODE1 is where GCC expects it, but anything
26956 bigger than that is not. */
26959 }
26961 {
26962 /* Similarly in reverse. */
26965 }
26967 /* Don't allow changes between partial Advanced SIMD structure modes
26968 and other modes that are bigger than 8 bytes. E.g. V16QI and V2x8QI
26969 are the same size, but the former occupies one Q register while the
26970 latter occupies two D registers. */
26977 {
26978 /* Don't allow changes between SVE modes and other modes that might
26979 be bigger than 128 bits. In particular, OImode, CImode and XImode
26980 divide into 128-bit quantities while SVE modes divide into
26981 BITS_PER_SVE_VECTOR quantities. */
26986 }
26989 {
26990 /* Don't allow changes between SVE data modes and non-SVE modes.
26991 See the comment at the head of aarch64-sve.md for details. */
26995 /* Don't allow changes in element size: lane 0 of the new vector
26996 would not then be lane 0 of the old vector. See the comment
26997 above aarch64_maybe_expand_sve_subreg_move for a more detailed
26998 description.
27000 In the worst case, this forces a register to be spilled in
27001 one mode and reloaded in the other, which handles the
27002 endianness correctly. */
27005 }
27007 }
27009 /* Implement TARGET_MODES_TIEABLE_P. In principle we should always defer
27010 to aarch64_modes_compatible_p. However due to issues with register
27011 allocation it is preferable to avoid tieing integer scalar and FP
27012 scalar modes. Executing integer operations in general registers is
27013 better than treating them as scalar vector operations. This reduces
27014 latency and avoids redundant int<->FP moves. So tie modes if they
27015 are either the same class, or one of them is a vector mode. */
27017 static bool
27019 {
27021 {
27026 }
27028 }
27030 /* Return a new RTX holding the result of moving POINTER forward by
27031 AMOUNT bytes. */
27033 static rtx
27035 {
27040 }
27042 /* Expand a cpymem/movmem using the MOPS extension. OPERANDS are taken
27043 from the cpymem/movmem pattern. IS_MEMMOVE is true if this is a memmove
27044 rather than memcpy. Return true iff we succeeded. */
27045 bool
27047 {
27051 /* All three registers are changed by the instruction, so each one
27052 must be a fresh pseudo. */
27060 else
27063 }
27065 /* Expand cpymem/movmem, as if from a __builtin_memcpy/memmove.
27066 OPERANDS are taken from the cpymem/movmem pattern. IS_MEMMOVE is true
27067 if this is a memmove rather than memcpy. Return true if we succeed,
27068 otherwise return false, indicating that a libcall should be emitted. */
27069 bool
27071 {
27076 rtx base;
27079 /* Variable-sized or strict-align copies may use the MOPS expansion. */
27085 /* Set inline limits for memmove/memcpy. MOPS has a separate threshold. */
27090 /* Reduce the maximum size with -Os. */
27094 /* Large copies use MOPS when available or a library call. */
27098 /* Default to 32-byte LDP/STP on large copies, however small copies or
27099 no SIMD support fall back to 16-byte chunks.
27100 ??? Although it would be possible to use LDP/STP Qn in streaming mode
27101 (so using TARGET_BASE_SIMD instead of TARGET_SIMD), it isn't clear
27102 whether that would improve performance. */
27115 {
27116 /* Find the largest mode in which to do the copy in without over reading
27117 or writing. */
27118 opt_scalar_int_mode mode_iter;
27127 /* Prefer Q-register accesses. */
27138 /* Emit trailing copies using overlapping unaligned accesses
27139 (when !STRICT_ALIGNMENT) - this is smaller and faster. */
27141 {
27142 next_mode = smallest_mode_for_size
27148 }
27149 }
27151 /* Memcpy interleaves loads with stores, memmove emits all loads first. */
27156 {
27158 /* Emit loads. */
27161 /* Emit stores. */
27164 }
27166 }
27168 /* Expand a setmem using the MOPS instructions. OPERANDS are the same
27169 as for the setmem pattern. Return true iff we succeed. */
27170 static bool
27172 {
27176 /* The first two registers are changed by the instruction, so both
27177 of them must be a fresh pseudo. */
27186 }
27188 /* Expand setmem, as if from a __builtin_memset. Return true if
27189 we succeed, otherwise return false. */
27191 bool
27193 {
27199 rtx base;
27202 /* Variable-sized or strict-align memset may use the MOPS expansion. */
27207 /* Set inline limits for memset. MOPS has a separate threshold. */
27213 /* Large memset uses MOPS when available or a library call. */
27220 /* Prepare the val using a DUP/MOVI v0.16B, val. */
27226 {
27227 /* Find the largest mode in which to do the copy without
27228 over writing. */
27229 opt_scalar_int_mode mode_iter;
27240 /* Prefer Q-register accesses. */
27243 else
27250 /* Emit trailing writes using overlapping unaligned accesses
27251 (when !STRICT_ALIGNMENT) - this is smaller and faster. */
27253 {
27254 next_mode = smallest_mode_for_size
27260 }
27261 }
27264 }
27267 /* Split a DImode store of a CONST_INT SRC to MEM DST as two
27268 SImode stores. Handle the case when the constant has identical
27269 bottom and top halves. This is beneficial when the two stores can be
27270 merged into an STP and we avoid synthesising potentially expensive
27271 immediates twice. Return true if such a split is possible. */
27273 bool
27275 {
27282 unsigned int orig_cost
27284 unsigned int lo_cost
27287 /* We want to transform:
27288 MOV x1, 49370
27289 MOVK x1, 0x140, lsl 16
27290 MOVK x1, 0xc0da, lsl 32
27291 MOVK x1, 0x140, lsl 48
27292 STR x1, [x0]
27293 into:
27294 MOV w1, 49370
27295 MOVK w1, 0x140, lsl 16
27296 STP w1, w1, [x0]
27297 So we want to perform this when we save at least one instruction. */
27308 /* Don't emit an explicit store pair as this may not be always profitable.
27309 Let the sched-fusion logic decide whether to merge them. */
27314 }
27316 /* Generate RTL for a conditional branch with rtx comparison CODE in
27317 mode CC_MODE. The destination of the unlikely conditional branch
27318 is LABEL_REF. */
27320 void
27322 rtx label_ref)
27323 {
27324 rtx x;
27327 const0_rtx);
27331 pc_rtx);
27333 }
27335 /* Generate DImode scratch registers for 128-bit (TImode) addition.
27337 OP1 represents the TImode destination operand 1
27338 OP2 represents the TImode destination operand 2
27339 LOW_DEST represents the low half (DImode) of TImode operand 0
27340 LOW_IN1 represents the low half (DImode) of TImode operand 1
27341 LOW_IN2 represents the low half (DImode) of TImode operand 2
27342 HIGH_DEST represents the high half (DImode) of TImode operand 0
27343 HIGH_IN1 represents the high half (DImode) of TImode operand 1
27344 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
27346 void
27351 {
27358 }
27360 /* Generate DImode scratch registers for 128-bit (TImode) subtraction.
27362 OP1 represents the TImode destination operand 1
27363 OP2 represents the TImode destination operand 2
27364 LOW_DEST represents the low half (DImode) of TImode operand 0
27365 LOW_IN1 represents the low half (DImode) of TImode operand 1
27366 LOW_IN2 represents the low half (DImode) of TImode operand 2
27367 HIGH_DEST represents the high half (DImode) of TImode operand 0
27368 HIGH_IN1 represents the high half (DImode) of TImode operand 1
27369 HIGH_IN2 represents the high half (DImode) of TImode operand 2. */
27372 void
27377 {
27385 }
27387 /* Generate RTL for 128-bit (TImode) subtraction with overflow.
27389 OP0 represents the TImode destination operand 0
27390 LOW_DEST represents the low half (DImode) of TImode operand 0
27391 LOW_IN1 represents the low half (DImode) of TImode operand 1
27392 LOW_IN2 represents the low half (DImode) of TImode operand 2
27393 HIGH_DEST represents the high half (DImode) of TImode operand 0
27394 HIGH_IN1 represents the high half (DImode) of TImode operand 1
27395 HIGH_IN2 represents the high half (DImode) of TImode operand 2
27396 UNSIGNED_P is true if the operation is being performed on unsigned
27397 values. */
27398 void
27402 {
27404 {
27409 else
27411 }
27412 else
27413 {
27417 else
27418 {
27421 }
27426 else
27428 }
27433 }
27435 /* Implement the TARGET_ASAN_SHADOW_OFFSET hook. */
27437 static unsigned HOST_WIDE_INT
27439 {
27442 else
27444 }
27446 static rtx
27449 {
27453 insn_code icode;
27467 {
27495 }
27500 {
27503 }
27512 {
27515 }
27521 }
27523 static rtx
27526 rtx_code bit_code)
27527 {
27531 insn_code icode;
27546 {
27570 }
27577 {
27580 }
27588 {
27589 /* Treat the ccmp patterns as canonical and use them where possible,
27590 but fall back to ccmp_rev patterns if there's no other option. */
27599 else
27600 {
27603 }
27605 }
27616 {
27619 }
27625 }
27627 #undef TARGET_GEN_CCMP_FIRST
27628 #define TARGET_GEN_CCMP_FIRST aarch64_gen_ccmp_first
27630 #undef TARGET_GEN_CCMP_NEXT
27631 #define TARGET_GEN_CCMP_NEXT aarch64_gen_ccmp_next
27633 /* Implement TARGET_SCHED_MACRO_FUSION_P. Return true if target supports
27634 instruction fusion of some sort. */
27636 static bool
27638 {
27640 }
27643 /* Implement TARGET_SCHED_MACRO_FUSION_PAIR_P. Return true if PREV and CURR
27644 should be kept together during scheduling. */
27646 static bool
27648 {
27649 rtx set_dest;
27652 /* prev and curr are simple SET insns i.e. no flag setting or branching. */
27659 {
27660 /* We are trying to match:
27661 prev (mov) == (set (reg r0) (const_int imm16))
27662 curr (movk) == (set (zero_extract (reg r0)
27663 (const_int 16)
27664 (const_int 16))
27665 (const_int imm16_1)) */
27677 {
27679 }
27680 }
27683 {
27685 /* We're trying to match:
27686 prev (adrp) == (set (reg r1)
27687 (high (symbol_ref ("SYM"))))
27688 curr (add) == (set (reg r0)
27689 (lo_sum (reg r1)
27690 (symbol_ref ("SYM"))))
27691 Note that r0 need not necessarily be the same as r1, especially
27692 during pre-regalloc scheduling. */
27696 {
27704 }
27705 }
27708 {
27710 /* We're trying to match:
27711 prev (movk) == (set (zero_extract (reg r0)
27712 (const_int 16)
27713 (const_int 32))
27714 (const_int imm16_1))
27715 curr (movk) == (set (zero_extract (reg r0)
27716 (const_int 16)
27717 (const_int 48))
27718 (const_int imm16_2)) */
27734 }
27736 {
27737 /* We're trying to match:
27738 prev (adrp) == (set (reg r0)
27739 (high (symbol_ref ("SYM"))))
27740 curr (ldr) == (set (reg r1)
27741 (mem (lo_sum (reg r0)
27742 (symbol_ref ("SYM")))))
27743 or
27744 curr (ldr) == (set (reg r1)
27745 (zero_extend (mem
27746 (lo_sum (reg r0)
27747 (symbol_ref ("SYM")))))) */
27750 {
27763 }
27764 }
27766 /* Fuse compare (CMP/CMN/TST/BICS) and conditional branch. */
27774 /* Fuse CMP and CSEL/CSET. */
27779 {
27789 == RTX_COMPARE
27792 }
27794 /* Fuse flag-setting ALU instructions and conditional branch. */
27797 {
27799 rtx cc_reg_1;
27804 && prev
27806 {
27809 /* FIXME: this misses some which is considered simple arthematic
27810 instructions for ThunderX. Simple shifts are missed here. */
27816 }
27817 }
27819 /* Fuse ALU instructions and CBZ/CBNZ. */
27821 && curr_set
27824 {
27825 /* We're trying to match:
27826 prev (alu_insn) == (set (r0) plus ((r0) (r1/imm)))
27827 curr (cbz) == (set (pc) (if_then_else (eq/ne) (r0)
27828 (const_int 0))
27829 (label_ref ("SYM"))
27830 (pc)) */
27837 {
27838 /* Fuse ALU operations followed by conditional branch instruction. */
27840 {
27861 }
27862 }
27863 }
27865 /* Fuse A+B+1 and A-B-1 */
27868 {
27869 /* We're trying to match:
27870 prev == (set (r0) (plus (r0) (r1)))
27871 curr == (set (r0) (plus (r0) (const_int 1)))
27872 or:
27873 prev == (set (r0) (minus (r0) (r1)))
27874 curr == (set (r0) (plus (r0) (const_int -1))) */
27891 }
27894 }
27896 /* Return true iff the instruction fusion described by OP is enabled. */
27898 bool
27900 {
27902 }
27904 /* If MEM is in the form of [base+offset], extract the two parts
27905 of address and set to BASE and OFFSET, otherwise return false
27906 after clearing BASE and OFFSET. */
27908 bool
27910 {
27911 rtx addr;
27918 {
27922 }
27926 {
27930 }
27936 }
27938 /* Types for scheduling fusion. */
27939 enum sched_fusion_type
27940 {
27942 SCHED_FUSION_LD_SIGN_EXTEND,
27943 SCHED_FUSION_LD_ZERO_EXTEND,
27944 SCHED_FUSION_LD,
27945 SCHED_FUSION_ST,
27946 SCHED_FUSION_NUM
27947 };
27949 /* If INSN is a load or store of address in the form of [base+offset],
27950 extract the two parts and set to BASE and OFFSET. Return scheduling
27951 fusion type this INSN is. */
27953 static enum sched_fusion_type
27955 {
27973 {
27978 }
27980 {
27985 }
27990 {
27993 }
27994 else
28001 }
28003 /* Implement the TARGET_SCHED_FUSION_PRIORITY hook.
28005 Currently we only support to fuse ldr or str instructions, so FUSION_PRI
28006 and PRI are only calculated for these instructions. For other instruction,
28007 FUSION_PRI and PRI are simply set to MAX_PRI - 1. In the future, other
28008 type instruction fusion can be added by returning different priorities.
28010 It's important that irrelevant instructions get the largest FUSION_PRI. */
28012 static void
28015 {
28025 {
28029 }
28031 /* Set FUSION_PRI according to fusion type and base register. */
28034 /* Calculate PRI. */
28037 /* INSN with smaller offset goes first. */
28041 else
28046 }
28048 /* Implement the TARGET_SCHED_ADJUST_PRIORITY hook.
28049 Adjust priority of sha1h instructions so they are scheduled before
28050 other SHA1 instructions. */
28052 static int
28054 {
28058 {
28063 }
28066 }
28068 /* If REVERSED is null, return true if memory reference *MEM2 comes
28069 immediately after memory reference *MEM1. Do not change the references
28070 in this case.
28072 Otherwise, check if *MEM1 and *MEM2 are consecutive memory references and,
28073 if they are, try to make them use constant offsets from the same base
28074 register. Return true on success. When returning true, set *REVERSED
28075 to true if *MEM1 comes after *MEM2, false if *MEM1 comes before *MEM2. */
28076 static bool
28078 {
28096 /* Make sure at least one memory is in base+offset form. */
28100 /* If both mems already use the same base register, just check the
28101 offsets. */
28103 {
28111 {
28114 }
28117 }
28119 /* Otherwise, check whether the MEM_EXPRs and MEM_OFFSETs together
28120 guarantee that the values are consecutive. */
28125 {
28126 poly_int64 expr_offset1;
28127 poly_int64 expr_offset2;
28133 || !expr_base2
28142 ;
28145 else
28149 {
28151 {
28155 }
28156 else
28157 {
28161 }
28162 }
28164 }
28167 }
28169 /* Test if MODE is suitable for a single transfer register in an ldp or stp
28170 instruction. */
28172 bool
28174 {
28181 }
28183 /* Return true if MEM1 and MEM2 can be combined into a single access
28184 of mode MODE, with the combined access having the same address as MEM1. */
28186 bool
28188 {
28192 }
28194 /* Return true if MEM agrees with the ldp-stp policy model.
28195 Otherwise, false. */
28197 bool
28199 {
28204 /* If we have AARCH64_LDP_STP_POLICY_NEVER, reject the load pair. */
28208 /* If we have AARCH64_LDP_STP_POLICY_ALIGNED,
28209 do not emit the load pair unless the alignment is checked to be
28210 at least double the alignment of the type. */
28217 }
28219 /* Given OPERANDS of consecutive load/store, check if we can merge
28220 them into ldp/stp. LOAD is true if they are load instructions. */
28222 bool
28224 {
28229 {
28239 }
28240 else
28241 {
28246 }
28248 /* The mems cannot be volatile. */
28252 /* Check if the addresses are in the form of [base+offset]. */
28257 /* The operands must be of the same size. */
28261 /* The lower memory access must be a mem-pair operand. */
28267 /* Check if lower_mem is ok with the ldp-stp policy model. */
28269 lower_mem_mode))
28274 else
28279 else
28282 /* Check if the registers are of same class. */
28287 }
28289 /* Given OPERANDS of consecutive load/store that can be merged,
28290 swap them if they are not in ascending order. */
28291 void
28293 {
28301 {
28302 /* Irrespective of whether this is a load or a store,
28303 we do the same swap. */
28306 }
28307 }
28309 /* Helper function used for generation of load/store pair instructions, called
28310 from peepholes in aarch64-ldpstp.md. OPERANDS is an array of
28311 operands as matched by the peepholes in that file. LOAD_P is true if we're
28312 generating a load pair, otherwise we're generating a store pair. CODE is
28313 either {ZERO,SIGN}_EXTEND for extending loads or UNKNOWN if we're generating a
28314 standard load/store pair. */
28316 void
28318 {
28324 else
28325 {
28329 }
28330 }
28332 /* Taking X and Y to be HOST_WIDE_INT pointers, return the result of a
28333 comparison between the two. */
28334 int
28336 {
28339 }
28341 /* Taking X and Y to be pairs of RTX, one pointing to a MEM rtx and the
28342 other pointing to a REG rtx containing an offset, compare the offsets
28343 of the two pairs.
28345 Return:
28347 1 iff offset (X) > offset (Y)
28348 0 iff offset (X) == offset (Y)
28349 -1 iff offset (X) < offset (Y) */
28350 int
28352 {
28359 else
28364 else
28367 /* Extract the offsets. */
28374 }
28376 /* Given OPERANDS of consecutive load/store, check if we can merge
28377 them into ldp/stp by adjusting the offset. LOAD is true if they
28378 are load instructions. MODE is the mode of memory operands.
28380 Given below consecutive stores:
28382 str w1, [xb, 0x100]
28383 str w1, [xb, 0x104]
28384 str w1, [xb, 0x108]
28385 str w1, [xb, 0x10c]
28387 Though the offsets are out of the range supported by stp, we can
28388 still pair them after adjusting the offset, like:
28390 add scratch, xb, 0x100
28391 stp w1, w1, [scratch]
28392 stp w1, w1, [scratch, 0x8]
28394 The peephole patterns detecting this opportunity should guarantee
28395 the scratch register is avaliable. */
28397 bool
28399 machine_mode mode)
28400 {
28407 {
28409 {
28414 }
28416 /* Do not attempt to merge the loads if the loads clobber each other. */
28421 }
28422 else
28424 {
28427 }
28429 /* Skip if memory operand is by itself valid for ldp/stp. */
28434 {
28435 /* The mems cannot be volatile. */
28439 /* Check if the addresses are in the form of [base+offset]. */
28443 }
28445 /* Check if the registers are of same class. */
28451 {
28454 }
28455 else
28456 {
28459 }
28461 /* Only the last register in the order in which they occur
28462 may be clobbered by the load. */
28468 /* Check if the bases are same. */
28478 /* Check if the offsets can be put in the right order to do a ldp/stp. */
28480 aarch64_host_wide_int_compare);
28486 /* Check that offsets are within range of each other. The ldp/stp
28487 instructions have 7 bit immediate offsets, so use 0x80. */
28491 /* The offsets must be aligned with respect to each other. */
28495 /* Check if mem[0] is ok with the ldp-stp policy model. */
28500 }
28502 /* Given OPERANDS of consecutive load/store, this function pairs them
28503 into LDP/STP after adjusting the offset. It depends on the fact
28504 that the operands can be sorted so the offsets are correct for STP.
28505 MODE is the mode of memory operands. CODE is the rtl operator
28506 which should be applied to all memory operands, it's SIGN_EXTEND,
28507 ZERO_EXTEND or UNKNOWN. */
28509 bool
28512 {
28519 /* We make changes on a copy as we may still bail out. */
28523 /* Sort the operands. Note for cases as below:
28524 [base + 0x310] = A
28525 [base + 0x320] = B
28526 [base + 0x330] = C
28527 [base + 0x320] = D
28528 We need stable sorting otherwise wrong data may be store to offset 0x320.
28529 Also note the dead store in above case should be optimized away, but no
28530 guarantees here. */
28532 aarch64_ldrstr_offset_compare);
28534 /* Copy the memory operands so that if we have to bail for some
28535 reason the original addresses are unchanged. */
28537 {
28540 }
28541 else
28542 {
28546 }
28553 /* Adjust offset so it can fit in LDP/STP instruction. */
28561 /* The base offset is optimally half way between the two STP/LDP offsets. */
28564 else
28565 /* However, due to issues with negative LDP/STP offset generation for
28566 larger modes, for DF, DD, DI and vector modes. we must not use negative
28567 addresses smaller than 9 signed unadjusted bits can store. This
28568 provides the most range in this case. */
28571 /* Adjust the base so that it is aligned with the addresses but still
28572 optimal. */
28574 /* Fix the offset, bearing in mind we want to make it bigger not
28575 smaller. */
28578 /* The negative range of LDP/STP is one larger than the positive range. */
28581 /* Check if base offset is too big or too small. We can attempt to resolve
28582 this issue by setting it to the maximum value and seeing if the offsets
28583 still fit. */
28585 {
28587 /* We must still make sure that the base offset is aligned with respect
28588 to the address. But it may not be made any bigger. */
28590 }
28592 /* Likewise for the case where the base is too small. */
28594 {
28597 }
28599 /* Offset of the first STP/LDP. */
28602 /* Offset of the second STP/LDP. */
28605 /* The offsets must be within the range of the LDP/STP instructions. */
28620 {
28627 }
28628 else
28629 {
28636 }
28638 /* Emit adjusting instruction. */
28640 /* Emit ldp/stp instructions. */
28642 {
28647 }
28648 else
28649 {
28654 }
28656 }
28658 /* Implement TARGET_VECTORIZE_CONDITIONAL_OPERATION_IS_EXPENSIVE. Assume that
28659 predicated operations when available are beneficial. */
28661 static bool
28663 {
28665 }
28667 /* Implement TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE. Assume for now that
28668 it isn't worth branching around empty masked ops (including masked
28669 stores). */
28671 static bool
28673 {
28675 }
28677 /* Return 1 if pseudo register should be created and used to hold
28678 GOT address for PIC code. */
28680 bool
28682 {
28684 }
28686 /* Implement TARGET_UNSPEC_MAY_TRAP_P. */
28688 static int
28690 {
28692 {
28699 }
28702 }
28705 /* If X is a positive CONST_DOUBLE with a value that is a power of 2
28706 return the log2 of that value. Otherwise return -1. */
28708 int
28710 {
28725 }
28727 /* If X is a positive CONST_DOUBLE with a value that is the reciprocal of a
28728 power of 2 (i.e 1/2^n) return the number of float bits. e.g. for x==(1/2^n)
28729 return n. Otherwise return -1. */
28731 int
28733 {
28734 REAL_VALUE_TYPE r0;
28742 {
28746 }
28748 }
28750 /* If X is a vector of equal CONST_DOUBLE values and that value is
28751 Y, return the aarch64_fpconst_pow_of_2 of Y. Otherwise return -1. */
28753 int
28755 {
28773 }
28775 /* Implement TARGET_PROMOTED_TYPE to promote 16-bit floating point types
28776 to float.
28778 __fp16 always promotes through this hook.
28779 _Float16 may promote if TARGET_FLT_EVAL_METHOD is 16, but we do that
28780 through the generic excess precision logic rather than here. */
28782 static tree
28784 {
28790 }
28792 /* Implement the TARGET_OPTAB_SUPPORTED_P hook. */
28794 static bool
28796 optimization_type opt_type)
28797 {
28799 {
28805 }
28806 }
28808 /* Implement the TARGET_DWARF_POLY_INDETERMINATE_VALUE hook. */
28810 static unsigned int
28813 {
28814 /* Polynomial invariant 1 == (VG / 2) - 1. */
28819 }
28821 /* Implement TARGET_LIBGCC_FLOATING_POINT_MODE_SUPPORTED_P - return TRUE
28822 if MODE is [BH]Fmode, and punt to the generic implementation otherwise. */
28824 static bool
28826 {
28830 }
28832 /* Implement TARGET_SCALAR_MODE_SUPPORTED_P - return TRUE
28833 if MODE is [BH]Fmode, and punt to the generic implementation otherwise. */
28835 static bool
28837 {
28844 }
28846 /* Set the value of FLT_EVAL_METHOD.
28847 ISO/IEC TS 18661-3 defines two values that we'd like to make use of:
28849 0: evaluate all operations and constants, whose semantic type has at
28850 most the range and precision of type float, to the range and
28851 precision of float; evaluate all other operations and constants to
28852 the range and precision of the semantic type;
28854 N, where _FloatN is a supported interchange floating type
28855 evaluate all operations and constants, whose semantic type has at
28856 most the range and precision of _FloatN type, to the range and
28857 precision of the _FloatN type; evaluate all other operations and
28858 constants to the range and precision of the semantic type;
28860 If we have the ARMv8.2-A extensions then we support _Float16 in native
28861 precision, so we should set this to 16. Otherwise, we support the type,
28862 but want to evaluate expressions in float precision, so set this to
28863 0. */
28865 static enum flt_eval_method
28867 {
28869 {
28872 /* We can calculate either in 16-bit range and precision or
28873 32-bit range and precision. Make that decision based on whether
28874 we have native support for the ARMv8.2-A 16-bit floating-point
28875 instructions or not. */
28877 ? FLT_EVAL_METHOD_PROMOTE_TO_FLOAT16
28884 }
28886 }
28888 /* Implement TARGET_C_BITINT_TYPE_INFO.
28889 Return true if _BitInt(N) is supported and fill its details into *INFO. */
28890 bool
28892 {
28906 else
28907 /* The AAPCS for AArch64 defines _BitInt(N > 128) as an array with
28908 type {signed,unsigned} __int128[M] where M*128 >= N. However, to be
28909 able to use libgcc's implementation to support large _BitInt's we need
28910 to use a LIMB_MODE that is no larger than 'long long'. This is why we
28911 use DImode for our internal LIMB_MODE and we define the ABI_LIMB_MODE to
28912 be TImode to ensure we are ABI compliant. */
28917 else
28922 }
28924 /* Implement TARGET_C_MODE_FOR_FLOATING_TYPE. Return TFmode for
28925 TI_LONG_DOUBLE_TYPE which is for long double type, go with the default
28926 one for the others. */
28928 static machine_mode
28930 {
28934 }
28936 /* Implement TARGET_SCHED_CAN_SPECULATE_INSN. Return true if INSN can be
28937 scheduled for speculative execution. Reject the long-running division
28938 and square-root instructions. */
28940 static bool
28942 {
28944 {
28962 }
28963 }
28965 /* Implement TARGET_COMPUTE_PRESSURE_CLASSES. */
28967 static int
28969 {
28973 /* PR_REGS isn't a useful pressure class because many predicate pseudo
28974 registers need to go in PR_LO_REGS at some point during their
28975 lifetime. Splitting it into two halves has the effect of making
28976 all predicates count against PR_LO_REGS, so that we try whenever
28977 possible to restrict the number of live predicates to 8. This
28978 greatly reduces the amount of spilling in certain loops. */
28982 }
28984 /* Implement TARGET_CAN_CHANGE_MODE_CLASS. */
28986 static bool
28989 {
28991 }
28993 /* Implement TARGET_EARLY_REMAT_MODES. */
28995 static void
28997 {
28998 /* SVE values are not normally live across a call, so it should be
28999 worth doing early rematerialization even in VL-specific mode. */
29003 }
29005 /* Override the default target speculation_safe_value. */
29006 static rtx
29009 {
29010 /* Maybe we should warn if falling back to hard barriers. They are
29011 likely to be noticably more expensive than the alternative below. */
29023 }
29025 /* Implement TARGET_ESTIMATED_POLY_VALUE.
29026 Look into the tuning structure for an estimate.
29027 KIND specifies the type of requested estimate: min, max or likely.
29028 For cores with a known SVE width all three estimates are the same.
29029 For generic SVE tuning we want to distinguish the maximum estimate from
29030 the minimum and likely ones.
29031 The likely estimate is the same as the minimum in that case to give a
29032 conservative behavior of auto-vectorizing with SVE when it is a win
29033 even for 128-bit SVE.
29034 When SVE width information is available VAL.coeffs[1] is multiplied by
29035 the number of VQ chunks over the initial Advanced SIMD 128 bits. */
29037 static HOST_WIDE_INT
29039 poly_value_estimate_kind kind
29041 {
29044 /* If there is no core-specific information then the minimum and likely
29045 values are based on 128-bit vectors and the maximum is based on
29046 the architectural maximum of 2048 bits. */
29049 {
29055 }
29057 /* Allow sve_width to be a bitmask of different VL, treating the lowest
29058 as likely. This could be made more general if future -mtune options
29059 need it to be. */
29062 else
29065 /* If the core provides width information, use that. */
29068 }
29071 /* Return true for types that could be supported as SIMD return or
29072 argument types. */
29074 static bool
29076 {
29078 {
29081 }
29083 }
29085 /* Determine the lane size for the clone argument/return type. This follows
29086 the LS(P) rule in the VFABIA64. */
29088 static unsigned
29090 {
29093 /* For non map-to-vector types that are pointers we use the element type it
29094 points to. */
29097 {
29105 }
29107 /* For types (or pointers of non map-to-vector types point to) that are
29108 integers or floating point, we use their size if they are 1, 2, 4 or 8.
29109 */
29113 {
29121 }
29122 /* For any other we use the size of uintptr_t. For map-to-vector types that
29123 are pointers, using the size of uintptr_t is the same as using the size of
29124 their type, seeing all pointers are the same size as uintptr_t. */
29126 }
29129 /* Implement TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN. */
29131 static int
29134 tree base_type ATTRIBUTE_UNUSED,
29136 {
29144 /* For now, SVE simdclones won't produce illegal simdlen, So only check
29145 const simdlens here. */
29151 {
29156 }
29159 /* According to AArch64's Vector ABI the type that determines the simdlen is
29160 the narrowest of types, so we ignore base_type for AArch64. */
29163 {
29165 ;
29168 "GCC does not currently support return type %qT "
29170 else
29173 ret_type);
29175 }
29179 /* We are looking for the NDS type here according to the VFABIA64. */
29181 {
29185 }
29186 else
29187 {
29190 }
29197 {
29201 {
29203 ;
29206 "GCC does not currently support argument type %qT "
29208 else
29211 type);
29213 }
29221 }
29223 /* If we could not determine the WDS type from available parameters/return,
29224 then fallback to using uintptr_t. */
29229 poly_uint64 simdlen;
29231 {
29232 poly_uint64 len;
29237 /* Keep track of the possible simdlens the clones of this function can have,
29238 and check them later to see if we support them. */
29240 {
29245 /* Only create an SVE simd clone if we aren't dealing with an unprototyped
29246 function.
29247 We have also disabled support for creating SVE simdclones for functions
29248 with function bodies and any simdclones when -msve-vector-bits is used.
29249 TODO: add support for these. */
29254 }
29255 else
29261 /* We currently do not support generating simdclones where vector arguments
29262 do not fit into a single vector register, i.e. vector types that are more
29263 than 128-bits large. This is because of how we currently represent such
29264 types in ACLE, where we use a struct to allow us to pass them as arguments
29265 and return.
29266 Hence why we have to check whether the simdlens available for this
29267 simdclone would cause a vector type to be larger than 128-bits, and reject
29268 such a clone. */
29271 {
29276 {
29277 /* Don't issue a warning for every simdclone when there is no
29278 specific simdlen clause. */
29281 "GCC does not currently support simdlen %wd for "
29286 }
29289 else
29290 j++;
29291 }
29295 {
29297 {
29298 /* Warn the user if we can't generate any simdclone. */
29301 "GCC does not currently support a simdclone with simdlens"
29305 }
29307 }
29312 /* SVE simdclones always have a Mask, so set inbranch to 1. */
29316 }
29318 /* Helper function to adjust an SVE vector type of an SVE simd clone. Returns
29319 an SVE vector type based on the element type of the vector TYPE, with SIMDLEN
29320 number of elements. If IS_MASK, returns an SVE mask type appropriate for use
29321 with the SVE type it would otherwise return. */
29323 static tree
29325 {
29328 machine_mode vector_mode;
29334 {
29337 }
29338 else
29341 /* We create new types here with the SVE type attribute instead of using ACLE
29342 types as we need to support unpacked vectors which aren't available as
29343 ACLE SVE types. */
29345 /* ??? This creates anonymous "SVE type" attributes for all types,
29346 even those that correspond to <arm_sve.h> types. This affects type
29347 compatibility in C/C++, but not in gimple. (Gimple type equivalence
29348 is instead decided by TARGET_COMPATIBLE_VECTOR_TYPES_P.)
29350 Thus a C/C++ definition of the implementation function will have a
29351 different function type from the declaration that this code creates.
29352 However, it doesn't seem worth trying to fix that until we have a
29353 way of handling implementations that operate on unpacked types. */
29357 }
29359 /* Implement TARGET_SIMD_CLONE_ADJUST. */
29360 static void
29362 {
29366 {
29367 /* This is additive and has no effect if SVE, or a superset thereof, is
29368 already enabled. */
29373 }
29374 else
29375 {
29376 /* Add aarch64_vector_pcs target attribute to SIMD clones so they
29377 use the correct ABI. */
29380 }
29385 {
29387 tree type;
29389 {
29405 }
29406 }
29408 {
29415 }
29416 }
29418 /* Implement TARGET_SIMD_CLONE_USABLE. */
29420 static int
29422 {
29424 {
29436 }
29437 }
29439 /* Implement TARGET_COMP_TYPE_ATTRIBUTES */
29441 static int
29443 {
29451 };
29474 }
29476 /* Implement TARGET_MERGE_DECL_ATTRIBUTES. */
29478 static tree
29480 {
29488 {
29492 newdecl);
29493 }
29494 else
29495 {
29497 {
29501 }
29504 }
29507 }
29509 /* Implement TARGET_GET_MULTILIB_ABI_NAME */
29513 {
29517 }
29519 /* Implement TARGET_STACK_PROTECT_GUARD. In case of a
29520 global variable based guard use the default else
29521 return a null tree. */
29522 static tree
29524 {
29529 }
29531 /* Implement TARGET_INVALID_UNARY_OP. */
29535 {
29536 /* Reject all single-operand operations on __mfp8 except for &. */
29540 /* Operation allowed. */
29542 }
29544 /* Implement TARGET_INVALID_BINARY_OP. */
29548 const_tree type2)
29549 {
29558 /* Reject all 2-operand operations on __mfp8. */
29563 /* Operation allowed. */
29565 }
29567 /* Implement TARGET_MEMTAG_CAN_TAG_ADDRESSES. Here we tell the rest of the
29568 compiler that we automatically ignore the top byte of our pointers, which
29569 allows using -fsanitize=hwaddress. */
29570 bool
29572 {
29574 }
29576 /* Implement TARGET_ASM_FILE_END for AArch64. This adds the AArch64 GNU NOTE
29577 section at the end if needed. */
29578 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000
29579 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0)
29580 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1)
29581 #define GNU_PROPERTY_AARCH64_FEATURE_1_GCS (1U << 2)
29582 void
29584 {
29598 {
29599 /* Generate .note.gnu.property section. */
29603 /* PT_NOTE header: namesz, descsz, type.
29604 namesz = 4 ("GNU\0")
29605 descsz = 16 (Size of the program property array)
29606 [(12 + padding) * Number of array elements]
29607 type = 5 (NT_GNU_PROPERTY_TYPE_0). */
29613 /* PT_NOTE name. */
29616 /* PT_NOTE contents for NT_GNU_PROPERTY_TYPE_0:
29617 type = GNU_PROPERTY_AARCH64_FEATURE_1_AND
29618 datasz = 4
29619 data = feature_1_and. */
29624 /* Pad the size of the note to the required alignment. */
29626 }
29627 }
29628 #undef GNU_PROPERTY_AARCH64_FEATURE_1_GCS
29629 #undef GNU_PROPERTY_AARCH64_FEATURE_1_PAC
29630 #undef GNU_PROPERTY_AARCH64_FEATURE_1_BTI
29631 #undef GNU_PROPERTY_AARCH64_FEATURE_1_AND
29633 /* Helper function for straight line speculation.
29634 Return what barrier should be emitted for straight line speculation
29635 mitigation.
29636 When not mitigating against straight line speculation this function returns
29637 an empty string.
29638 When mitigating against straight line speculation, use:
29639 * SB when the v8.5-A SB extension is enabled.
29640 * DSB+ISB otherwise. */
29643 {
29644 return mitigation_required
29647 }
29682 };
29684 /* Function to create a BLR thunk. This thunk is used to mitigate straight
29685 line speculation. Instead of a simple BLR that can be speculated past,
29686 we emit a BL to this thunk, and this thunk contains a BR to the relevant
29687 register. These thunks have the relevant speculation barries put after
29688 their indirect branch so that speculation is blocked.
29690 We use such a thunk so the speculation barriers are kept off the
29691 architecturally executed path in order to reduce the performance overhead.
29693 When optimizing for size we use stubs shared by the linked object.
29694 When optimizing for performance we emit stubs for each function in the hope
29695 that the branch predictor can better train on jumps specific for a given
29696 function. */
29697 rtx
29699 {
29702 {
29703 /* For the thunks shared between different functions in this compilation
29704 unit we use a named symbol -- this is just for users to more easily
29705 understand the generated assembly. */
29709 {
29710 /* Build a decl representing this function stub and record it for
29711 later. We build a decl here so we can use the GCC machinery for
29712 handling sections automatically (through `get_named_section` and
29713 `make_decl_one_only`). That saves us a lot of trouble handling
29714 the specifics of different output file formats. */
29718 NULL_TREE));
29728 }
29731 }
29737 }
29739 /* Helper function for aarch64_sls_emit_blr_function_thunks and
29740 aarch64_sls_emit_shared_blr_thunks below. */
29741 static void
29743 {
29744 /* Save in x16 and branch to that function so this transformation does
29745 not prevent jumping to `BTI c` instructions. */
29748 }
29750 /* Emit all BLR stubs for this particular function.
29751 Here we emit all the BLR stubs needed for the current function. Since we
29752 emit these stubs in a consecutive block we know there will be no speculation
29753 gadgets between each stub, and hence we only emit a speculation barrier at
29754 the end of the stub sequences.
29756 This is called in the TARGET_ASM_FUNCTION_EPILOGUE hook. */
29757 void
29759 {
29764 /* We must save and restore the current function section since this assembly
29765 is emitted at the end of the function. This means it can be emitted *just
29766 after* the cold section of a function. That cold part would be emitted in
29767 a different section. That switch would trigger a `.cfi_endproc` directive
29768 to be emitted in the original section and a `.cfi_startproc` directive to
29769 be emitted in the new section. Switching to the original section without
29770 restoring would mean that the `.cfi_endproc` emitted as a function ends
29771 would happen in a different section -- leaving an unmatched
29772 `.cfi_startproc` in the cold text section and an unmatched `.cfi_endproc`
29773 in the standard text section. */
29777 {
29786 }
29788 /* Can use the SB if needs be here, since this stub will only be used
29789 by the current function, and hence for the current target. */
29792 }
29794 /* Emit shared BLR stubs for the current compilation unit.
29795 Over the course of compiling this unit we may have converted some BLR
29796 instructions to a BL to a shared stub function. This is where we emit those
29797 stub functions.
29798 This function is for the stubs shared between different functions in this
29799 compilation unit. We share when optimizing for size instead of speed.
29801 This function is called through the TARGET_ASM_FILE_END hook. */
29802 void
29804 {
29809 {
29818 /* Only emits if the compiler is configured for an assembler that can
29819 handle visibility directives. */
29824 /* Use the most conservative target to ensure it can always be used by any
29825 function in the translation unit. */
29828 }
29829 }
29831 /* Implement TARGET_ASM_FILE_END. */
29832 void
29834 {
29836 /* Since this function will be called for the ASM_FILE_END hook, we ensure
29837 that what would be called otherwise (e.g. `file_end_indicate_exec_stack`
29838 for FreeBSD) still gets called. */
29839 #ifdef TARGET_ASM_FILE_END
29841 #endif
29842 }
29846 {
29849 {
29852 }
29853 else
29856 }
29858 /* Emit the assembly instruction to load the thread pointer into DEST.
29859 Select between different tpidr_elN registers depending on -mtp= setting. */
29863 {
29871 }
29873 /* Set up the value of REG_ALLOC_ORDER from scratch.
29875 It was previously good practice to put call-clobbered registers ahead
29876 of call-preserved registers, but that isn't necessary these days.
29877 IRA's model of register save/restore costs is much more sophisticated
29878 than the model that a simple ordering could provide. We leave
29879 HONOR_REG_ALLOC_ORDER undefined so that we can get the full benefit
29880 of IRA's model.
29882 However, it is still useful to list registers that are members of
29883 multiple classes after registers that are members of fewer classes.
29884 For example, we have:
29886 - FP_LO8_REGS: v0-v7
29887 - FP_LO_REGS: v0-v15
29888 - FP_REGS: v0-v31
29890 If, as a tie-breaker, we allocate FP_REGS in the order v0-v31,
29891 we run the risk of starving other (lower-priority) pseudos that
29892 require FP_LO8_REGS or FP_LO_REGS. Allocating FP_LO_REGS in the
29893 order v0-v15 could similarly starve pseudos that require FP_LO8_REGS.
29894 Allocating downwards rather than upwards avoids this problem, at least
29895 in code that has reasonable register pressure.
29897 The situation for predicate registers is similar. */
29899 void
29901 {
29907 else
29909 }
29911 /* Return true if the PARALLEL PAR can be used in a VEC_SELECT expression
29912 of vector mode MODE to select half the elements of that vector.
29913 Allow any combination of indices except duplicates (or out of range of
29914 the mode units). */
29916 bool
29918 {
29923 /* Put all the elements of PAR into a hash_set and use its
29924 uniqueness guarantees to check that we don't try to insert the same
29925 element twice. */
29928 {
29934 }
29936 }
29938 /* Return true if PAR1 and PAR2, two PARALLEL rtxes of CONST_INT values,
29939 contain any common elements. */
29941 bool
29943 {
29953 }
29955 /* Implement OPTIMIZE_MODE_SWITCHING. */
29957 bool
29959 {
29967 {
29970 {
29973 }
29974 }
29977 {
29981 }
29983 }
29985 /* Implement TARGET_MODE_EMIT for ZA_SAVE_BUFFER. */
29987 static void
29989 aarch64_tristate_mode prev_mode)
29990 {
29992 {
29995 }
29996 else
29998 }
30000 /* Implement TARGET_MODE_EMIT for LOCAL_SME_STATE. */
30002 static void
30004 aarch64_local_sme_state prev_mode)
30005 {
30006 /* Back-propagation should ensure that we're always starting from
30007 a known mode. */
30011 {
30012 /* Commit any uncommitted lazy save. This leaves ZA either active
30013 and zero (lazy save case) or off (normal case).
30015 The sequence is:
30017 mrs <temp>, tpidr2_el0
30018 cbz <temp>, no_save
30019 bl __arm_tpidr2_save
30020 msr tpidr2_el0, xzr
30021 zero { za } // Only if ZA is live
30022 zero { zt0 } // Only if ZT0 is live
30023 no_save: */
30034 {
30039 }
30041 }
30045 {
30047 {
30048 /* Make ZA active after being inactive.
30050 First handle the case in which the lazy save we set up was
30051 committed by a callee. If the function's source-level ZA state
30052 is live then we must conditionally restore it from the lazy
30053 save buffer. Otherwise we can just force PSTATE.ZA to 1. */
30056 else
30059 /* Now handle the case in which the lazy save was not committed.
30060 In that case, ZA still contains the current function's ZA state,
30061 and we just need to cancel the lazy save. */
30064 /* Restore the ZT0 state, if we have some. */
30069 }
30072 {
30073 /* Retrieve the current function's ZA state from the lazy save
30074 buffer. */
30077 /* Restore the ZT0 state, if we have some. */
30081 }
30085 {
30086 /* INACTIVE_CALLER means that we are enabling ZA for the first
30087 time in this function. The code above means that ZA is either
30088 active and zero (if we committed a lazy save) or off. Handle
30089 the latter case by forcing ZA on.
30091 OFF means that PSTATE.ZA is guaranteed to be 0. We just need
30092 to force it to 1.
30094 Both cases leave ZA zeroed. */
30097 /* Restore the ZT0 state, if we have some. */
30102 }
30106 /* A simple change in liveness, such as in a CFG structure where
30107 ZA is only conditionally defined. No code is needed. */
30111 }
30114 {
30118 {
30119 /* Save the ZT0 state, if we have some. */
30123 /* A transition from ACTIVE_LIVE to INACTIVE_LOCAL is the usual
30124 case of setting up a lazy save buffer before a call.
30125 A transition from INACTIVE_CALLER is similar, except that
30126 the contents of ZA are known to be zero.
30128 A transition from ACTIVE_DEAD means that ZA is live at the
30129 point of the transition, but is dead on at least one incoming
30130 edge. (That is, ZA is only conditionally initialized.)
30131 For efficiency, we want to set up a lazy save even for
30132 dead contents, since forcing ZA off would make later code
30133 restore ZA from the lazy save buffer. */
30136 }
30140 /* We're simply discarding the information about which inactive
30141 state applies. */
30145 }
30149 {
30150 /* Save the ZT0 state, if we have some. */
30157 /* The transition to INACTIVE_CALLER is used before returning from
30158 new("za") functions. Any state in ZA belongs to the current
30159 function rather than a caller, but that state is no longer
30160 needed. Clear any pending lazy save and turn ZA off.
30162 The transition to OFF is used before calling a private-ZA function.
30163 We committed any incoming lazy save above, so at this point any
30164 contents in ZA belong to the current function. */
30173 }
30176 {
30177 /* This is a transition to an exception handler. */
30181 }
30184 }
30186 /* Implement TARGET_MODE_EMIT. */
30188 static void
30190 {
30196 {
30206 }
30210 /* Get the set of clobbered registers that are currently live. */
30211 HARD_REG_SET clobbers = {};
30213 {
30216 vec_rtx_properties properties;
30221 }
30224 /* Emit instructions to save clobbered registers to pseudos. Queue
30225 instructions to restore the registers afterwards.
30227 This should only needed in rare situations. */
30231 {
30236 }
30238 {
30242 }
30244 /* Emit the transition instructions themselves. */
30247 /* Restore the clobbered registers. */
30250 }
30252 /* Return true if INSN references the SME state represented by hard register
30253 REGNO. */
30255 static bool
30257 {
30258 df_ref ref;
30267 }
30269 /* Implement TARGET_MODE_NEEDED for LOCAL_SME_STATE. */
30271 static aarch64_local_sme_state
30273 {
30276 {
30279 {
30282 }
30283 /* Aim for graceful error recovery by picking the value that is
30284 least likely to generate an ICE. */
30286 }
30288 /* A non-local goto is equivalent to a return. We disallow non-local
30289 receivers in functions with SME state, so we know that the target
30290 expects ZA to be dormant or off. */
30295 /* start_private_za_call and end_private_za_call bracket a sequence
30296 that calls a private-ZA function. Force ZA to be turned off if the
30297 function doesn't have any live ZA state, otherwise require ZA to be
30298 inactive. */
30306 /* Force ZA to contain the current function's ZA state if INSN wants
30307 to access it. Do the same for accesses to ZT0, since ZA and ZT0
30308 are both controlled by PSTATE.ZA. */
30316 }
30318 /* Implement TARGET_MODE_NEEDED for ZA_SAVE_BUFFER. */
30320 static aarch64_tristate_mode
30322 {
30323 /* We need to set up a lazy save buffer no later than the first
30324 transition to INACTIVE_LOCAL (which involves setting up a lazy save). */
30329 /* Also make sure that the lazy save buffer is set up before the first
30330 insn that throws internally. The exception handler will sometimes
30331 load from it. */
30336 }
30338 /* Implement TARGET_MODE_NEEDED. */
30340 static int
30342 {
30344 {
30350 }
30352 }
30354 /* Implement TARGET_MODE_AFTER for LOCAL_SME_STATE. */
30356 static aarch64_local_sme_state
30358 HARD_REG_SET live)
30359 {
30360 /* Note places where ZA dies, so that we can try to avoid saving and
30361 restoring state that isn't needed. */
30366 /* Note where ZA is born, e.g. when moving past an __arm_out("za")
30367 function. */
30373 }
30375 /* Implement TARGET_MODE_AFTER. */
30377 static int
30379 {
30381 {
30388 }
30390 }
30392 /* Implement TARGET_MODE_CONFLUENCE for LOCAL_SME_STATE. */
30394 static aarch64_local_sme_state
30396 aarch64_local_sme_state mode2)
30397 {
30398 /* Perform a symmetrical check for two values. */
30400 aarch64_local_sme_state val2)
30401 {
30404 };
30406 /* INACTIVE_CALLER means ZA is off or it has dormant contents belonging
30407 to a caller. OFF is one of the options. */
30412 /* Similarly for dormant contents belonging to the current function. */
30417 /* Treat a conditionally-initialized value as a fully-initialized value. */
30423 }
30425 /* Implement TARGET_MODE_CONFLUENCE. */
30427 static int
30429 {
30432 {
30440 }
30442 }
30444 /* Implement TARGET_MODE_BACKPROP for an entity that either stays
30445 NO throughput, or makes one transition from NO to YES. */
30447 static aarch64_tristate_mode
30449 aarch64_tristate_mode mode2)
30450 {
30451 /* Keep bringing the transition forward until it starts from NO. */
30457 }
30459 /* Implement TARGET_MODE_BACKPROP for LOCAL_SME_STATE. */
30461 static aarch64_local_sme_state
30463 aarch64_local_sme_state mode2)
30464 {
30465 /* We always need to know what the current state is when transitioning
30466 to a new state. Force any location with indeterminate starting state
30467 to be active. */
30470 {
30474 /* The current function's ZA state is not live. */
30479 /* The current function's ZA state is live. */
30483 /* This is a transition to an exception handler. Since we don't
30484 support non-call exceptions for SME functions, the source of
30485 the transition must be known. We'll assert later if that's
30486 not the case. */
30491 }
30494 }
30496 /* Implement TARGET_MODE_BACKPROP. */
30498 static int
30500 {
30502 {
30511 }
30513 }
30515 /* Implement TARGET_MODE_ENTRY. */
30517 static int
30519 {
30521 {
30531 }
30533 }
30535 /* Implement TARGET_MODE_EXIT. */
30537 static int
30539 {
30541 {
30551 }
30553 }
30555 /* Implement TARGET_MODE_EH_HANDLER. */
30557 static int
30559 {
30561 {
30563 /* Require a lazy save buffer to be allocated before the first
30564 insn that can throw. */
30569 }
30571 }
30573 /* Implement TARGET_MODE_PRIORITY. */
30575 static int
30577 {
30579 }
30581 /* Implement TARGET_MD_ASM_ADJUST. */
30589 {
30593 /* "za" in the clobber list of a function with ZA state is defined to
30594 mean that the asm can read from and write to ZA. We can model the
30595 read using a USE, but unfortunately, it's not possible to model the
30596 write directly. Use a separate insn to model the effect.
30598 We must ensure that ZA is active on entry, which is enforced by using
30599 SME_STATE_REGNUM. The asm must ensure that ZA is active on return.
30601 The same thing applies to ZT0. */
30604 {
30608 {
30627 }
30628 }
30630 }
30632 /* BB is the target of an exception or nonlocal goto edge, which means
30633 that PSTATE.SM is known to be 0 on entry. Put it into the state that
30634 the current function requires. */
30636 static bool
30638 {
30646 AARCH64_ISA_MODE_SM_OFF);
30647 aarch64_sme_mode_switch_regs args_switch;
30659 }
30661 /* JUMP is a nonlocal goto. Its target requires PSTATE.SM to be 0 on entry,
30662 so arrange to make it so. */
30664 static bool
30666 {
30674 AARCH64_ISA_MODE_SM_OFF);
30683 }
30685 /* If CALL involves a change in PSTATE.SM, emit the instructions needed
30686 to switch to the new mode and the instructions needed to restore the
30687 original mode. Return true if something changed. */
30688 static bool
30690 {
30691 /* Mode switches for sibling calls are handled via the epilogue. */
30699 /* Switch mode before the call, preserving any argument registers
30700 across the switch. */
30705 callee_isa_mode);
30706 aarch64_sme_mode_switch_regs args_switch;
30720 /* Switch mode after the call, preserving any return registers across
30721 the switch. */
30726 callee_isa_mode);
30727 aarch64_sme_mode_switch_regs return_switch;
30738 }
30743 {
30753 };
30756 {
30760 {}
30762 // opt_pass methods:
30765 };
30767 bool
30769 {
30772 }
30774 /* Emit any instructions needed to switch PSTATE.SM. */
30775 unsigned int
30777 {
30778 basic_block bb;
30783 {
30789 {
30795 }
30802 }
30806 }
30808 }
30810 rtl_opt_pass *
30812 {
30814 }
30816 /* Parse an implementation-defined system register name of
30817 the form S[0-3]_[0-7]_C[0-15]_C[0-15]_[0-7].
30818 Return true if name matched against above pattern, false
30819 otherwise. */
30820 bool
30822 {
30829 {
30836 pos++;
30838 {
30848 }
30852 };
30856 pos++;
30859 pos++;
30864 pos++;
30878 }
30880 /* Return true if REGNAME matches either a known permitted system
30881 register name, or a generic sysreg specification. For use in
30882 back-end predicate `aarch64_sysreg_string'. */
30883 bool
30885 {
30892 }
30894 /* Return the generic sysreg specification for a valid system register
30895 name, otherwise NULL. WRITE_P is true iff the register is being
30896 written to. IS128OP indicates the requested system register should
30897 be checked for a 128-bit implementation. */
30900 {
30903 {
30906 else
30908 }
30917 }
30919 /* Report that LOCATION has a call to FNDECL in which argument ARGNO
30920 was not an integer constant expression. ARGNO counts from zero. */
30921 void
30923 {
30926 }
30928 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
30929 the value ACTUAL, whereas the function requires a value in the range
30930 [MIN, MAX]. ARGNO counts from zero. */
30931 void
30935 {
30939 else
30943 }
30945 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
30946 the value ACTUAL, whereas the function requires either VALUE0 or
30947 VALUE1. ARGNO counts from zero. */
30948 void
30952 {
30955 }
30957 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
30958 the value ACTUAL, whereas the function requires one of VALUE0..3.
30959 ARGNO counts from zero. */
30960 void
30964 HOST_WIDE_INT value2,
30965 HOST_WIDE_INT value3)
30966 {
30970 }
30972 /* Report that LOCATION has a call to FNDECL in which argument ARGNO has
30973 the value ACTUAL, whereas the function requires a valid value of
30974 enum type ENUMTYPE. ARGNO counts from zero. */
30975 void
30978 {
30981 }
30983 /* Generate assembly to calculate CRC
30984 using carry-less multiplication instruction.
30985 OPERANDS[1] is input CRC,
30986 OPERANDS[2] is data (message),
30987 OPERANDS[3] is the polynomial without the leading 1. */
30989 void
30991 scalar_mode data_mode,
30993 {
30994 /* Check and keep arguments. */
31006 /* Calculate the quotient. */
31007 unsigned HOST_WIDE_INT
31009 /* CRC calculation's main part. */
31020 OPTAB_WIDEN);
31032 {
31036 OPTAB_DIRECT);
31037 }
31040 }
31042 /* Generate assembly to calculate reversed CRC
31043 using carry-less multiplication instruction.
31044 OPERANDS[1] is input CRC,
31045 OPERANDS[2] is data,
31046 OPERANDS[3] is the polynomial without the leading 1. */
31048 void
31050 scalar_mode data_mode,
31052 {
31053 /* Check and keep arguments. */
31065 /* Calculate the quotient. */
31066 unsigned HOST_WIDE_INT
31068 /* Reflect the calculated quotient. */
31072 /* Reflect the polynomial. */
31074 crc_size);
31075 /* An unshifted multiplier would require the final result to be extracted
31076 using a shift right by DATA_SIZE - 1 bits. Shift the multiplier left
31077 so that the shift right can be by CRC_SIZE bits instead. */
31081 /* CRC calculation's main part. */
31083 OPTAB_WIDEN);
31085 /* Perform carry-less multiplication and get low part. */
31094 /* Perform carry-less multiplication. */
31097 /* Perform a shift right by CRC_SIZE as an extraction of lane 1. */
31105 {
31109 OPTAB_WIDEN);
31111 }
31112 }
31114 /* Target-specific selftests. */
31116 #if CHECKING_P
31120 /* Selftest for the RTL loader.
31121 Verify that the RTL loader copes with a dump from
31122 print_rtx_function. This is essentially just a test that class
31123 function_reader can handle a real dump, but it also verifies
31124 that lookup_reg_by_dump_name correctly handles hard regs.
31125 The presence of hard reg names in the dump means that the test is
31126 target-specific, hence it is in this file. */
31128 static void
31130 {
31142 /* Verify crtl->return_rtx. */
31146 }
31148 /* Test the fractional_cost class. */
31150 static void
31152 {
31239 }
31241 /* Calculate whether our system register data, as imported from
31242 `aarch64-sys-reg.def' has any duplicate entries. */
31243 static void
31245 {
31249 dup_instances_t duplicate_instances;
31251 /* Every time an encoding is established to come up more than once
31252 we add it to a "clash-analysis queue", which is then used to extract
31253 necessary information from our hash map when establishing whether
31254 repeated encodings are valid. */
31256 /* 1) Collect recurrence information. */
31258 {
31265 }
31267 /* 2) Carry out analysis on collected data. */
31269 {
31274 {
31279 }
31280 }
31281 }
31283 /* Run all target-specific selftests. */
31285 static void
31287 {
31291 }
31297 #undef TARGET_STACK_PROTECT_GUARD
31298 #define TARGET_STACK_PROTECT_GUARD aarch64_stack_protect_guard
31300 #undef TARGET_ADDRESS_COST
31301 #define TARGET_ADDRESS_COST aarch64_address_cost
31303 /* This hook will determines whether unnamed bitfields affect the alignment
31304 of the containing structure. The hook returns true if the structure
31305 should inherit the alignment requirements of an unnamed bitfield's
31306 type. */
31307 #undef TARGET_ALIGN_ANON_BITFIELD
31308 #define TARGET_ALIGN_ANON_BITFIELD hook_bool_void_true
31310 #undef TARGET_ASM_ALIGNED_DI_OP
31313 #undef TARGET_ASM_ALIGNED_HI_OP
31316 #undef TARGET_ASM_ALIGNED_SI_OP
31319 #if TARGET_PECOFF
31320 #undef TARGET_ASM_UNALIGNED_HI_OP
31321 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
31322 #undef TARGET_ASM_UNALIGNED_SI_OP
31323 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
31324 #undef TARGET_ASM_UNALIGNED_DI_OP
31325 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
31326 #endif
31328 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
31329 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK \
31330 hook_bool_const_tree_hwi_hwi_const_tree_true
31332 #undef TARGET_ASM_FILE_START
31333 #define TARGET_ASM_FILE_START aarch64_start_file
31335 #undef TARGET_ASM_OUTPUT_MI_THUNK
31336 #define TARGET_ASM_OUTPUT_MI_THUNK aarch64_output_mi_thunk
31338 #undef TARGET_ASM_SELECT_RTX_SECTION
31339 #define TARGET_ASM_SELECT_RTX_SECTION aarch64_select_rtx_section
31341 #undef TARGET_ASM_TRAMPOLINE_TEMPLATE
31342 #define TARGET_ASM_TRAMPOLINE_TEMPLATE aarch64_asm_trampoline_template
31344 #undef TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY
31345 #define TARGET_ASM_PRINT_PATCHABLE_FUNCTION_ENTRY aarch64_print_patchable_function_entry
31347 #undef TARGET_BUILD_BUILTIN_VA_LIST
31348 #define TARGET_BUILD_BUILTIN_VA_LIST aarch64_build_builtin_va_list
31350 #undef TARGET_CALLEE_COPIES
31351 #define TARGET_CALLEE_COPIES hook_bool_CUMULATIVE_ARGS_arg_info_false
31353 #undef TARGET_FRAME_POINTER_REQUIRED
31354 #define TARGET_FRAME_POINTER_REQUIRED aarch64_frame_pointer_required
31356 #undef TARGET_CAN_ELIMINATE
31357 #define TARGET_CAN_ELIMINATE aarch64_can_eliminate
31359 #undef TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P
31360 #define TARGET_FUNCTION_ATTRIBUTE_INLINABLE_P \
31361 aarch64_function_attribute_inlinable_p
31363 #undef TARGET_NEED_IPA_FN_TARGET_INFO
31364 #define TARGET_NEED_IPA_FN_TARGET_INFO aarch64_need_ipa_fn_target_info
31366 #undef TARGET_UPDATE_IPA_FN_TARGET_INFO
31367 #define TARGET_UPDATE_IPA_FN_TARGET_INFO aarch64_update_ipa_fn_target_info
31369 #undef TARGET_CAN_INLINE_P
31370 #define TARGET_CAN_INLINE_P aarch64_can_inline_p
31372 #undef TARGET_CANNOT_FORCE_CONST_MEM
31373 #define TARGET_CANNOT_FORCE_CONST_MEM aarch64_cannot_force_const_mem
31375 #undef TARGET_CASE_VALUES_THRESHOLD
31376 #define TARGET_CASE_VALUES_THRESHOLD aarch64_case_values_threshold
31378 #undef TARGET_CONDITIONAL_REGISTER_USAGE
31379 #define TARGET_CONDITIONAL_REGISTER_USAGE aarch64_conditional_register_usage
31381 #undef TARGET_MEMBER_TYPE_FORCES_BLK
31382 #define TARGET_MEMBER_TYPE_FORCES_BLK aarch64_member_type_forces_blk
31384 /* Only the least significant bit is used for initialization guard
31385 variables. */
31386 #undef TARGET_CXX_GUARD_MASK_BIT
31387 #define TARGET_CXX_GUARD_MASK_BIT hook_bool_void_true
31389 #undef TARGET_C_MODE_FOR_SUFFIX
31390 #define TARGET_C_MODE_FOR_SUFFIX aarch64_c_mode_for_suffix
31392 #ifdef TARGET_BIG_ENDIAN_DEFAULT
31393 #undef TARGET_DEFAULT_TARGET_FLAGS
31394 #define TARGET_DEFAULT_TARGET_FLAGS (MASK_BIG_END)
31395 #endif
31397 #undef TARGET_CLASS_MAX_NREGS
31398 #define TARGET_CLASS_MAX_NREGS aarch64_class_max_nregs
31400 #undef TARGET_BUILTIN_DECL
31401 #define TARGET_BUILTIN_DECL aarch64_builtin_decl
31403 #undef TARGET_BUILTIN_RECIPROCAL
31404 #define TARGET_BUILTIN_RECIPROCAL aarch64_builtin_reciprocal
31406 #undef TARGET_C_EXCESS_PRECISION
31407 #define TARGET_C_EXCESS_PRECISION aarch64_excess_precision
31409 #undef TARGET_C_BITINT_TYPE_INFO
31410 #define TARGET_C_BITINT_TYPE_INFO aarch64_bitint_type_info
31412 #undef TARGET_C_MODE_FOR_FLOATING_TYPE
31413 #define TARGET_C_MODE_FOR_FLOATING_TYPE aarch64_c_mode_for_floating_type
31415 #undef TARGET_EXPAND_BUILTIN
31416 #define TARGET_EXPAND_BUILTIN aarch64_expand_builtin
31418 #undef TARGET_EXPAND_BUILTIN_VA_START
31419 #define TARGET_EXPAND_BUILTIN_VA_START aarch64_expand_builtin_va_start
31421 #undef TARGET_FOLD_BUILTIN
31422 #define TARGET_FOLD_BUILTIN aarch64_fold_builtin
31424 #undef TARGET_FUNCTION_ARG
31425 #define TARGET_FUNCTION_ARG aarch64_function_arg
31427 #undef TARGET_FUNCTION_ARG_ADVANCE
31428 #define TARGET_FUNCTION_ARG_ADVANCE aarch64_function_arg_advance
31430 #undef TARGET_FUNCTION_ARG_BOUNDARY
31431 #define TARGET_FUNCTION_ARG_BOUNDARY aarch64_function_arg_boundary
31433 #undef TARGET_FUNCTION_ARG_PADDING
31434 #define TARGET_FUNCTION_ARG_PADDING aarch64_function_arg_padding
31436 #undef TARGET_GET_RAW_RESULT_MODE
31437 #define TARGET_GET_RAW_RESULT_MODE aarch64_get_reg_raw_mode
31438 #undef TARGET_GET_RAW_ARG_MODE
31439 #define TARGET_GET_RAW_ARG_MODE aarch64_get_reg_raw_mode
31441 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
31442 #define TARGET_FUNCTION_OK_FOR_SIBCALL aarch64_function_ok_for_sibcall
31444 #undef TARGET_FUNCTION_VALUE
31445 #define TARGET_FUNCTION_VALUE aarch64_function_value
31447 #undef TARGET_FUNCTION_VALUE_REGNO_P
31448 #define TARGET_FUNCTION_VALUE_REGNO_P aarch64_function_value_regno_p
31450 #undef TARGET_START_CALL_ARGS
31451 #define TARGET_START_CALL_ARGS aarch64_start_call_args
31453 #undef TARGET_END_CALL_ARGS
31454 #define TARGET_END_CALL_ARGS aarch64_end_call_args
31456 #undef TARGET_GIMPLE_FOLD_BUILTIN
31457 #define TARGET_GIMPLE_FOLD_BUILTIN aarch64_gimple_fold_builtin
31459 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
31460 #define TARGET_GIMPLIFY_VA_ARG_EXPR aarch64_gimplify_va_arg_expr
31462 #undef TARGET_INIT_BUILTINS
31463 #define TARGET_INIT_BUILTINS aarch64_init_builtins
31465 #undef TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS
31466 #define TARGET_IRA_CHANGE_PSEUDO_ALLOCNO_CLASS \
31467 aarch64_ira_change_pseudo_allocno_class
31469 #undef TARGET_LEGITIMATE_ADDRESS_P
31470 #define TARGET_LEGITIMATE_ADDRESS_P aarch64_legitimate_address_hook_p
31472 #undef TARGET_LEGITIMATE_CONSTANT_P
31473 #define TARGET_LEGITIMATE_CONSTANT_P aarch64_legitimate_constant_p
31475 #undef TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT
31476 #define TARGET_LEGITIMIZE_ADDRESS_DISPLACEMENT \
31477 aarch64_legitimize_address_displacement
31479 #undef TARGET_LIBGCC_CMP_RETURN_MODE
31480 #define TARGET_LIBGCC_CMP_RETURN_MODE aarch64_libgcc_cmp_return_mode
31482 #undef TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P
31483 #define TARGET_LIBGCC_FLOATING_MODE_SUPPORTED_P \
31484 aarch64_libgcc_floating_mode_supported_p
31486 #undef TARGET_MANGLE_TYPE
31487 #define TARGET_MANGLE_TYPE aarch64_mangle_type
31489 #undef TARGET_INVALID_CONVERSION
31490 #define TARGET_INVALID_CONVERSION aarch64_invalid_conversion
31492 #undef TARGET_INVALID_UNARY_OP
31493 #define TARGET_INVALID_UNARY_OP aarch64_invalid_unary_op
31495 #undef TARGET_INVALID_BINARY_OP
31496 #define TARGET_INVALID_BINARY_OP aarch64_invalid_binary_op
31498 #undef TARGET_VERIFY_TYPE_CONTEXT
31499 #define TARGET_VERIFY_TYPE_CONTEXT aarch64_verify_type_context
31501 #undef TARGET_MEMORY_MOVE_COST
31502 #define TARGET_MEMORY_MOVE_COST aarch64_memory_move_cost
31504 #undef TARGET_MIN_DIVISIONS_FOR_RECIP_MUL
31505 #define TARGET_MIN_DIVISIONS_FOR_RECIP_MUL aarch64_min_divisions_for_recip_mul
31507 #undef TARGET_MUST_PASS_IN_STACK
31508 #define TARGET_MUST_PASS_IN_STACK must_pass_in_stack_var_size
31510 /* This target hook should return true if accesses to volatile bitfields
31511 should use the narrowest mode possible. It should return false if these
31512 accesses should use the bitfield container type. */
31513 #undef TARGET_NARROW_VOLATILE_BITFIELD
31514 #define TARGET_NARROW_VOLATILE_BITFIELD hook_bool_void_false
31516 #undef TARGET_OPTION_OVERRIDE
31517 #define TARGET_OPTION_OVERRIDE aarch64_override_options
31519 #undef TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE
31520 #define TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE \
31521 aarch64_override_options_after_change
31523 #undef TARGET_OFFLOAD_OPTIONS
31524 #define TARGET_OFFLOAD_OPTIONS aarch64_offload_options
31526 #undef TARGET_OPTION_RESTORE
31527 #define TARGET_OPTION_RESTORE aarch64_option_restore
31529 #undef TARGET_OPTION_PRINT
31530 #define TARGET_OPTION_PRINT aarch64_option_print
31532 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
31533 #define TARGET_OPTION_VALID_ATTRIBUTE_P aarch64_option_valid_attribute_p
31535 #undef TARGET_OPTION_VALID_VERSION_ATTRIBUTE_P
31536 #define TARGET_OPTION_VALID_VERSION_ATTRIBUTE_P \
31537 aarch64_option_valid_version_attribute_p
31539 #undef TARGET_SET_CURRENT_FUNCTION
31540 #define TARGET_SET_CURRENT_FUNCTION aarch64_set_current_function
31542 #undef TARGET_PASS_BY_REFERENCE
31543 #define TARGET_PASS_BY_REFERENCE aarch64_pass_by_reference
31545 #undef TARGET_PREFERRED_RELOAD_CLASS
31546 #define TARGET_PREFERRED_RELOAD_CLASS aarch64_preferred_reload_class
31548 #undef TARGET_SCHED_REASSOCIATION_WIDTH
31549 #define TARGET_SCHED_REASSOCIATION_WIDTH aarch64_reassociation_width
31551 #undef TARGET_DWARF_FRAME_REG_MODE
31552 #define TARGET_DWARF_FRAME_REG_MODE aarch64_dwarf_frame_reg_mode
31554 #undef TARGET_OUTPUT_CFI_DIRECTIVE
31555 #define TARGET_OUTPUT_CFI_DIRECTIVE aarch64_output_cfi_directive
31557 #undef TARGET_DW_CFI_OPRND1_DESC
31558 #define TARGET_DW_CFI_OPRND1_DESC aarch64_dw_cfi_oprnd1_desc
31560 #undef TARGET_PROMOTED_TYPE
31561 #define TARGET_PROMOTED_TYPE aarch64_promoted_type
31563 #undef TARGET_SECONDARY_RELOAD
31564 #define TARGET_SECONDARY_RELOAD aarch64_secondary_reload
31566 #undef TARGET_SECONDARY_MEMORY_NEEDED
31567 #define TARGET_SECONDARY_MEMORY_NEEDED aarch64_secondary_memory_needed
31569 #undef TARGET_SHIFT_TRUNCATION_MASK
31570 #define TARGET_SHIFT_TRUNCATION_MASK aarch64_shift_truncation_mask
31572 #undef TARGET_SETUP_INCOMING_VARARGS
31573 #define TARGET_SETUP_INCOMING_VARARGS aarch64_setup_incoming_varargs
31575 #undef TARGET_STRUCT_VALUE_RTX
31576 #define TARGET_STRUCT_VALUE_RTX aarch64_struct_value_rtx
31578 #undef TARGET_REGISTER_MOVE_COST
31579 #define TARGET_REGISTER_MOVE_COST aarch64_register_move_cost
31581 #undef TARGET_RETURN_IN_MEMORY
31582 #define TARGET_RETURN_IN_MEMORY aarch64_return_in_memory
31584 #undef TARGET_RETURN_IN_MSB
31585 #define TARGET_RETURN_IN_MSB aarch64_return_in_msb
31587 #undef TARGET_RTX_COSTS
31588 #define TARGET_RTX_COSTS aarch64_rtx_costs_wrapper
31590 #undef TARGET_INSN_COST
31591 #define TARGET_INSN_COST aarch64_insn_cost
31593 #undef TARGET_SCALAR_MODE_SUPPORTED_P
31594 #define TARGET_SCALAR_MODE_SUPPORTED_P aarch64_scalar_mode_supported_p
31596 #undef TARGET_SCHED_ISSUE_RATE
31597 #define TARGET_SCHED_ISSUE_RATE aarch64_sched_issue_rate
31599 #undef TARGET_SCHED_VARIABLE_ISSUE
31600 #define TARGET_SCHED_VARIABLE_ISSUE aarch64_sched_variable_issue
31602 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
31603 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
31604 aarch64_sched_first_cycle_multipass_dfa_lookahead
31606 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD
31607 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD_GUARD \
31608 aarch64_first_cycle_multipass_dfa_lookahead_guard
31610 #undef TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS
31611 #define TARGET_SHRINK_WRAP_GET_SEPARATE_COMPONENTS \
31612 aarch64_get_separate_components
31614 #undef TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB
31615 #define TARGET_SHRINK_WRAP_COMPONENTS_FOR_BB \
31616 aarch64_components_for_bb
31618 #undef TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS
31619 #define TARGET_SHRINK_WRAP_DISQUALIFY_COMPONENTS \
31620 aarch64_disqualify_components
31622 #undef TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS
31623 #define TARGET_SHRINK_WRAP_EMIT_PROLOGUE_COMPONENTS \
31624 aarch64_emit_prologue_components
31626 #undef TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS
31627 #define TARGET_SHRINK_WRAP_EMIT_EPILOGUE_COMPONENTS \
31628 aarch64_emit_epilogue_components
31630 #undef TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS
31631 #define TARGET_SHRINK_WRAP_SET_HANDLED_COMPONENTS \
31632 aarch64_set_handled_components
31634 #undef TARGET_TRAMPOLINE_INIT
31635 #define TARGET_TRAMPOLINE_INIT aarch64_trampoline_init
31637 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
31638 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P aarch64_use_blocks_for_constant_p
31640 #undef TARGET_VECTOR_MODE_SUPPORTED_P
31641 #define TARGET_VECTOR_MODE_SUPPORTED_P aarch64_vector_mode_supported_p
31643 #undef TARGET_VECTOR_MODE_SUPPORTED_ANY_TARGET_P
31644 #define TARGET_VECTOR_MODE_SUPPORTED_ANY_TARGET_P aarch64_vector_mode_supported_any_target_p
31646 #undef TARGET_COMPATIBLE_VECTOR_TYPES_P
31647 #define TARGET_COMPATIBLE_VECTOR_TYPES_P aarch64_compatible_vector_types_p
31649 #undef TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT
31650 #define TARGET_VECTORIZE_SUPPORT_VECTOR_MISALIGNMENT \
31651 aarch64_builtin_support_vector_misalignment
31653 #undef TARGET_ARRAY_MODE
31654 #define TARGET_ARRAY_MODE aarch64_array_mode
31656 #undef TARGET_ARRAY_MODE_SUPPORTED_P
31657 #define TARGET_ARRAY_MODE_SUPPORTED_P aarch64_array_mode_supported_p
31659 #undef TARGET_VECTORIZE_CREATE_COSTS
31660 #define TARGET_VECTORIZE_CREATE_COSTS aarch64_vectorize_create_costs
31662 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
31663 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
31664 aarch64_builtin_vectorization_cost
31666 #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
31667 #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE aarch64_preferred_simd_mode
31669 #undef TARGET_VECTORIZE_BUILTINS
31670 #define TARGET_VECTORIZE_BUILTINS
31672 #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES
31673 #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_MODES \
31674 aarch64_autovectorize_vector_modes
31676 #undef TARGET_ATOMIC_ASSIGN_EXPAND_FENV
31677 #define TARGET_ATOMIC_ASSIGN_EXPAND_FENV \
31678 aarch64_atomic_assign_expand_fenv
31680 /* Section anchor support. */
31682 #undef TARGET_MIN_ANCHOR_OFFSET
31683 #define TARGET_MIN_ANCHOR_OFFSET -256
31685 /* Limit the maximum anchor offset to 4k-1, since that's the limit for a
31686 byte offset; we can do much more for larger data types, but have no way
31687 to determine the size of the access. We assume accesses are aligned. */
31688 #undef TARGET_MAX_ANCHOR_OFFSET
31689 #define TARGET_MAX_ANCHOR_OFFSET 4095
31691 #undef TARGET_VECTORIZE_PREFERRED_DIV_AS_SHIFTS_OVER_MULT
31692 #define TARGET_VECTORIZE_PREFERRED_DIV_AS_SHIFTS_OVER_MULT \
31693 aarch64_vectorize_preferred_div_as_shifts_over_mult
31695 #undef TARGET_VECTOR_ALIGNMENT
31696 #define TARGET_VECTOR_ALIGNMENT aarch64_simd_vector_alignment
31698 #undef TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT
31699 #define TARGET_VECTORIZE_PREFERRED_VECTOR_ALIGNMENT \
31700 aarch64_vectorize_preferred_vector_alignment
31701 #undef TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE
31702 #define TARGET_VECTORIZE_VECTOR_ALIGNMENT_REACHABLE \
31703 aarch64_simd_vector_alignment_reachable
31705 /* vec_perm support. */
31707 #undef TARGET_VECTORIZE_VEC_PERM_CONST
31708 #define TARGET_VECTORIZE_VEC_PERM_CONST \
31709 aarch64_vectorize_vec_perm_const
31711 #undef TARGET_VECTORIZE_RELATED_MODE
31712 #define TARGET_VECTORIZE_RELATED_MODE aarch64_vectorize_related_mode
31713 #undef TARGET_VECTORIZE_GET_MASK_MODE
31714 #define TARGET_VECTORIZE_GET_MASK_MODE aarch64_get_mask_mode
31715 #undef TARGET_VECTORIZE_CONDITIONAL_OPERATION_IS_EXPENSIVE
31716 #define TARGET_VECTORIZE_CONDITIONAL_OPERATION_IS_EXPENSIVE \
31717 aarch64_conditional_operation_is_expensive
31718 #undef TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE
31719 #define TARGET_VECTORIZE_EMPTY_MASK_IS_EXPENSIVE \
31720 aarch64_empty_mask_is_expensive
31721 #undef TARGET_PREFERRED_ELSE_VALUE
31722 #define TARGET_PREFERRED_ELSE_VALUE \
31723 aarch64_preferred_else_value
31725 #undef TARGET_INIT_LIBFUNCS
31726 #define TARGET_INIT_LIBFUNCS aarch64_init_libfuncs
31728 #undef TARGET_FIXED_CONDITION_CODE_REGS
31729 #define TARGET_FIXED_CONDITION_CODE_REGS aarch64_fixed_condition_code_regs
31731 #undef TARGET_FLAGS_REGNUM
31732 #define TARGET_FLAGS_REGNUM CC_REGNUM
31734 #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS
31735 #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true
31737 #undef TARGET_ASAN_SHADOW_OFFSET
31738 #define TARGET_ASAN_SHADOW_OFFSET aarch64_asan_shadow_offset
31740 #undef TARGET_LEGITIMIZE_ADDRESS
31741 #define TARGET_LEGITIMIZE_ADDRESS aarch64_legitimize_address
31743 #undef TARGET_SCHED_CAN_SPECULATE_INSN
31744 #define TARGET_SCHED_CAN_SPECULATE_INSN aarch64_sched_can_speculate_insn
31746 #undef TARGET_CAN_USE_DOLOOP_P
31747 #define TARGET_CAN_USE_DOLOOP_P can_use_doloop_if_innermost
31749 #undef TARGET_SCHED_ADJUST_PRIORITY
31750 #define TARGET_SCHED_ADJUST_PRIORITY aarch64_sched_adjust_priority
31752 #undef TARGET_SCHED_MACRO_FUSION_P
31753 #define TARGET_SCHED_MACRO_FUSION_P aarch64_macro_fusion_p
31755 #undef TARGET_SCHED_MACRO_FUSION_PAIR_P
31756 #define TARGET_SCHED_MACRO_FUSION_PAIR_P aarch_macro_fusion_pair_p
31758 #undef TARGET_SCHED_FUSION_PRIORITY
31759 #define TARGET_SCHED_FUSION_PRIORITY aarch64_sched_fusion_priority
31761 #undef TARGET_UNSPEC_MAY_TRAP_P
31762 #define TARGET_UNSPEC_MAY_TRAP_P aarch64_unspec_may_trap_p
31764 #undef TARGET_USE_PSEUDO_PIC_REG
31765 #define TARGET_USE_PSEUDO_PIC_REG aarch64_use_pseudo_pic_reg
31767 #undef TARGET_PRINT_OPERAND
31768 #define TARGET_PRINT_OPERAND aarch64_print_operand
31770 #undef TARGET_PRINT_OPERAND_ADDRESS
31771 #define TARGET_PRINT_OPERAND_ADDRESS aarch64_print_operand_address
31773 #undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
31774 #define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA aarch64_output_addr_const_extra
31776 #undef TARGET_OPTAB_SUPPORTED_P
31777 #define TARGET_OPTAB_SUPPORTED_P aarch64_optab_supported_p
31779 #undef TARGET_OMIT_STRUCT_RETURN_REG
31780 #define TARGET_OMIT_STRUCT_RETURN_REG true
31782 #undef TARGET_DWARF_POLY_INDETERMINATE_VALUE
31783 #define TARGET_DWARF_POLY_INDETERMINATE_VALUE \
31784 aarch64_dwarf_poly_indeterminate_value
31786 /* The architecture reserves bits 0 and 1 so use bit 2 for descriptors. */
31787 #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS
31788 #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 4
31790 #undef TARGET_HARD_REGNO_NREGS
31791 #define TARGET_HARD_REGNO_NREGS aarch64_hard_regno_nregs
31792 #undef TARGET_HARD_REGNO_MODE_OK
31793 #define TARGET_HARD_REGNO_MODE_OK aarch64_hard_regno_mode_ok
31795 #undef TARGET_MODES_TIEABLE_P
31796 #define TARGET_MODES_TIEABLE_P aarch64_modes_tieable_p
31798 #undef TARGET_HARD_REGNO_CALL_PART_CLOBBERED
31799 #define TARGET_HARD_REGNO_CALL_PART_CLOBBERED \
31800 aarch64_hard_regno_call_part_clobbered
31802 #undef TARGET_INSN_CALLEE_ABI
31803 #define TARGET_INSN_CALLEE_ABI aarch64_insn_callee_abi
31805 #undef TARGET_CONSTANT_ALIGNMENT
31806 #define TARGET_CONSTANT_ALIGNMENT aarch64_constant_alignment
31808 #undef TARGET_STACK_CLASH_PROTECTION_ALLOCA_PROBE_RANGE
31809 #define TARGET_STACK_CLASH_PROTECTION_ALLOCA_PROBE_RANGE \
31810 aarch64_stack_clash_protection_alloca_probe_range
31812 #undef TARGET_COMPUTE_PRESSURE_CLASSES
31813 #define TARGET_COMPUTE_PRESSURE_CLASSES aarch64_compute_pressure_classes
31815 #undef TARGET_CAN_CHANGE_MODE_CLASS
31816 #define TARGET_CAN_CHANGE_MODE_CLASS aarch64_can_change_mode_class
31818 #undef TARGET_SELECT_EARLY_REMAT_MODES
31819 #define TARGET_SELECT_EARLY_REMAT_MODES aarch64_select_early_remat_modes
31821 #undef TARGET_SPECULATION_SAFE_VALUE
31822 #define TARGET_SPECULATION_SAFE_VALUE aarch64_speculation_safe_value
31824 #undef TARGET_ESTIMATED_POLY_VALUE
31825 #define TARGET_ESTIMATED_POLY_VALUE aarch64_estimated_poly_value
31827 #undef TARGET_ATTRIBUTE_TABLE
31828 #define TARGET_ATTRIBUTE_TABLE aarch64_attribute_table
31830 #undef TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN
31831 #define TARGET_SIMD_CLONE_COMPUTE_VECSIZE_AND_SIMDLEN \
31832 aarch64_simd_clone_compute_vecsize_and_simdlen
31834 #undef TARGET_SIMD_CLONE_ADJUST
31835 #define TARGET_SIMD_CLONE_ADJUST aarch64_simd_clone_adjust
31837 #undef TARGET_SIMD_CLONE_USABLE
31838 #define TARGET_SIMD_CLONE_USABLE aarch64_simd_clone_usable
31840 #undef TARGET_COMP_TYPE_ATTRIBUTES
31841 #define TARGET_COMP_TYPE_ATTRIBUTES aarch64_comp_type_attributes
31843 #undef TARGET_MERGE_DECL_ATTRIBUTES
31844 #define TARGET_MERGE_DECL_ATTRIBUTES aarch64_merge_decl_attributes
31846 #undef TARGET_GET_MULTILIB_ABI_NAME
31847 #define TARGET_GET_MULTILIB_ABI_NAME aarch64_get_multilib_abi_name
31849 #undef TARGET_FNTYPE_ABI
31850 #define TARGET_FNTYPE_ABI aarch64_fntype_abi
31852 #undef TARGET_MEMTAG_CAN_TAG_ADDRESSES
31853 #define TARGET_MEMTAG_CAN_TAG_ADDRESSES aarch64_can_tag_addresses
31855 #if CHECKING_P
31856 #undef TARGET_RUN_TARGET_SELFTESTS
31857 #define TARGET_RUN_TARGET_SELFTESTS selftest::aarch64_run_selftests
31860 #undef TARGET_ASM_POST_CFI_STARTPROC
31861 #define TARGET_ASM_POST_CFI_STARTPROC aarch64_post_cfi_startproc
31863 #undef TARGET_STRICT_ARGUMENT_NAMING
31864 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
31866 #undef TARGET_MODE_EMIT
31867 #define TARGET_MODE_EMIT aarch64_mode_emit
31869 #undef TARGET_MODE_NEEDED
31870 #define TARGET_MODE_NEEDED aarch64_mode_needed
31872 #undef TARGET_MODE_AFTER
31873 #define TARGET_MODE_AFTER aarch64_mode_after
31875 #undef TARGET_MODE_CONFLUENCE
31876 #define TARGET_MODE_CONFLUENCE aarch64_mode_confluence
31878 #undef TARGET_MODE_BACKPROP
31879 #define TARGET_MODE_BACKPROP aarch64_mode_backprop
31881 #undef TARGET_MODE_ENTRY
31882 #define TARGET_MODE_ENTRY aarch64_mode_entry
31884 #undef TARGET_MODE_EXIT
31885 #define TARGET_MODE_EXIT aarch64_mode_exit
31887 #undef TARGET_MODE_EH_HANDLER
31888 #define TARGET_MODE_EH_HANDLER aarch64_mode_eh_handler
31890 #undef TARGET_MODE_PRIORITY
31891 #define TARGET_MODE_PRIORITY aarch64_mode_priority
31893 #undef TARGET_MD_ASM_ADJUST
31894 #define TARGET_MD_ASM_ADJUST aarch64_md_asm_adjust
31896 #undef TARGET_ASM_FILE_END
31897 #define TARGET_ASM_FILE_END aarch64_asm_file_end
31899 #undef TARGET_ASM_FUNCTION_EPILOGUE
31900 #define TARGET_ASM_FUNCTION_EPILOGUE aarch64_sls_emit_blr_function_thunks
31902 #undef TARGET_HAVE_SHADOW_CALL_STACK
31903 #define TARGET_HAVE_SHADOW_CALL_STACK true
31905 #undef TARGET_CONST_ANCHOR
31906 #define TARGET_CONST_ANCHOR 0x1000000
31908 #undef TARGET_EXTRA_LIVE_ON_ENTRY
31909 #define TARGET_EXTRA_LIVE_ON_ENTRY aarch64_extra_live_on_entry
31911 #undef TARGET_USE_LATE_PROLOGUE_EPILOGUE
31912 #define TARGET_USE_LATE_PROLOGUE_EPILOGUE aarch64_use_late_prologue_epilogue
31914 #undef TARGET_EMIT_EPILOGUE_FOR_SIBCALL
31915 #define TARGET_EMIT_EPILOGUE_FOR_SIBCALL aarch64_expand_epilogue
31917 #undef TARGET_OPTION_FUNCTION_VERSIONS
31918 #define TARGET_OPTION_FUNCTION_VERSIONS aarch64_common_function_versions
31920 #undef TARGET_COMPARE_VERSION_PRIORITY
31921 #define TARGET_COMPARE_VERSION_PRIORITY aarch64_compare_version_priority
31923 #undef TARGET_GENERATE_VERSION_DISPATCHER_BODY
31924 #define TARGET_GENERATE_VERSION_DISPATCHER_BODY \
31925 aarch64_generate_version_dispatcher_body
31927 #undef TARGET_GET_FUNCTION_VERSIONS_DISPATCHER
31928 #define TARGET_GET_FUNCTION_VERSIONS_DISPATCHER \
31929 aarch64_get_function_versions_dispatcher
31931 #undef TARGET_MANGLE_DECL_ASSEMBLER_NAME
31932 #define TARGET_MANGLE_DECL_ASSEMBLER_NAME aarch64_mangle_decl_assembler_name
31934 #undef TARGET_DOCUMENTATION_NAME