gcc/config/aarch64/aarch64-sve-builtins.h

   1 /* ACLE support for AArch64 SVE
   2    Copyright (C) 2018-2025 Free Software Foundation, Inc.
   3
   4    This file is part of GCC.
   5
   6    GCC is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3, or (at your option)
   9    any later version.
  10
  11    GCC is distributed in the hope that it will be useful, but
  12    WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14    General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with GCC; see the file COPYING3.  If not see
  18    <http://www.gnu.org/licenses/>.  */
  19
  20 #ifndef GCC_AARCH64_SVE_BUILTINS_H
  21 #define GCC_AARCH64_SVE_BUILTINS_H
  22
  23 #include "aarch64-builtins.h"
  24
  25 /* The full name of an SVE ACLE function is the concatenation of:
  26
  27    - the base name ("svadd", etc.)
  28    - the "mode" suffix ("_n", "_index", etc.)
  29    - the type suffixes ("_s32", "_b8", etc.)
  30    - the predication suffix ("_x", "_z", etc.)
  31    - the "_fpm" suffix when the floating point mode register is set
  32
  33    Each piece of information is individually useful, so we retain this
  34    classification throughout:
  35
  36    - function_base represents the base name
  37
  38    - mode_suffix_index represents the mode suffix
  39
  40    - type_suffix_index represents individual type suffixes, while
  41      type_suffix_pair represents a pair of them
  42
  43    - prediction_index extends the predication suffix with an additional
  44      alternative: PRED_implicit for implicitly-predicated operations
  45
  46    - fpm_mode represents whether the fpm register is set or not
  47
  48    In addition to its unique full name, a function may have a shorter
  49    overloaded alias.  This alias removes pieces of the suffixes that
  50    can be inferred from the arguments, such as by shortening the mode
  51    suffix or dropping some of the type suffixes.  The base name and the
  52    predication suffix stay the same.
  53
  54    The function_shape class describes what arguments a given function
  55    takes and what its overloaded alias is called.  In broad terms,
  56    function_base describes how the underlying instruction behaves while
  57    function_shape describes how that instruction has been presented at
  58    the language level.
  59
  60    The static list of functions uses function_group to describe a group
  61    of related functions.  The function_builder class is responsible for
  62    expanding this static description into a list of individual functions
  63    and registering the associated built-in functions.  function_instance
  64    describes one of these individual functions in terms of the properties
  65    described above.
  66
  67    The classes involved in compiling a function call are:
  68
  69    - function_resolver, which resolves an overloaded function call to a
  70      specific function_instance and its associated function decl
  71
  72    - function_checker, which checks whether the values of the arguments
  73      conform to the ACLE specification
  74
  75    - gimple_folder, which tries to fold a function call at the gimple level
  76
  77    - function_expander, which expands a function call into rtl instructions
  78
  79    function_resolver and function_checker operate at the language level
  80    and so are associated with the function_shape.  gimple_folder and
  81    function_expander are concerned with the behavior of the function
  82    and so are associated with the function_base.
  83
  84    Note that we've specifically chosen not to fold calls in the frontend,
  85    since SVE intrinsics will hardly ever fold a useful language-level
  86    constant.  */
  87 namespace aarch64_sve
  88 {
  89 /* The maximum number of vectors in an ACLE tuple type.  */
  90 const unsigned int MAX_TUPLE_SIZE = 4;
  91
  92 /* Used to represent the default merge argument index for _m functions.
  93    The actual index depends on how many arguments the function takes.  */
  94 const unsigned int DEFAULT_MERGE_ARGNO = ~0U;
  95
  96 /* Flags that describe what a function might do, in addition to reading
  97    its arguments and returning a result.  */
  98 const unsigned int CP_READ_FPCR = 1U << 0;
  99 const unsigned int CP_RAISE_FP_EXCEPTIONS = 1U << 1;
 100 const unsigned int CP_READ_MEMORY = 1U << 2;
 101 const unsigned int CP_PREFETCH_MEMORY = 1U << 3;
 102 const unsigned int CP_WRITE_MEMORY = 1U << 4;
 103 const unsigned int CP_READ_FFR = 1U << 5;
 104 const unsigned int CP_WRITE_FFR = 1U << 6;
 105 const unsigned int CP_READ_ZA = 1U << 7;
 106 const unsigned int CP_WRITE_ZA = 1U << 8;
 107 const unsigned int CP_READ_ZT0 = 1U << 9;
 108 const unsigned int CP_WRITE_ZT0 = 1U << 10;
 109
 110 /* Enumerates the SVE predicate and (data) vector types, together called
 111    "vector types" for brevity.  */
 112 enum vector_type_index
 113 {
 114 #define DEF_SVE_TYPE(ACLE_NAME, NCHARS, ABI_NAME, SCALAR_TYPE) \
 115   VECTOR_TYPE_ ## ACLE_NAME,
 116 #include "aarch64-sve-builtins.def"
 117   NUM_VECTOR_TYPES
 118 };
 119
 120 /* Classifies the available measurement units for an address displacement.  */
 121 enum units_index
 122 {
 123   UNITS_none,
 124   UNITS_bytes,
 125   UNITS_elements,
 126   UNITS_vectors
 127 };
 128
 129 /* Enumerates the pragma handlers.  */
 130 enum handle_pragma_index
 131 {
 132   arm_sve_handle,
 133   arm_sme_handle,
 134   arm_neon_sve_handle,
 135   NUM_PRAGMA_HANDLERS
 136 };
 137
 138 /* Describes the various uses of a governing predicate.  */
 139 enum predication_index
 140 {
 141   /* No governing predicate is present.  */
 142   PRED_none,
 143
 144   /* A governing predicate is present but there is no predication suffix
 145      associated with it.  This is used when the result is neither a vector
 146      nor a predicate, since the distinction between "zeroing" and "merging"
 147      doesn't apply in that case.  It is also used when a suffix would be
 148      redundant (such as for loads and comparisons, which are inherently
 149      zeroing operations).  */
 150   PRED_implicit,
 151
 152   /* Merging predication: copy inactive lanes from the first data argument
 153      to the vector result.  */
 154   PRED_m,
 155
 156   /* "Don't care" predication: set inactive lanes of the vector result
 157      to arbitrary values.  */
 158   PRED_x,
 159
 160   /* Zero predication: set inactive lanes of the vector result to zero.  */
 161   PRED_z,
 162
 163   /* Merging predication for SME's ZA: merge into slices of the array
 164      instead of overwriting the whole slices.  */
 165   PRED_za_m,
 166
 167   NUM_PREDS
 168 };
 169
 170 /* Classifies intrinsics on whether they set the FPM register */
 171 enum fpm_mode_index
 172 {
 173   FPM_unused,
 174   FPM_set,
 175   NUM_FPM_MODES
 176 };
 177
 178 /* Classifies element types, based on type suffixes with the bit count
 179    removed.  "count" isn't really an element type, but we pretend it is
 180    for consistency.  */
 181 enum type_class_index
 182 {
 183   TYPE_bool,
 184   TYPE_bfloat,
 185   TYPE_count,
 186   TYPE_float,
 187   TYPE_mfloat,
 188   TYPE_signed,
 189   TYPE_unsigned,
 190   NUM_TYPE_CLASSES
 191 };
 192
 193 /* Classifies an operation into "modes"; for example, to distinguish
 194    vector-scalar operations from vector-vector operations, or to
 195    distinguish between different addressing modes.  This classification
 196    accounts for the function suffixes that occur between the base name
 197    and the first type suffix.  */
 198 enum mode_suffix_index
 199 {
 200 #define DEF_SVE_MODE(NAME, BASE, DISPLACEMENT, UNITS) MODE_##NAME,
 201 #include "aarch64-sve-builtins.def"
 202   MODE_none
 203 };
 204
 205 /* Enumerates the possible type suffixes.  Each suffix is associated with
 206    a vector type, but for predicates provides extra information about the
 207    element size.  */
 208 enum type_suffix_index
 209 {
 210 #define DEF_SVE_TYPE_SUFFIX(NAME, ACLE_TYPE, CLASS, BITS, MODE) \
 211   TYPE_SUFFIX_ ## NAME,
 212 #define DEF_SME_ZA_SUFFIX(NAME, BITS, MODE) \
 213   TYPE_SUFFIX_ ## NAME,
 214 #include "aarch64-sve-builtins.def"
 215   NUM_TYPE_SUFFIXES
 216 };
 217
 218 /* Enumerates the possible group suffixes.  Each suffix combines two
 219    optional pieces of information: the vector group size in a ZA index,
 220    and the number of vectors in the largest tuple argument.  */
 221 enum group_suffix_index
 222 {
 223 #define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) GROUP_##NAME,
 224 #include "aarch64-sve-builtins.def"
 225   GROUP_none,
 226   NUM_GROUP_SUFFIXES
 227 };
 228
 229 /* Combines two type suffixes.  */
 230 typedef enum type_suffix_index type_suffix_pair[2];
 231
 232 class function_base;
 233 class function_shape;
 234
 235 /* Static information about a mode suffix.  */
 236 struct mode_suffix_info
 237 {
 238   /* The suffix string itself.  */
 239   const char *string;
 240
 241   /* The type of the vector base address, or NUM_VECTOR_TYPES if the
 242      mode does not include a vector base address.  */
 243   vector_type_index base_vector_type;
 244
 245   /* The type of the vector displacement, or NUM_VECTOR_TYPES if the
 246      mode does not include a vector displacement.  (Note that scalar
 247      displacements are always int64_t.)  */
 248   vector_type_index displacement_vector_type;
 249
 250   /* The units in which the vector or scalar displacement is measured,
 251      or UNITS_none if the mode doesn't take a displacement.  */
 252   units_index displacement_units;
 253 };
 254
 255 #define ENTRY(E, M, Q, G) E,
 256 enum aarch64_simd_type
 257 {
 258 #include "aarch64-simd-builtin-types.def"
 259   ARM_NEON_H_TYPES_LAST
 260 };
 261 #undef ENTRY
 262
 263 /* Static information about a type suffix.  */
 264 struct type_suffix_info
 265 {
 266   /* The suffix string itself.  */
 267   const char *string;
 268
 269   /* The associated ACLE vector or predicate type.  */
 270   vector_type_index vector_type : 8;
 271
 272   /* What kind of type the suffix represents.  */
 273   type_class_index tclass : 8;
 274
 275   /* The number of bits and bytes in an element.  For predicates this
 276      measures the associated data elements.  */
 277   unsigned int element_bits : 8;
 278   unsigned int element_bytes : 8;
 279
 280   /* True if the suffix is for an integer type.  */
 281   unsigned int integer_p : 1;
 282   /* True if the suffix is for an unsigned type.  */
 283   unsigned int unsigned_p : 1;
 284   /* True if the suffix is for a floating-point type.  */
 285   unsigned int float_p : 1;
 286   /* True if the suffix is for a vector type (integer or float).  */
 287   unsigned int vector_p : 1;
 288   /* True if the suffix is for a boolean type.  */
 289   unsigned int bool_p : 1;
 290   /* True if the suffix is for SME's ZA.  */
 291   unsigned int za_p : 1;
 292   unsigned int spare : 10;
 293
 294   /* The associated vector or predicate mode.  */
 295   machine_mode vector_mode : 16;
 296
 297   /* The corresponding 64-bit and 128-bit arm_neon.h types, or
 298      ARM_NEON_H_TYPES_LAST if none.  */
 299   aarch64_simd_type neon64_type;
 300   aarch64_simd_type neon128_type;
 301 };
 302
 303 /* Static information about a group suffix.  */
 304 struct group_suffix_info
 305 {
 306   /* The suffix string itself.  */
 307   const char *string;
 308
 309   /* If the suffix describes a vector group in a ZA index, this is the
 310      size of that group, otherwise it is zero.  */
 311   unsigned int vg;
 312
 313   /* The number of vectors in the largest (or only) tuple argument,
 314      or 1 if the suffix does not convey this information.  */
 315   unsigned int vectors_per_tuple;
 316 };
 317
 318 /* Represents an SVE vector, predicate, tuple of vectors, or tuple of
 319    predicates.  There is also a representation of "no type"/"invalid type".  */
 320 struct sve_type
 321 {
 322   sve_type () = default;
 323   sve_type (type_suffix_index type) : type (type), num_vectors (1) {}
 324   sve_type (type_suffix_index type, unsigned int num_vectors)
 325     : type (type), num_vectors (num_vectors) {}
 326
 327   /* Return true if the type is valid.  */
 328   explicit operator bool () const { return type != NUM_TYPE_SUFFIXES; }
 329
 330   bool operator== (const sve_type &) const;
 331   bool operator!= (const sve_type &x) const { return !operator== (x); }
 332
 333   /* This is one of:
 334
 335      - TYPE_SUFFIX_b for svbool_t-based types
 336      - TYPE_SUFFIX_c for svcount_t-based types
 337      - the type suffix of a data element for SVE data vectors and tuples
 338      - NUM_TYPE_SUFFIXES for invalid types.  */
 339   type_suffix_index type = NUM_TYPE_SUFFIXES;
 340
 341   /* If the type is a tuple, this is the number of vectors in the tuple,
 342      otherwise it is 1.  */
 343   unsigned int num_vectors = 1;
 344 };
 345
 346 inline bool
 347 sve_type::operator== (const sve_type &other) const
 348 {
 349   return type == other.type && num_vectors == other.num_vectors;
 350 }
 351
 352 /* Static information about a set of functions.  */
 353 struct function_group_info
 354 {
 355   /* The base name, as a string.  */
 356   const char *base_name;
 357
 358   /* Describes the behavior associated with the function base name.  */
 359   const function_base *const *base;
 360
 361   /* The shape of the functions, as described above the class definition.
 362      It's possible to have entries with the same base name but different
 363      shapes.  */
 364   const function_shape *const *shape;
 365
 366   /* A list of the available type suffixes, group suffixes, and predication
 367      types.  The function supports every combination of the three.
 368
 369      The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES.
 370      It is lexicographically ordered based on the index value.
 371
 372      The list of group suffixes is terminated by NUM_GROUP_SUFFIXES
 373      and the list of predication types is terminated by NUM_PREDS.  */
 374   const type_suffix_pair *types;
 375   const group_suffix_index *groups;
 376   const predication_index *preds;
 377
 378   /* The architecture extensions that the functions require.  */
 379   aarch64_required_extensions required_extensions;
 380
 381   /* Whether the floating point register is set */
 382   fpm_mode_index fpm_mode;
 383 };
 384
 385 /* Describes a single fully-resolved function (i.e. one that has a
 386    unique full name).  */
 387 class GTY((user)) function_instance
 388 {
 389 public:
 390   function_instance (const char *, const function_base *,
 391                      const function_shape *, mode_suffix_index,
 392                      const type_suffix_pair &, group_suffix_index,
 393                      predication_index, fpm_mode_index);
 394
 395   bool operator== (const function_instance &) const;
 396   bool operator!= (const function_instance &) const;
 397   hashval_t hash () const;
 398
 399   unsigned int call_properties () const;
 400   bool reads_global_state_p () const;
 401   bool modifies_global_state_p () const;
 402   bool could_trap_p () const;
 403
 404   vector_type_index gp_type_index () const;
 405   tree gp_type () const;
 406
 407   unsigned int vectors_per_tuple () const;
 408   tree memory_scalar_type () const;
 409   machine_mode memory_vector_mode () const;
 410
 411   const mode_suffix_info &mode_suffix () const;
 412   tree base_vector_type () const;
 413   tree displacement_vector_type () const;
 414   units_index displacement_units () const;
 415
 416   unsigned int num_za_tiles () const;
 417
 418   const type_suffix_info &type_suffix (unsigned int) const;
 419   const group_suffix_info &group_suffix () const;
 420
 421   tree scalar_type (unsigned int) const;
 422   tree vector_type (unsigned int) const;
 423   tree tuple_type (unsigned int) const;
 424   unsigned int elements_per_vq (unsigned int) const;
 425   machine_mode vector_mode (unsigned int) const;
 426   machine_mode tuple_mode (unsigned int) const;
 427   machine_mode gp_mode (unsigned int) const;
 428
 429   /* The properties of the function.  */
 430   const char *base_name;
 431   const function_base *base;
 432   const function_shape *shape;
 433   mode_suffix_index mode_suffix_id;
 434   type_suffix_pair type_suffix_ids;
 435   group_suffix_index group_suffix_id;
 436   predication_index pred;
 437   fpm_mode_index fpm_mode;
 438 };
 439
 440 class registered_function;
 441
 442 /* A class for building and registering function decls.  */
 443 class function_builder
 444 {
 445 public:
 446   function_builder (handle_pragma_index, bool);
 447   ~function_builder ();
 448
 449   void add_unique_function (const function_instance &, tree,
 450                             vec<tree> &, aarch64_required_extensions, bool);
 451   void add_overloaded_function (const function_instance &,
 452                                 aarch64_required_extensions);
 453   void add_overloaded_functions (const function_group_info &,
 454                                  mode_suffix_index);
 455
 456   void register_function_group (const function_group_info &);
 457
 458 private:
 459   void append_name (const char *);
 460   char *finish_name ();
 461
 462   char *get_name (const function_instance &, bool);
 463
 464   tree get_attributes (const function_instance &, aarch64_required_extensions);
 465
 466   registered_function &add_function (const function_instance &,
 467                                      const char *, tree, tree,
 468                                      aarch64_required_extensions, bool, bool);
 469
 470   /* The function type to use for functions that are resolved by
 471      function_resolver.  */
 472   tree m_overload_type;
 473
 474   /* True if we should create a separate decl for each instance of an
 475      overloaded function, instead of using function_resolver.  */
 476   bool m_direct_overloads;
 477
 478   /* Used for building up function names.  */
 479   obstack m_string_obstack;
 480
 481   /* Used to store the index for the current function.  */
 482   unsigned int m_function_index;
 483
 484   /* Stores the mode of the current pragma handler.  */
 485   bool m_function_nulls;
 486 };
 487
 488 /* A base class for handling calls to built-in functions.  */
 489 class function_call_info : public function_instance
 490 {
 491 public:
 492   function_call_info (location_t, const function_instance &, tree);
 493
 494   bool function_returns_void_p ();
 495
 496   /* The location of the call.  */
 497   location_t location;
 498
 499   /* The FUNCTION_DECL that is being called.  */
 500   tree fndecl;
 501 };
 502
 503 /* A class for resolving an overloaded function call.  */
 504 class function_resolver : public function_call_info
 505 {
 506 public:
 507   enum target_type_restrictions { TARGET_ANY, TARGET_32_64 };
 508   enum { SAME_SIZE = 256, HALF_SIZE, QUARTER_SIZE };
 509   static const type_class_index SAME_TYPE_CLASS = NUM_TYPE_CLASSES;
 510
 511   function_resolver (location_t, const function_instance &, tree,
 512                      vec<tree, va_gc> &);
 513
 514   const char *get_scalar_type_name (type_suffix_index);
 515   tree get_argument_type (unsigned int);
 516   bool scalar_argument_p (unsigned int);
 517
 518   void report_incorrect_num_vectors (unsigned int, sve_type, unsigned int);
 519   void report_mismatched_num_vectors (unsigned int, sve_type,
 520                                       unsigned int, sve_type);
 521
 522   tree report_no_such_form (sve_type);
 523   tree lookup_form (mode_suffix_index,
 524                     type_suffix_index = NUM_TYPE_SUFFIXES,
 525                     type_suffix_index = NUM_TYPE_SUFFIXES,
 526                     group_suffix_index = GROUP_none);
 527   tree lookup_form (mode_suffix_index, sve_type);
 528   tree resolve_to (mode_suffix_index,
 529                    type_suffix_index = NUM_TYPE_SUFFIXES,
 530                    type_suffix_index = NUM_TYPE_SUFFIXES,
 531                    group_suffix_index = GROUP_none);
 532   tree resolve_to (mode_suffix_index, sve_type);
 533   tree resolve_conversion (mode_suffix_index, sve_type);
 534
 535   vector_type_index infer_predicate_type (unsigned int);
 536   type_suffix_index infer_integer_scalar_type (unsigned int);
 537   type_suffix_index infer_64bit_scalar_integer_pair (unsigned int);
 538   type_suffix_index infer_pointer_type (unsigned int, bool = false,
 539                                         target_type_restrictions = TARGET_ANY);
 540   sve_type infer_sve_type (unsigned int);
 541   sve_type infer_vector_or_tuple_type (unsigned int, unsigned int);
 542   type_suffix_index infer_vector_type (unsigned int);
 543   type_suffix_index infer_integer_vector_type (unsigned int);
 544   type_suffix_index infer_neon128_vector_type (unsigned int);
 545   type_suffix_index infer_unsigned_vector_type (unsigned int);
 546   type_suffix_index infer_sd_vector_type (unsigned int);
 547   sve_type infer_tuple_type (unsigned int);
 548
 549   bool require_vector_or_scalar_type (unsigned int);
 550
 551   bool require_matching_predicate_type (vector_type_index, sve_type);
 552   bool require_vector_type (unsigned int, vector_type_index);
 553   bool require_matching_vector_type (unsigned int, unsigned int, sve_type);
 554   bool require_derived_vector_type (unsigned int, unsigned int, sve_type,
 555                                     type_class_index = SAME_TYPE_CLASS,
 556                                     unsigned int = SAME_SIZE,
 557                                     unsigned int = 1);
 558
 559   bool require_scalar_type (unsigned int, const char *);
 560   bool require_nonscalar_type (unsigned int);
 561   bool require_pointer_type (unsigned int);
 562   bool require_matching_integer_scalar_type (unsigned int, unsigned int,
 563                                              type_suffix_index);
 564   bool require_derived_scalar_type (unsigned int, type_class_index,
 565                                     unsigned int = SAME_SIZE);
 566   bool require_matching_pointer_type (unsigned int, unsigned int,
 567                                       type_suffix_index);
 568   bool require_integer_immediate (unsigned int);
 569
 570   vector_type_index infer_vector_base_type (unsigned int);
 571   vector_type_index infer_vector_displacement_type (unsigned int);
 572
 573   mode_suffix_index resolve_sv_displacement (unsigned int,
 574                                              type_suffix_index, bool);
 575   mode_suffix_index resolve_gather_address (unsigned int,
 576                                             type_suffix_index, bool);
 577   mode_suffix_index resolve_adr_address (unsigned int);
 578
 579   bool check_num_arguments (unsigned int);
 580   bool check_gp_argument (unsigned int, unsigned int &, unsigned int &);
 581   tree resolve_unary (type_class_index = SAME_TYPE_CLASS,
 582                       unsigned int = SAME_SIZE, bool = false);
 583   tree resolve_uniform (unsigned int, unsigned int = 0);
 584   tree resolve_uniform_opt_n (unsigned int);
 585   tree finish_opt_n_resolution (unsigned int, unsigned int, type_suffix_index,
 586                                 type_class_index = SAME_TYPE_CLASS,
 587                                 unsigned int = SAME_SIZE,
 588                                 type_suffix_index = NUM_TYPE_SUFFIXES);
 589   tree finish_opt_single_resolution (unsigned int, unsigned int, sve_type,
 590                                      type_class_index = SAME_TYPE_CLASS);
 591
 592   tree resolve ();
 593
 594 private:
 595   /* The arguments to the overloaded function.  */
 596   vec<tree, va_gc> &m_arglist;
 597 };
 598
 599 /* A class for checking that the semantic constraints on a function call are
 600    satisfied, such as arguments being integer constant expressions with
 601    a particular range.  The parent class's FNDECL is the decl that was
 602    called in the original source, before overload resolution.  */
 603 class function_checker : public function_call_info
 604 {
 605 public:
 606   function_checker (location_t, const function_instance &, tree,
 607                     tree, unsigned int, tree *);
 608
 609   bool require_immediate_either_or (unsigned int, HOST_WIDE_INT,
 610                                     HOST_WIDE_INT);
 611   bool require_immediate_enum (unsigned int, tree);
 612   bool require_immediate_lane_index (unsigned int, unsigned int,
 613                                      unsigned int = 1);
 614   bool require_immediate_one_of (unsigned int, HOST_WIDE_INT, HOST_WIDE_INT,
 615                                  HOST_WIDE_INT, HOST_WIDE_INT);
 616   bool require_immediate_range (unsigned int, HOST_WIDE_INT, HOST_WIDE_INT);
 617
 618   bool check ();
 619
 620 private:
 621   bool argument_exists_p (unsigned int);
 622
 623   bool require_immediate (unsigned int, HOST_WIDE_INT &);
 624
 625   /* The type of the resolved function.  */
 626   tree m_fntype;
 627
 628   /* The arguments to the function.  */
 629   unsigned int m_nargs;
 630   tree *m_args;
 631
 632   /* The first argument not associated with the function's predication
 633      type.  */
 634   unsigned int m_base_arg;
 635 };
 636
 637 /* A class for folding a gimple function call.  */
 638 class gimple_folder : public function_call_info
 639 {
 640 public:
 641   gimple_folder (const function_instance &, tree,
 642                  gimple_stmt_iterator *, gcall *);
 643
 644   tree force_vector (gimple_seq &, tree, tree);
 645   tree convert_pred (gimple_seq &, tree, unsigned int);
 646   tree fold_contiguous_base (gimple_seq &, tree);
 647   tree load_store_cookie (tree);
 648
 649   gcall *redirect_call (const function_instance &);
 650   gimple *redirect_pred_x ();
 651   gimple *fold_pfalse ();
 652   gimple *convert_and_fold (tree, gimple *(*) (gimple_folder &,
 653                                                tree, vec<tree> &));
 654
 655   gimple *fold_to_cstu (poly_uint64);
 656   gimple *fold_to_pfalse ();
 657   gimple *fold_to_ptrue ();
 658   gimple *fold_to_vl_pred (unsigned int);
 659   gimple *fold_const_binary (enum tree_code);
 660   gimple *fold_active_lanes_to (tree);
 661   gimple *fold_call_to (tree);
 662   gimple *fold_to_stmt_vops (gimple *);
 663
 664   gimple *fold ();
 665
 666   /* Where to insert extra statements that feed the final replacement.  */
 667   gimple_stmt_iterator *gsi;
 668
 669   /* The call we're folding.  */
 670   gcall *call;
 671
 672   /* The result of the call, or null if none.  */
 673   tree lhs;
 674 };
 675
 676 /* A class for expanding a function call into RTL.  */
 677 class function_expander : public function_call_info
 678 {
 679 public:
 680   function_expander (const function_instance &, tree, tree, rtx);
 681   rtx expand ();
 682
 683   insn_code direct_optab_handler (optab, unsigned int = 0);
 684   insn_code direct_optab_handler_for_sign (optab, optab, unsigned int = 0,
 685                                            machine_mode = E_VOIDmode);
 686   insn_code convert_optab_handler_for_sign (optab, optab, unsigned int,
 687                                             machine_mode, machine_mode);
 688
 689   machine_mode result_mode () const;
 690
 691   bool overlaps_input_p (rtx);
 692
 693   rtx convert_to_pmode (rtx);
 694   rtx get_contiguous_base (machine_mode, unsigned int = 1, unsigned int = 2,
 695                            aarch64_feature_flags = 0);
 696   rtx get_fallback_value (machine_mode, unsigned int,
 697                           unsigned int, unsigned int &);
 698   rtx get_reg_target ();
 699   rtx get_nonoverlapping_reg_target ();
 700
 701   void add_output_operand (insn_code);
 702   void add_input_operand (insn_code, rtx);
 703   void add_integer_operand (poly_int64);
 704   void add_mem_operand (machine_mode, rtx);
 705   void add_address_operand (rtx);
 706   void add_fixed_operand (rtx);
 707   rtx generate_insn (insn_code);
 708
 709   void prepare_gather_address_operands (unsigned int, bool = true);
 710   void prepare_prefetch_operands ();
 711   void add_ptrue_hint (unsigned int, machine_mode);
 712   void rotate_inputs_left (unsigned int, unsigned int);
 713   bool try_negating_argument (unsigned int, machine_mode);
 714
 715   rtx use_exact_insn (insn_code);
 716   rtx use_unpred_insn (insn_code);
 717   rtx use_pred_x_insn (insn_code);
 718   rtx use_cond_insn (insn_code, unsigned int = DEFAULT_MERGE_ARGNO);
 719   rtx use_vcond_mask_insn (insn_code, unsigned int = DEFAULT_MERGE_ARGNO);
 720   rtx use_contiguous_load_insn (insn_code, bool = false);
 721   rtx use_contiguous_prefetch_insn (insn_code);
 722   rtx use_contiguous_store_insn (insn_code);
 723
 724   rtx map_to_rtx_codes (rtx_code, rtx_code, int, int,
 725                         unsigned int = DEFAULT_MERGE_ARGNO);
 726   rtx map_to_unspecs (int, int, int, unsigned int = DEFAULT_MERGE_ARGNO);
 727
 728   /* The function call expression.  */
 729   tree call_expr;
 730
 731   /* For functions that return a value, this is the preferred location
 732      of that value.  It could be null or could have a different mode
 733      from the function return type.  */
 734   rtx possible_target;
 735
 736   /* The expanded arguments.  */
 737   auto_vec<rtx, 16> args;
 738
 739 private:
 740   /* Used to build up the operands to an instruction.  */
 741   auto_vec<expand_operand, 8> m_ops;
 742 };
 743
 744 /* Provides information about a particular function base name, and handles
 745    tasks related to the base name.  */
 746 class function_base
 747 {
 748 public:
 749   /* Return a set of CP_* flags that describe what the function might do,
 750      in addition to reading its arguments and returning a result.  */
 751   virtual unsigned int call_properties (const function_instance &) const;
 752
 753   /* If the function operates on tuples of vectors, return the number
 754      of vectors in the tuples, otherwise return 1.  */
 755   virtual unsigned int vectors_per_tuple (const function_instance &) const;
 756
 757   /* If the function addresses memory, return the type of a single
 758      scalar memory element.  */
 759   virtual tree
 760   memory_scalar_type (const function_instance &) const
 761   {
 762     gcc_unreachable ();
 763   }
 764
 765   /* If the function addresses memory, return a vector mode whose
 766      GET_MODE_NUNITS is the number of elements addressed and whose
 767      GET_MODE_INNER is the mode of a single scalar memory element.  */
 768   virtual machine_mode
 769   memory_vector_mode (const function_instance &) const
 770   {
 771     gcc_unreachable ();
 772   }
 773
 774   /* Try to fold the given gimple call.  Return the new gimple statement
 775      on success, otherwise return null.  */
 776   virtual gimple *fold (gimple_folder &) const { return NULL; }
 777
 778   /* Expand the given call into rtl.  Return the result of the function,
 779      or an arbitrary value if the function doesn't return a result.  */
 780   virtual rtx expand (function_expander &) const = 0;
 781 };
 782
 783 /* Classifies functions into "shapes".  The idea is to take all the
 784    type signatures for a set of functions, remove the governing predicate
 785    (if any), and classify what's left based on:
 786
 787    - the number of arguments
 788
 789    - the process of determining the types in the signature from the mode
 790      and type suffixes in the function name (including types that are not
 791      affected by the suffixes)
 792
 793    - which arguments must be integer constant expressions, and what range
 794      those arguments have
 795
 796    - the process for mapping overloaded names to "full" names.  */
 797 class function_shape
 798 {
 799 public:
 800   virtual bool has_merge_argument_p (const function_instance &,
 801                                      unsigned int) const;
 802
 803   virtual bool explicit_type_suffix_p (unsigned int) const = 0;
 804
 805   /* True if the group suffix is present in overloaded names.
 806      This isn't meaningful for pre-SME intrinsics, and true is
 807      more common than false, so provide a default definition.  */
 808   virtual bool explicit_group_suffix_p () const { return true; }
 809
 810   virtual type_suffix_index vector_base_type (type_suffix_index) const;
 811
 812   /* Define all functions associated with the given group.  */
 813   virtual void build (function_builder &,
 814                       const function_group_info &) const = 0;
 815
 816   /* Try to resolve the overloaded call.  Return the non-overloaded
 817      function decl on success and error_mark_node on failure.  */
 818   virtual tree resolve (function_resolver &) const = 0;
 819
 820   /* Check whether the given call is semantically valid.  Return true
 821      if it is, otherwise report an error and return false.  */
 822   virtual bool check (function_checker &) const { return true; }
 823 };
 824
 825 /* RAII class for enabling enough SVE features to define the built-in
 826    types and implement the arm_sve.h pragma.  */
 827 class sve_switcher : public aarch64_simd_switcher
 828 {
 829 public:
 830   sve_switcher (aarch64_feature_flags = 0);
 831   ~sve_switcher ();
 832
 833 private:
 834   unsigned int m_old_maximum_field_alignment;
 835   bool m_old_have_regs_of_mode[MAX_MACHINE_MODE];
 836 };
 837
 838 /* Extends sve_switch enough for defining arm_sme.h.  */
 839 class sme_switcher : public sve_switcher
 840 {
 841 public:
 842   sme_switcher () : sve_switcher (AARCH64_FL_SME) {}
 843 };
 844
 845 extern const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1];
 846 extern const mode_suffix_info mode_suffixes[MODE_none + 1];
 847 extern const group_suffix_info group_suffixes[NUM_GROUP_SUFFIXES];
 848
 849 extern tree scalar_types[NUM_VECTOR_TYPES + 1];
 850 extern tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1];
 851 extern tree acle_svpattern;
 852 extern tree acle_svprfop;
 853
 854 bool vector_cst_all_same (tree, unsigned int);
 855 bool is_ptrue (tree, unsigned int);
 856 bool is_pfalse (tree);
 857 const function_instance *lookup_fndecl (tree);
 858
 859 /* Try to find a mode with the given mode_suffix_info fields.  Return the
 860    mode on success or MODE_none on failure.  */
 861 inline mode_suffix_index
 862 find_mode_suffix (vector_type_index base_vector_type,
 863                   vector_type_index displacement_vector_type,
 864                   units_index displacement_units)
 865 {
 866   for (unsigned int mode_i = 0; mode_i < ARRAY_SIZE (mode_suffixes); ++mode_i)
 867     {
 868       const mode_suffix_info &mode = mode_suffixes[mode_i];
 869       if (mode.base_vector_type == base_vector_type
 870           && mode.displacement_vector_type == displacement_vector_type
 871           && mode.displacement_units == displacement_units)
 872         return mode_suffix_index (mode_i);
 873     }
 874   return MODE_none;
 875 }
 876
 877 /* Return the type suffix associated with ELEMENT_BITS-bit elements of type
 878    class TCLASS.  */
 879 inline type_suffix_index
 880 find_type_suffix (type_class_index tclass, unsigned int element_bits)
 881 {
 882   for (unsigned int i = 0; i < NUM_TYPE_SUFFIXES; ++i)
 883     if (type_suffixes[i].tclass == tclass
 884         && type_suffixes[i].element_bits == element_bits)
 885       return type_suffix_index (i);
 886   gcc_unreachable ();
 887 }
 888
 889 /* Return the single field in tuple type TYPE.  */
 890 inline tree
 891 tuple_type_field (tree type)
 892 {
 893   for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
 894     if (TREE_CODE (field) == FIELD_DECL)
 895       return field;
 896   gcc_unreachable ();
 897 }
 898
 899 /* Return the vector type associated with TYPE.  */
 900 inline tree
 901 get_vector_type (sve_type type)
 902 {
 903   auto vector_type = type_suffixes[type.type].vector_type;
 904   return acle_vector_types[type.num_vectors - 1][vector_type];
 905 }
 906
 907 inline function_instance::
 908 function_instance (const char *base_name_in, const function_base *base_in,
 909                    const function_shape *shape_in,
 910                    mode_suffix_index mode_suffix_id_in,
 911                    const type_suffix_pair &type_suffix_ids_in,
 912                    group_suffix_index group_suffix_id_in,
 913                    predication_index pred_in, fpm_mode_index fpm_mode_in)
 914   : base_name (base_name_in), base (base_in), shape (shape_in),
 915     mode_suffix_id (mode_suffix_id_in), group_suffix_id (group_suffix_id_in),
 916     pred (pred_in), fpm_mode (fpm_mode_in)
 917 {
 918   memcpy (type_suffix_ids, type_suffix_ids_in, sizeof (type_suffix_ids));
 919 }
 920
 921 inline bool
 922 function_instance::operator== (const function_instance &other) const
 923 {
 924   return (base == other.base
 925           && shape == other.shape
 926           && mode_suffix_id == other.mode_suffix_id
 927           && type_suffix_ids[0] == other.type_suffix_ids[0]
 928           && type_suffix_ids[1] == other.type_suffix_ids[1]
 929           && group_suffix_id == other.group_suffix_id
 930           && pred == other.pred
 931           && fpm_mode == other.fpm_mode);
 932 }
 933
 934 inline bool
 935 function_instance::operator!= (const function_instance &other) const
 936 {
 937   return !operator== (other);
 938 }
 939
 940 /* Return the index of the type that should be used as the governing
 941    predicate of this function.  */
 942 inline vector_type_index
 943 function_instance::gp_type_index () const
 944 {
 945   if (group_suffix ().vectors_per_tuple > 1)
 946     return VECTOR_TYPE_svcount_t;
 947   return VECTOR_TYPE_svbool_t;
 948 }
 949
 950 /* Return the type that should be used as the governing predicate of
 951    this function.  */
 952 inline tree
 953 function_instance::gp_type () const
 954 {
 955   return acle_vector_types[0][gp_type_index ()];
 956 }
 957
 958 /* If the function operates on tuples of vectors, return the number
 959    of vectors in the tuples, otherwise return 1.  */
 960 inline unsigned int
 961 function_instance::vectors_per_tuple () const
 962 {
 963   return base->vectors_per_tuple (*this);
 964 }
 965
 966 /* If the function addresses memory, return the type of a single
 967    scalar memory element.  */
 968 inline tree
 969 function_instance::memory_scalar_type () const
 970 {
 971   return base->memory_scalar_type (*this);
 972 }
 973
 974 /* If the function addresses memory, return a vector mode whose
 975    GET_MODE_NUNITS is the number of elements addressed and whose
 976    GET_MODE_INNER is the mode of a single scalar memory element.  */
 977 inline machine_mode
 978 function_instance::memory_vector_mode () const
 979 {
 980   return base->memory_vector_mode (*this);
 981 }
 982
 983 /* Return information about the function's mode suffix.  */
 984 inline const mode_suffix_info &
 985 function_instance::mode_suffix () const
 986 {
 987   return mode_suffixes[mode_suffix_id];
 988 }
 989
 990 /* Return the type of the function's vector base address argument,
 991    or null it doesn't have a vector base address.  */
 992 inline tree
 993 function_instance::base_vector_type () const
 994 {
 995   return acle_vector_types[0][mode_suffix ().base_vector_type];
 996 }
 997
 998 /* Return the type of the function's vector index or offset argument,
 999    or null if doesn't have a vector index or offset argument.  */
1000 inline tree
1001 function_instance::displacement_vector_type () const
1002 {
1003   return acle_vector_types[0][mode_suffix ().displacement_vector_type];
1004 }
1005
1006 /* Return the number of ZA tiles associated with the _za<N> suffix
1007    (which is always the first type suffix).  */
1008 inline unsigned int
1009 function_instance::num_za_tiles () const
1010 {
1011   auto &suffix = type_suffix (0);
1012   gcc_checking_assert (suffix.za_p);
1013   return suffix.element_bytes;
1014 }
1015
1016 /* If the function takes a vector or scalar displacement, return the units
1017    in which the displacement is measured, otherwise return UNITS_none.  */
1018 inline units_index
1019 function_instance::displacement_units () const
1020 {
1021   return mode_suffix ().displacement_units;
1022 }
1023
1024 /* Return information about type suffix I.  */
1025 inline const type_suffix_info &
1026 function_instance::type_suffix (unsigned int i) const
1027 {
1028   return type_suffixes[type_suffix_ids[i]];
1029 }
1030
1031 /* Return information about the function's group suffix.  */
1032 inline const group_suffix_info &
1033 function_instance::group_suffix () const
1034 {
1035   return group_suffixes[group_suffix_id];
1036 }
1037
1038 /* Return the scalar type associated with type suffix I.  */
1039 inline tree
1040 function_instance::scalar_type (unsigned int i) const
1041 {
1042   return scalar_types[type_suffix (i).vector_type];
1043 }
1044
1045 /* Return the vector type associated with type suffix I.  */
1046 inline tree
1047 function_instance::vector_type (unsigned int i) const
1048 {
1049   return acle_vector_types[0][type_suffix (i).vector_type];
1050 }
1051
1052 /* If the function operates on tuples of vectors, return the tuple type
1053    associated with type suffix I, otherwise return the vector type associated
1054    with type suffix I.  */
1055 inline tree
1056 function_instance::tuple_type (unsigned int i) const
1057 {
1058   unsigned int num_vectors = vectors_per_tuple ();
1059   return acle_vector_types[num_vectors - 1][type_suffix (i).vector_type];
1060 }
1061
1062 /* Return the number of elements of type suffix I that fit within a
1063    128-bit block.  */
1064 inline unsigned int
1065 function_instance::elements_per_vq (unsigned int i) const
1066 {
1067   return 128 / type_suffix (i).element_bits;
1068 }
1069
1070 /* Return the vector or predicate mode associated with type suffix I.  */
1071 inline machine_mode
1072 function_instance::vector_mode (unsigned int i) const
1073 {
1074   return type_suffix (i).vector_mode;
1075 }
1076
1077 /* Return the mode of tuple_type (I).  */
1078 inline machine_mode
1079 function_instance::tuple_mode (unsigned int i) const
1080 {
1081   if (group_suffix ().vectors_per_tuple > 1)
1082     return TYPE_MODE (tuple_type (i));
1083   return vector_mode (i);
1084 }
1085
1086 /* Return the mode of the governing predicate to use when operating on
1087    type suffix I.  */
1088 inline machine_mode
1089 function_instance::gp_mode (unsigned int i) const
1090 {
1091   /* Multi-vector operations are predicated on an svcount_t, which has
1092      mode VNx16BI.  */
1093   if (group_suffix ().vectors_per_tuple > 1)
1094     return VNx16BImode;
1095   return aarch64_sve_pred_mode (type_suffix (i).element_bytes).require ();
1096 }
1097
1098 /* Return true if the function has no return value.  */
1099 inline bool
1100 function_call_info::function_returns_void_p ()
1101 {
1102   return TREE_TYPE (TREE_TYPE (fndecl)) == void_type_node;
1103 }
1104
1105 /* Default implementation of function::call_properties, with conservatively
1106    correct behavior for floating-point instructions.  */
1107 inline unsigned int
1108 function_base::call_properties (const function_instance &instance) const
1109 {
1110   unsigned int flags = 0;
1111   if (instance.type_suffix (0).float_p || instance.type_suffix (1).float_p)
1112     flags |= CP_READ_FPCR | CP_RAISE_FP_EXCEPTIONS;
1113   return flags;
1114 }
1115
1116 inline unsigned int
1117 function_base::vectors_per_tuple (const function_instance &instance) const
1118 {
1119   return instance.group_suffix ().vectors_per_tuple;
1120 }
1121
1122 /* Return true if INSTANCE (which has NARGS arguments) has an initial
1123    vector argument whose only purpose is to specify the values of
1124    inactive lanes.  */
1125 inline bool
1126 function_shape::has_merge_argument_p (const function_instance &instance,
1127                                       unsigned int nargs) const
1128 {
1129   return nargs == 1 && instance.pred == PRED_m;
1130 }
1131
1132 /* Return the mode of the result of a call.  */
1133 inline machine_mode
1134 function_expander::result_mode () const
1135 {
1136   return TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl)));
1137 }
1138
1139 }
1140
1141 #endif