gcc/config/aarch64/aarch64-simd-builtins.def

   1 /* Machine description for AArch64 architecture.
   2    Copyright (C) 2012-2025 Free Software Foundation, Inc.
   3    Contributed by ARM Ltd.
   4
   5    This file is part of GCC.
   6
   7    GCC is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    GCC is distributed in the hope that it will be useful, but
  13    WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15    General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with GCC; see the file COPYING3.  If not see
  19    <http://www.gnu.org/licenses/>.  */
  20
  21 /* In the list below, the BUILTIN_<ITERATOR> macros expand to create
  22    builtins for each of the modes described by <ITERATOR>.  When adding
  23    new builtins to this list, a helpful idiom to follow is to add
  24    a line for each pattern in the md file.  Thus, ADDP, which has one
  25    pattern defined for the VD_BHSI iterator, and one for DImode, has two
  26    entries below.
  27
  28    Parameter 1 is the 'type' of the intrinsic.  This is used to
  29    describe the type modifiers (for example; unsigned) applied to
  30    each of the parameters to the intrinsic function.
  31
  32    Parameter 2 is the name of the intrinsic.  This is appended
  33    to `__builtin_aarch64_<name><mode>` to give the intrinsic name
  34    as exported to the front-ends.
  35
  36    Parameter 3 describes how to map from the name to the CODE_FOR_
  37    macro holding the RTL pattern for the intrinsic.  This mapping is:
  38    0 - CODE_FOR_aarch64_<name><mode>
  39    1-9 - CODE_FOR_<name><mode><1-9>
  40    10 - CODE_FOR_<name><mode>.
  41
  42    Parameter 4 is the 'flag' of the intrinsic.  This is used to
  43    help describe the attributes (for example, pure) for the intrinsic
  44    function.  */
  45
  46   BUILTIN_V12DIF (LOADSTRUCT_LANE, vec_ldap1_lane, 0, ALL)
  47   BUILTIN_V12DI (LOADSTRUCT_LANE_U, vec_ldap1_lane, 0, ALL)
  48   BUILTIN_V12DI (LOADSTRUCT_LANE_P, vec_ldap1_lane, 0, ALL)
  49   BUILTIN_V12DIF (STORESTRUCT_LANE, vec_stl1_lane, 0, ALL)
  50   BUILTIN_V12DI (STORESTRUCT_LANE_U, vec_stl1_lane, 0, ALL)
  51   BUILTIN_V12DI (STORESTRUCT_LANE_P, vec_stl1_lane, 0, ALL)
  52
  53   BUILTIN_VDC (BINOP, combine, 0, QUIET)
  54   BUILTIN_VD_I (BINOPU, combine, 0, DEFAULT)
  55   BUILTIN_VDC_P (BINOPP, combine, 0, DEFAULT)
  56   BUILTIN_VB (BINOPP, pmul, 0, DEFAULT)
  57   VAR1 (BINOPP, pmull, 0, DEFAULT, v8qi)
  58   VAR1 (BINOPP, pmull_hi, 0, DEFAULT, v16qi)
  59   BUILTIN_VHSDF_HSDF (BINOP, fmulx, 0, FP)
  60   BUILTIN_VHSDF_DF (UNOP, sqrt, 2, FP)
  61   BUILTIN_VDQ_I (BINOP, addp, 0, DEFAULT)
  62   BUILTIN_VDQ_I (BINOPU, addp, 0, DEFAULT)
  63   BUILTIN_VDQ_BHSI (UNOP, clrsb, 2, DEFAULT)
  64   BUILTIN_VDQ_BHSI (UNOP, clz, 2, DEFAULT)
  65   BUILTIN_VS (UNOP, ctz, 2, DEFAULT)
  66   BUILTIN_VB (UNOP, popcount, 2, DEFAULT)
  67
  68   /* Implemented by aarch64_<sur>q<r>shl<mode>.  */
  69   BUILTIN_VSDQ_I (BINOP, sqshl, 0, DEFAULT)
  70   BUILTIN_VSDQ_I (BINOP_UUS, uqshl, 0, DEFAULT)
  71   BUILTIN_VSDQ_I (BINOP, sqrshl, 0, DEFAULT)
  72   BUILTIN_VSDQ_I (BINOP_UUS, uqrshl, 0, DEFAULT)
  73   /* Implemented by aarch64_<su_optab><optab><mode>.  */
  74   BUILTIN_VSDQ_I (BINOP, ssadd, 3, DEFAULT)
  75   BUILTIN_VSDQ_I (BINOPU, usadd, 3, DEFAULT)
  76   BUILTIN_VSDQ_I (BINOP, sssub, 3, DEFAULT)
  77   BUILTIN_VSDQ_I (BINOPU, ussub, 3, DEFAULT)
  78   /* Implemented by aarch64_<sur>qadd<mode>.  */
  79   BUILTIN_VSDQ_I (BINOP_SSU, suqadd, 0, DEFAULT)
  80   BUILTIN_VSDQ_I (BINOP_UUS, usqadd, 0, DEFAULT)
  81
  82   /* Implemented by aarch64_ld1x2<vstruct_elt>. */
  83   BUILTIN_VALLDIF (LOADSTRUCT, ld1x2, 0, LOAD)
  84   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld1x2, 0, LOAD)
  85   BUILTIN_VALLP (LOADSTRUCT_P, ld1x2, 0, LOAD)
  86   /* Implemented by aarch64_ld1x3<vstruct_elt>.  */
  87   BUILTIN_VALLDIF (LOADSTRUCT, ld1x3, 0, LOAD)
  88   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld1x3, 0, LOAD)
  89   BUILTIN_VALLP (LOADSTRUCT_P, ld1x3, 0, LOAD)
  90   /* Implemented by aarch64_ld1x4<vstruct_elt>.  */
  91   BUILTIN_VALLDIF (LOADSTRUCT, ld1x4, 0, LOAD)
  92   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld1x4, 0, LOAD)
  93   BUILTIN_VALLP (LOADSTRUCT_P, ld1x4, 0, LOAD)
  94
  95   /* Implemented by aarch64_st1x2<vstruct_elt>.  */
  96   BUILTIN_VALLDIF (STORESTRUCT, st1x2, 0, STORE)
  97   BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st1x2, 0, STORE)
  98   BUILTIN_VALLP (STORESTRUCT_P, st1x2, 0, STORE)
  99   /* Implemented by aarch64_st1x3<vstruct_elt>.  */
 100   BUILTIN_VALLDIF (STORESTRUCT, st1x3, 0, STORE)
 101   BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st1x3, 0, STORE)
 102   BUILTIN_VALLP (STORESTRUCT_P, st1x3, 0, STORE)
 103   /* Implemented by aarch64_st1x4<vstruct_elt>.  */
 104   BUILTIN_VALLDIF (STORESTRUCT, st1x4, 0, STORE)
 105   BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st1x4, 0, STORE)
 106   BUILTIN_VALLP (STORESTRUCT_P, st1x4, 0, STORE)
 107
 108   /* Implemented by aarch64_ld<nregs><vstruct_elt>.  */
 109   BUILTIN_VALLDIF (LOADSTRUCT, ld2, 0, LOAD)
 110   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld2, 0, LOAD)
 111   BUILTIN_VALLP (LOADSTRUCT_P, ld2, 0, LOAD)
 112   BUILTIN_VALLDIF (LOADSTRUCT, ld3, 0, LOAD)
 113   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld3, 0, LOAD)
 114   BUILTIN_VALLP (LOADSTRUCT_P, ld3, 0, LOAD)
 115   BUILTIN_VALLDIF (LOADSTRUCT, ld4, 0, LOAD)
 116   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld4, 0, LOAD)
 117   BUILTIN_VALLP (LOADSTRUCT_P, ld4, 0, LOAD)
 118
 119   /* Implemented by aarch64_st<nregs><vstruct_elt>.  */
 120   BUILTIN_VALLDIF (STORESTRUCT, st2, 0, STORE)
 121   BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st2, 0, STORE)
 122   BUILTIN_VALLP (STORESTRUCT_P, st2, 0, STORE)
 123   BUILTIN_VALLDIF (STORESTRUCT, st3, 0, STORE)
 124   BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st3, 0, STORE)
 125   BUILTIN_VALLP (STORESTRUCT_P, st3, 0, STORE)
 126   BUILTIN_VALLDIF (STORESTRUCT, st4, 0, STORE)
 127   BUILTIN_VSDQ_I_DI (STORESTRUCT_U, st4, 0, STORE)
 128   BUILTIN_VALLP (STORESTRUCT_P, st4, 0, STORE)
 129
 130   /* Implemented by aarch64_ld<nregs>r<vstruct_elt>.  */
 131   BUILTIN_VALLDIF (LOADSTRUCT, ld2r, 0, LOAD)
 132   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld2r, 0, LOAD)
 133   BUILTIN_VALLP (LOADSTRUCT_P, ld2r, 0, LOAD)
 134   BUILTIN_VALLDIF (LOADSTRUCT, ld3r, 0, LOAD)
 135   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld3r, 0, LOAD)
 136   BUILTIN_VALLP (LOADSTRUCT_P, ld3r, 0, LOAD)
 137   BUILTIN_VALLDIF (LOADSTRUCT, ld4r, 0, LOAD)
 138   BUILTIN_VSDQ_I_DI (LOADSTRUCT_U, ld4r, 0, LOAD)
 139   BUILTIN_VALLP (LOADSTRUCT_P, ld4r, 0, LOAD)
 140
 141   /* Implemented by aarch64_ld<nregs>_lane<vstruct_elt>.  */
 142   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld2_lane, 0, ALL)
 143   BUILTIN_VSDQ_I_DI (LOADSTRUCT_LANE_U, ld2_lane, 0, ALL)
 144   BUILTIN_VALLP (LOADSTRUCT_LANE_P, ld2_lane, 0, ALL)
 145   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld3_lane, 0, ALL)
 146   BUILTIN_VSDQ_I_DI (LOADSTRUCT_LANE_U, ld3_lane, 0, ALL)
 147   BUILTIN_VALLP (LOADSTRUCT_LANE_P, ld3_lane, 0, ALL)
 148   BUILTIN_VALLDIF (LOADSTRUCT_LANE, ld4_lane, 0, ALL)
 149   BUILTIN_VSDQ_I_DI (LOADSTRUCT_LANE_U, ld4_lane, 0, ALL)
 150   BUILTIN_VALLP (LOADSTRUCT_LANE_P, ld4_lane, 0, ALL)
 151
 152   /* Implemented by aarch64_st<nregs>_lane<vstruct_elt>.  */
 153   BUILTIN_VALLDIF (STORESTRUCT_LANE, st2_lane, 0, ALL)
 154   BUILTIN_VSDQ_I_DI (STORESTRUCT_LANE_U, st2_lane, 0, ALL)
 155   BUILTIN_VALLP (STORESTRUCT_LANE_P, st2_lane, 0, ALL)
 156   BUILTIN_VALLDIF (STORESTRUCT_LANE, st3_lane, 0, ALL)
 157   BUILTIN_VSDQ_I_DI (STORESTRUCT_LANE_U, st3_lane, 0, ALL)
 158   BUILTIN_VALLP (STORESTRUCT_LANE_P, st3_lane, 0, ALL)
 159   BUILTIN_VALLDIF (STORESTRUCT_LANE, st4_lane, 0, ALL)
 160   BUILTIN_VSDQ_I_DI (STORESTRUCT_LANE_U, st4_lane, 0, ALL)
 161   BUILTIN_VALLP (STORESTRUCT_LANE_P, st4_lane, 0, ALL)
 162
 163   BUILTIN_VQW (BINOP, saddl2, 0, DEFAULT)
 164   BUILTIN_VQW (BINOPU, uaddl2, 0, DEFAULT)
 165   BUILTIN_VQW (BINOP, ssubl2, 0, DEFAULT)
 166   BUILTIN_VQW (BINOPU, usubl2, 0, DEFAULT)
 167   BUILTIN_VQW (BINOP, saddw2, 0, DEFAULT)
 168   BUILTIN_VQW (BINOPU, uaddw2, 0, DEFAULT)
 169   BUILTIN_VQW (BINOP, ssubw2, 0, DEFAULT)
 170   BUILTIN_VQW (BINOPU, usubw2, 0, DEFAULT)
 171   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>l<mode>.  */
 172   BUILTIN_VD_BHSI (BINOP, saddl, 0, DEFAULT)
 173   BUILTIN_VD_BHSI (BINOPU, uaddl, 0, DEFAULT)
 174   BUILTIN_VD_BHSI (BINOP, ssubl, 0, DEFAULT)
 175   BUILTIN_VD_BHSI (BINOPU, usubl, 0, DEFAULT)
 176   /* Implemented by aarch64_<ANY_EXTEND:su><ADDSUB:optab>w<mode>.  */
 177   BUILTIN_VD_BHSI (BINOP, saddw, 0, DEFAULT)
 178   BUILTIN_VD_BHSI (BINOPU, uaddw, 0, DEFAULT)
 179   BUILTIN_VD_BHSI (BINOP, ssubw, 0, DEFAULT)
 180   BUILTIN_VD_BHSI (BINOPU, usubw, 0, DEFAULT)
 181   /* Implemented by aarch64_<sur>h<addsub><mode>.  */
 182   BUILTIN_VDQ_BHSI (BINOP, shadd, 0, DEFAULT)
 183   BUILTIN_VDQ_BHSI (BINOP, shsub, 0, DEFAULT)
 184   BUILTIN_VDQ_BHSI (BINOPU, uhadd, 0, DEFAULT)
 185   BUILTIN_VDQ_BHSI (BINOPU, uhsub, 0, DEFAULT)
 186   BUILTIN_VDQ_BHSI (BINOP, srhadd, 0, DEFAULT)
 187   BUILTIN_VDQ_BHSI (BINOPU, urhadd, 0, DEFAULT)
 188
 189   /* Implemented by aarch64_<su>addlp<mode>.  */
 190   BUILTIN_VDQV_L (UNOP, saddlp, 0, DEFAULT)
 191   BUILTIN_VDQV_L (UNOPU, uaddlp, 0, DEFAULT)
 192
 193   /* Implemented by aarch64_<su>addlv<mode>.  */
 194   BUILTIN_VDQV_L (UNOP, saddlv, 0, DEFAULT)
 195   BUILTIN_VDQV_L (UNOPU, uaddlv, 0, DEFAULT)
 196
 197   /* Implemented by aarch64_<su>abd<mode>.  */
 198   BUILTIN_VDQ_BHSI (BINOP, sabd, 0, DEFAULT)
 199   BUILTIN_VDQ_BHSI (BINOPU, uabd, 0, DEFAULT)
 200
 201   /* Implemented by aarch64_<su>aba<mode>.  */
 202   BUILTIN_VDQ_BHSI (TERNOP, saba, 0, DEFAULT)
 203   BUILTIN_VDQ_BHSI (TERNOPU, uaba, 0, DEFAULT)
 204
 205   BUILTIN_VDQV_L (BINOP, sadalp, 0, DEFAULT)
 206   BUILTIN_VDQV_L (BINOPU, uadalp, 0, DEFAULT)
 207
 208   /* Implemented by aarch64_<sur>abal<mode>.  */
 209   BUILTIN_VD_BHSI (TERNOP, sabal, 0, DEFAULT)
 210   BUILTIN_VD_BHSI (TERNOPU, uabal, 0, DEFAULT)
 211
 212   /* Implemented by aarch64_<sur>abal2<mode>.  */
 213   BUILTIN_VQW (TERNOP, sabal2, 0, DEFAULT)
 214   BUILTIN_VQW (TERNOPU, uabal2, 0, DEFAULT)
 215
 216   /* Implemented by aarch64_<sur>abdl<mode>.  */
 217   BUILTIN_VD_BHSI (BINOP, sabdl, 0, DEFAULT)
 218   BUILTIN_VD_BHSI (BINOPU, uabdl, 0, DEFAULT)
 219
 220   /* Implemented by aarch64_<sur>abdl2<mode>.  */
 221   BUILTIN_VQW (BINOP, sabdl2, 0, DEFAULT)
 222   BUILTIN_VQW (BINOPU, uabdl2, 0, DEFAULT)
 223
 224   /* Implemented by aarch64_<sur><addsub>hn<mode>.  */
 225   BUILTIN_VQN (BINOP, addhn, 0, DEFAULT)
 226   BUILTIN_VQN (BINOPU, addhn, 0, DEFAULT)
 227   BUILTIN_VQN (BINOP, subhn, 0, DEFAULT)
 228   BUILTIN_VQN (BINOPU, subhn, 0, DEFAULT)
 229   BUILTIN_VQN (BINOP, raddhn, 0, DEFAULT)
 230   BUILTIN_VQN (BINOPU, raddhn, 0, DEFAULT)
 231   BUILTIN_VQN (BINOP, rsubhn, 0, DEFAULT)
 232   BUILTIN_VQN (BINOPU, rsubhn, 0, DEFAULT)
 233   /* Implemented by aarch64_<sur><addsub>hn2<mode>.  */
 234   BUILTIN_VQN (TERNOP, addhn2, 0, DEFAULT)
 235   BUILTIN_VQN (TERNOPU, addhn2, 0, DEFAULT)
 236   BUILTIN_VQN (TERNOP, subhn2, 0, DEFAULT)
 237   BUILTIN_VQN (TERNOPU, subhn2, 0, DEFAULT)
 238   BUILTIN_VQN (TERNOP, raddhn2, 0, DEFAULT)
 239   BUILTIN_VQN (TERNOPU, raddhn2, 0, DEFAULT)
 240   BUILTIN_VQN (TERNOP, rsubhn2, 0, DEFAULT)
 241   BUILTIN_VQN (TERNOPU, rsubhn2, 0, DEFAULT)
 242
 243   /* Implemented by aarch64_<us>xtl<mode>.  */
 244   BUILTIN_VQN (UNOP, sxtl, 0, DEFAULT)
 245   BUILTIN_VQN (UNOPU, uxtl, 0, DEFAULT)
 246
 247   /* Implemented by aarch64_xtn<mode>.  */
 248   BUILTIN_VQN (UNOP, xtn, 0, DEFAULT)
 249   BUILTIN_VQN (UNOPU, xtn, 0, DEFAULT)
 250
 251   /* Implemented by aarch64_mla<mode>.  */
 252   BUILTIN_VDQ_BHSI (TERNOP, mla, 0, DEFAULT)
 253   BUILTIN_VDQ_BHSI (TERNOPU, mla, 0, DEFAULT)
 254   /* Implemented by aarch64_mla_n<mode>.  */
 255   BUILTIN_VDQHS (TERNOP, mla_n, 0, DEFAULT)
 256   BUILTIN_VDQHS (TERNOPU, mla_n, 0, DEFAULT)
 257
 258   /* Implemented by aarch64_mls<mode>.  */
 259   BUILTIN_VDQ_BHSI (TERNOP, mls, 0, DEFAULT)
 260   BUILTIN_VDQ_BHSI (TERNOPU, mls, 0, DEFAULT)
 261   /* Implemented by aarch64_mls_n<mode>.  */
 262   BUILTIN_VDQHS (TERNOP, mls_n, 0, DEFAULT)
 263   BUILTIN_VDQHS (TERNOPU, mls_n, 0, DEFAULT)
 264
 265   BUILTIN_VQN (SHIFTIMM, shrn_n, 0, DEFAULT)
 266   BUILTIN_VQN (USHIFTIMM, shrn_n, 0, DEFAULT)
 267
 268   BUILTIN_VQN (SHIFT2IMM, ushrn2_n, 0, DEFAULT)
 269   BUILTIN_VQN (USHIFT2IMM, ushrn2_n, 0, DEFAULT)
 270
 271   BUILTIN_VQN (SHIFTIMM, rshrn_n, 0, DEFAULT)
 272   BUILTIN_VQN (USHIFTIMM, rshrn_n, 0, DEFAULT)
 273
 274   BUILTIN_VQN (SHIFT2IMM, rshrn2_n, 0, DEFAULT)
 275   BUILTIN_VQN (USHIFT2IMM, rshrn2_n, 0, DEFAULT)
 276
 277   /* Implemented by aarch64_<su>mlsl<mode>.  */
 278   BUILTIN_VD_BHSI (TERNOP, smlsl, 0, DEFAULT)
 279   BUILTIN_VD_BHSI (TERNOPU, umlsl, 0, DEFAULT)
 280
 281   /* Implemented by aarch64_<su>mlsl_n<mode>.  */
 282   BUILTIN_VD_HSI (TERNOP, smlsl_n, 0, DEFAULT)
 283   BUILTIN_VD_HSI (TERNOPU, umlsl_n, 0, DEFAULT)
 284
 285   /* Implemented by aarch64_<su>mlal<mode>.  */
 286   BUILTIN_VD_BHSI (TERNOP, smlal, 0, DEFAULT)
 287   BUILTIN_VD_BHSI (TERNOPU, umlal, 0, DEFAULT)
 288
 289   /* Implemented by aarch64_<su>mlal_n<mode>.  */
 290   BUILTIN_VD_HSI (TERNOP, smlal_n, 0, DEFAULT)
 291   BUILTIN_VD_HSI (TERNOPU, umlal_n, 0, DEFAULT)
 292
 293   /* Implemented by aarch64_<su>mlsl_hi<mode>.  */
 294   BUILTIN_VQW (TERNOP, smlsl_hi, 0, DEFAULT)
 295   BUILTIN_VQW (TERNOPU, umlsl_hi, 0, DEFAULT)
 296
 297   /* Implemented by aarch64_<su>mlsl_hi_n<mode>.  */
 298   BUILTIN_VQ_HSI (TERNOP, smlsl_hi_n, 0, DEFAULT)
 299   BUILTIN_VQ_HSI (TERNOPU, umlsl_hi_n, 0, DEFAULT)
 300
 301   /* Implemented by aarch64_<su>mlal_hi<mode>.  */
 302   BUILTIN_VQW (TERNOP, smlal_hi, 0, DEFAULT)
 303   BUILTIN_VQW (TERNOPU, umlal_hi, 0, DEFAULT)
 304
 305   /* Implemented by aarch64_<su>mlal_hi_n<mode>.  */
 306   BUILTIN_VQ_HSI (TERNOP, smlal_hi_n, 0, DEFAULT)
 307   BUILTIN_VQ_HSI (TERNOPU, umlal_hi_n, 0, DEFAULT)
 308
 309   /* Implemented by aarch64_sqmovun<mode>.  */
 310   BUILTIN_VQN (UNOPUS, sqmovun, 0, DEFAULT)
 311   BUILTIN_SD_HSDI (UNOPUS, sqmovun, 0, DEFAULT)
 312
 313   /* Implemented by aarch64_sqxtun2<mode>.  */
 314   BUILTIN_VQN (BINOP_UUS, sqxtun2, 0, DEFAULT)
 315
 316   /* Implemented by aarch64_<su>qmovn<mode>.  */
 317   BUILTIN_VQN (UNOP, sqmovn, 0, DEFAULT)
 318   BUILTIN_SD_HSDI (UNOP, sqmovn, 0, DEFAULT)
 319   BUILTIN_VQN (UNOP, uqmovn, 0, DEFAULT)
 320   BUILTIN_SD_HSDI (UNOP, uqmovn, 0, DEFAULT)
 321
 322   /* Implemented by aarch64_<su>qxtn2<mode>.  */
 323   BUILTIN_VQN (BINOP, sqxtn2, 0, DEFAULT)
 324   BUILTIN_VQN (BINOPU, uqxtn2, 0, DEFAULT)
 325
 326   /* Implemented by aarch64_s<optab><mode>.  */
 327   BUILTIN_VSDQ_I (UNOP, sqabs, 0, DEFAULT)
 328   BUILTIN_VSDQ_I (UNOP, sqneg, 0, DEFAULT)
 329
 330   /* Implemented by aarch64_sqdml<SBINQOPS:as>l<mode>.  */
 331   BUILTIN_VSD_HSI (TERNOP, sqdmlal, 0, DEFAULT)
 332   BUILTIN_VSD_HSI (TERNOP, sqdmlsl, 0, DEFAULT)
 333   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_lane<mode>.  */
 334   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_lane, 0, DEFAULT)
 335   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_lane, 0, DEFAULT)
 336   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_laneq<mode>.  */
 337   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlal_laneq, 0, DEFAULT)
 338   BUILTIN_VSD_HSI (QUADOP_LANE, sqdmlsl_laneq, 0, DEFAULT)
 339   /* Implemented by aarch64_sqdml<SBINQOPS:as>l_n<mode>.  */
 340   BUILTIN_VD_HSI (TERNOP, sqdmlal_n, 0, DEFAULT)
 341   BUILTIN_VD_HSI (TERNOP, sqdmlsl_n, 0, DEFAULT)
 342
 343   BUILTIN_VQ_HSI (TERNOP, sqdmlal2, 0, DEFAULT)
 344   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2, 0, DEFAULT)
 345   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_lane, 0, DEFAULT)
 346   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_lane, 0, DEFAULT)
 347   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlal2_laneq, 0, DEFAULT)
 348   BUILTIN_VQ_HSI (QUADOP_LANE, sqdmlsl2_laneq, 0, DEFAULT)
 349   BUILTIN_VQ_HSI (TERNOP, sqdmlal2_n, 0, DEFAULT)
 350   BUILTIN_VQ_HSI (TERNOP, sqdmlsl2_n, 0, DEFAULT)
 351
 352   BUILTIN_VD_BHSI (BINOP, intrinsic_vec_smult_lo_, 0, DEFAULT)
 353   BUILTIN_VD_BHSI (BINOPU, intrinsic_vec_umult_lo_, 0, DEFAULT)
 354
 355   BUILTIN_VQW (BINOP, vec_widen_smult_hi_, 10, DEFAULT)
 356   BUILTIN_VQW (BINOPU, vec_widen_umult_hi_, 10, DEFAULT)
 357
 358   BUILTIN_VD_HSI (BINOP, smull_n, 0, DEFAULT)
 359   BUILTIN_VD_HSI (BINOPU, umull_n, 0, DEFAULT)
 360
 361   BUILTIN_VQ_HSI (BINOP, smull_hi_n, 0, DEFAULT)
 362   BUILTIN_VQ_HSI (BINOPU, umull_hi_n, 0, DEFAULT)
 363
 364   BUILTIN_VQ_HSI (TERNOP_LANE, smull_hi_lane, 0, DEFAULT)
 365   BUILTIN_VQ_HSI (TERNOP_LANE, smull_hi_laneq, 0, DEFAULT)
 366   BUILTIN_VQ_HSI (TERNOPU_LANE, umull_hi_lane, 0, DEFAULT)
 367   BUILTIN_VQ_HSI (TERNOPU_LANE, umull_hi_laneq, 0, DEFAULT)
 368
 369   BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_lane_, 0, DEFAULT)
 370   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_lane_, 0, DEFAULT)
 371   BUILTIN_VD_HSI (TERNOP_LANE, vec_smult_laneq_, 0, DEFAULT)
 372   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlal_laneq_, 0, DEFAULT)
 373   BUILTIN_VD_HSI (TERNOPU_LANE, vec_umult_lane_, 0, DEFAULT)
 374   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlal_lane_, 0, DEFAULT)
 375   BUILTIN_VD_HSI (TERNOPU_LANE, vec_umult_laneq_, 0, DEFAULT)
 376   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlal_laneq_, 0, DEFAULT)
 377
 378   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlsl_lane_, 0, DEFAULT)
 379   BUILTIN_VD_HSI (QUADOP_LANE, vec_smlsl_laneq_, 0, DEFAULT)
 380   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlsl_lane_, 0, DEFAULT)
 381   BUILTIN_VD_HSI (QUADOPU_LANE, vec_umlsl_laneq_, 0, DEFAULT)
 382
 383   BUILTIN_VQ_HSI (QUADOP_LANE, smlal_hi_lane, 0, DEFAULT)
 384   BUILTIN_VQ_HSI (QUADOP_LANE, smlal_hi_laneq, 0, DEFAULT)
 385   BUILTIN_VQ_HSI (QUADOPU_LANE, umlal_hi_lane, 0, DEFAULT)
 386   BUILTIN_VQ_HSI (QUADOPU_LANE, umlal_hi_laneq, 0, DEFAULT)
 387
 388   BUILTIN_VQ_HSI (QUADOP_LANE, smlsl_hi_lane, 0, DEFAULT)
 389   BUILTIN_VQ_HSI (QUADOP_LANE, smlsl_hi_laneq, 0, DEFAULT)
 390   BUILTIN_VQ_HSI (QUADOPU_LANE, umlsl_hi_lane, 0, DEFAULT)
 391   BUILTIN_VQ_HSI (QUADOPU_LANE, umlsl_hi_laneq, 0, DEFAULT)
 392
 393   BUILTIN_VSD_HSI (BINOP, sqdmull, 0, DEFAULT)
 394   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_lane, 0, DEFAULT)
 395   BUILTIN_VSD_HSI (TERNOP_LANE, sqdmull_laneq, 0, DEFAULT)
 396   BUILTIN_VD_HSI (BINOP, sqdmull_n, 0, DEFAULT)
 397   BUILTIN_VQ_HSI (BINOP, sqdmull2, 0, DEFAULT)
 398   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_lane, 0, DEFAULT)
 399   BUILTIN_VQ_HSI (TERNOP_LANE, sqdmull2_laneq, 0, DEFAULT)
 400   BUILTIN_VQ_HSI (BINOP, sqdmull2_n, 0, DEFAULT)
 401   /* Implemented by aarch64_sq<r>dmulh<mode>.  */
 402   BUILTIN_VSDQ_HSI (BINOP, sqdmulh, 0, DEFAULT)
 403   BUILTIN_VSDQ_HSI (BINOP, sqrdmulh, 0, DEFAULT)
 404   /* Implemented by aarch64_sq<r>dmulh_n<mode>.  */
 405   BUILTIN_VDQHS (BINOP, sqdmulh_n, 0, DEFAULT)
 406   BUILTIN_VDQHS (BINOP, sqrdmulh_n, 0, DEFAULT)
 407   /* Implemented by aarch64_sq<r>dmulh_lane<q><mode>.  */
 408   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_lane, 0, DEFAULT)
 409   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqdmulh_laneq, 0, DEFAULT)
 410   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_lane, 0, DEFAULT)
 411   BUILTIN_VSDQ_HSI (TERNOP_LANE, sqrdmulh_laneq, 0, DEFAULT)
 412
 413   BUILTIN_VSDQ_I_DI (BINOP, ashl, 3, DEFAULT)
 414   /* Implemented by aarch64_<sur>shl<mode>.  */
 415   BUILTIN_VSDQ_I_DI (BINOP, sshl, 0, DEFAULT)
 416   BUILTIN_VSDQ_I_DI (BINOP_UUS, ushl, 0, DEFAULT)
 417   BUILTIN_VSDQ_I_DI (BINOP, srshl, 0, DEFAULT)
 418   BUILTIN_VSDQ_I_DI (BINOP_UUS, urshl, 0, DEFAULT)
 419
 420   /* Implemented by <sur><dotprod>_prod<dot_mode>.  */
 421   BUILTIN_VB (TERNOP, sdot_prod, 0, DEFAULT)
 422   BUILTIN_VB (TERNOPU, udot_prod, 0, DEFAULT)
 423   BUILTIN_VB (TERNOP_SUSS, usdot_prod, 0, DEFAULT)
 424   /* Implemented by aarch64_<sur><dotprod>_lane{q}<dot_mode>.  */
 425   BUILTIN_VB (QUADOP_LANE, sdot_lane, 0, DEFAULT)
 426   BUILTIN_VB (QUADOPU_LANE, udot_lane, 0, DEFAULT)
 427   BUILTIN_VB (QUADOP_LANE, sdot_laneq, 0, DEFAULT)
 428   BUILTIN_VB (QUADOPU_LANE, udot_laneq, 0, DEFAULT)
 429   BUILTIN_VB (QUADOPSSUS_LANE_QUADTUP, usdot_lane, 0, DEFAULT)
 430   BUILTIN_VB (QUADOPSSUS_LANE_QUADTUP, usdot_laneq, 0, DEFAULT)
 431   BUILTIN_VB (QUADOPSSSU_LANE_QUADTUP, sudot_lane, 0, DEFAULT)
 432   BUILTIN_VB (QUADOPSSSU_LANE_QUADTUP, sudot_laneq, 0, DEFAULT)
 433
 434   /* Implemented by aarch64_fcadd<rot><mode>.   */
 435   BUILTIN_VHSDF (BINOP, fcadd90, 0, FP)
 436   BUILTIN_VHSDF (BINOP, fcadd270, 0, FP)
 437
 438   /* Implemented by aarch64_fcmla{_lane}{q}<rot><mode>.   */
 439   BUILTIN_VHSDF (TERNOP, fcmla0, 0, FP)
 440   BUILTIN_VHSDF (TERNOP, fcmla90, 0, FP)
 441   BUILTIN_VHSDF (TERNOP, fcmla180, 0, FP)
 442   BUILTIN_VHSDF (TERNOP, fcmla270, 0, FP)
 443   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane0, 0, FP)
 444   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane90, 0, FP)
 445   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane180, 0, FP)
 446   BUILTIN_VHSDF (QUADOP_LANE_PAIR, fcmla_lane270, 0, FP)
 447
 448   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane0, 0, FP)
 449   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane90, 0, FP)
 450   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane180, 0, FP)
 451   BUILTIN_VQ_HSF (QUADOP_LANE_PAIR, fcmlaq_lane270, 0, FP)
 452
 453   BUILTIN_VDQ_I (SHIFTIMM, ashr, 3, DEFAULT)
 454   VAR1 (SHIFTIMM, ashr_simd, 0, DEFAULT, di)
 455   BUILTIN_VDQ_I (USHIFTIMM, lshr, 3, DEFAULT)
 456   VAR1 (USHIFTIMM, lshr_simd, 0, DEFAULT, di)
 457   /* Implemented by aarch64_<sur>shr_n<mode>.  */
 458   BUILTIN_VSDQ_I_DI (SHIFTIMM, srshr_n, 0, DEFAULT)
 459   BUILTIN_VSDQ_I_DI (USHIFTIMM, urshr_n, 0, DEFAULT)
 460   /* Implemented by aarch64_<sur>sra_n<mode>.  */
 461   BUILTIN_VSDQ_I_DI (SHIFTACC, ssra_n, 0, DEFAULT)
 462   BUILTIN_VSDQ_I_DI (USHIFTACC, usra_n, 0, DEFAULT)
 463   BUILTIN_VSDQ_I_DI (SHIFTACC, srsra_n, 0, DEFAULT)
 464   BUILTIN_VSDQ_I_DI (USHIFTACC, ursra_n, 0, DEFAULT)
 465   /* Implemented by aarch64_<sur>shll_n<mode>.  */
 466   BUILTIN_VD_BHSI (SHIFTIMM, sshll_n, 0, DEFAULT)
 467   BUILTIN_VD_BHSI (USHIFTIMM, ushll_n, 0, DEFAULT)
 468   /* Implemented by aarch64_<sur>shll2_n<mode>.  */
 469   BUILTIN_VQW (SHIFTIMM, sshll2_n, 0, DEFAULT)
 470   BUILTIN_VQW (SHIFTIMM, ushll2_n, 0, DEFAULT)
 471   BUILTIN_VQN (SHIFTIMM, sqshrun_n, 0, DEFAULT)
 472   BUILTIN_VQN (SHIFTIMM, sqrshrun_n, 0, DEFAULT)
 473   BUILTIN_VQN (SHIFTIMM, sqshrn_n, 0, DEFAULT)
 474   BUILTIN_VQN (USHIFTIMM, uqshrn_n, 0, DEFAULT)
 475   BUILTIN_VQN (SHIFTIMM, sqrshrn_n, 0, DEFAULT)
 476   BUILTIN_VQN (USHIFTIMM, uqrshrn_n, 0, DEFAULT)
 477   BUILTIN_SD_HSDI (SHIFTIMM, sqshrun_n, 0, DEFAULT)
 478   BUILTIN_SD_HSDI (SHIFTIMM, sqrshrun_n, 0, DEFAULT)
 479   BUILTIN_SD_HSDI (SHIFTIMM, sqshrn_n, 0, DEFAULT)
 480   BUILTIN_SD_HSDI (USHIFTIMM, uqshrn_n, 0, DEFAULT)
 481   BUILTIN_SD_HSDI (SHIFTIMM, sqrshrn_n, 0, DEFAULT)
 482   BUILTIN_SD_HSDI (USHIFTIMM, uqrshrn_n, 0, DEFAULT)
 483   BUILTIN_VQN (SHIFT2IMM_UUSS, sqshrun2_n, 0, DEFAULT)
 484   BUILTIN_VQN (SHIFT2IMM_UUSS, sqrshrun2_n, 0, DEFAULT)
 485   BUILTIN_VQN (SHIFT2IMM, sqsshrn2_n, 0, DEFAULT)
 486   BUILTIN_VQN (USHIFT2IMM, uqushrn2_n, 0, DEFAULT)
 487   BUILTIN_VQN (SHIFT2IMM, sqrshrn2_n, 0, DEFAULT)
 488   BUILTIN_VQN (USHIFT2IMM, uqrshrn2_n, 0, DEFAULT)
 489   /* Implemented by aarch64_<sur>s<lr>i_n<mode>.  */
 490   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssri_n, 0, DEFAULT)
 491   BUILTIN_VALLP (SHIFTINSERTP, ssri_n, 0, DEFAULT)
 492   BUILTIN_VSDQ_I_DI (USHIFTACC, usri_n, 0, DEFAULT)
 493   BUILTIN_VSDQ_I_DI (SHIFTINSERT, ssli_n, 0, DEFAULT)
 494   BUILTIN_VALLP (SHIFTINSERTP, ssli_n, 0, DEFAULT)
 495   BUILTIN_VSDQ_I_DI (USHIFTACC, usli_n, 0, DEFAULT)
 496   /* Implemented by aarch64_<sur>qshl<u>_n<mode>.  */
 497   BUILTIN_VSDQ_I (SHIFTIMM_USS, sqshlu_n, 0, DEFAULT)
 498   BUILTIN_VSDQ_I (SHIFTIMM, sqshl_n, 0, DEFAULT)
 499   BUILTIN_VSDQ_I (USHIFTIMM, uqshl_n, 0, DEFAULT)
 500
 501   /* Implemented by aarch64_xtn2<mode>.  */
 502   BUILTIN_VQN (BINOP, xtn2, 0, DEFAULT)
 503   BUILTIN_VQN (BINOPU, xtn2, 0, DEFAULT)
 504
 505   /* Implemented by vec_unpack<su>_hi_<mode>.  */
 506   BUILTIN_VQW (UNOP, vec_unpacks_hi_, 10, DEFAULT)
 507   BUILTIN_VQW (UNOPU, vec_unpacku_hi_, 10, DEFAULT)
 508
 509   /* Implemented by aarch64_reduc_plus_<mode>.  */
 510   BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, DEFAULT)
 511   BUILTIN_VDQ_I (UNOPU, reduc_plus_scal_, 10, DEFAULT)
 512
 513   /* Implemented by reduc_<maxmin_uns>_scal_<mode> (producing scalar).  */
 514   BUILTIN_VDQIF_F16 (UNOP, reduc_smax_scal_, 10, DEFAULT)
 515   BUILTIN_VDQIF_F16 (UNOP, reduc_smin_scal_, 10, DEFAULT)
 516   BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10, DEFAULT)
 517   BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10, DEFAULT)
 518   BUILTIN_VHSDF (UNOP, reduc_smax_nan_scal_, 10, DEFAULT)
 519   BUILTIN_VHSDF (UNOP, reduc_smin_nan_scal_, 10, DEFAULT)
 520
 521   /* Implemented by <optab><mode>3.  */
 522   BUILTIN_VDQ_BHSI (BINOP, smax, 3, DEFAULT)
 523   BUILTIN_VDQ_BHSI (BINOP, smin, 3, DEFAULT)
 524   BUILTIN_VDQ_BHSI (BINOP, umax, 3, DEFAULT)
 525   BUILTIN_VDQ_BHSI (BINOP, umin, 3, DEFAULT)
 526
 527   /* Implemented by <fmaxmin><mode>3.  */
 528   BUILTIN_VHSDF_HSDF (BINOP, fmax, 3, FP)
 529   BUILTIN_VHSDF_HSDF (BINOP, fmin, 3, FP)
 530   BUILTIN_VHSDF_DF (BINOP, fmax_nan, 3, FP)
 531   BUILTIN_VHSDF_DF (BINOP, fmin_nan, 3, FP)
 532
 533   /* Implemented by aarch64_<optab>p<mode>.  */
 534   BUILTIN_VDQ_BHSI (BINOP, smaxp, 0, DEFAULT)
 535   BUILTIN_VDQ_BHSI (BINOP, sminp, 0, DEFAULT)
 536   BUILTIN_VDQ_BHSI (BINOP, umaxp, 0, DEFAULT)
 537   BUILTIN_VDQ_BHSI (BINOP, uminp, 0, DEFAULT)
 538   BUILTIN_VHSDF (BINOP, smaxp, 0, DEFAULT)
 539   BUILTIN_VHSDF (BINOP, sminp, 0, DEFAULT)
 540   BUILTIN_VHSDF (BINOP, smax_nanp, 0, DEFAULT)
 541   BUILTIN_VHSDF (BINOP, smin_nanp, 0, DEFAULT)
 542
 543   /* Implemented by <frint_pattern><mode>2.  */
 544   BUILTIN_VHSDF (UNOP, btrunc, 2, FP)
 545   BUILTIN_VHSDF (UNOP, ceil, 2, FP)
 546   BUILTIN_VHSDF (UNOP, floor, 2, FP)
 547   BUILTIN_VHSDF (UNOP, nearbyint, 2, FP)
 548   BUILTIN_VHSDF (UNOP, rint, 2, FP)
 549   BUILTIN_VHSDF (UNOP, round, 2, FP)
 550   BUILTIN_VHSDF_HSDF (UNOP, roundeven, 2, FP)
 551
 552   VAR1 (UNOP, btrunc, 2, FP, hf)
 553   VAR1 (UNOP, ceil, 2, FP, hf)
 554   VAR1 (UNOP, floor, 2, FP, hf)
 555   VAR1 (UNOP, nearbyint, 2, FP, hf)
 556   VAR1 (UNOP, rint, 2, FP, hf)
 557   VAR1 (UNOP, round, 2, FP, hf)
 558
 559   /* Implemented by l<fcvt_pattern><su_optab><VQDF:mode><vcvt_target>2.  */
 560   VAR1 (UNOP, lbtruncv4hf, 2, FP, v4hi)
 561   VAR1 (UNOP, lbtruncv8hf, 2, FP, v8hi)
 562   VAR1 (UNOP, lbtruncv2sf, 2, FP, v2si)
 563   VAR1 (UNOP, lbtruncv4sf, 2, FP, v4si)
 564   VAR1 (UNOP, lbtruncv2df, 2, FP, v2di)
 565
 566   VAR1 (UNOPUS, lbtruncuv4hf, 2, FP, v4hi)
 567   VAR1 (UNOPUS, lbtruncuv8hf, 2, FP, v8hi)
 568   VAR1 (UNOPUS, lbtruncuv2sf, 2, FP, v2si)
 569   VAR1 (UNOPUS, lbtruncuv4sf, 2, FP, v4si)
 570   VAR1 (UNOPUS, lbtruncuv2df, 2, FP, v2di)
 571
 572   VAR1 (UNOP, lroundv4hf, 2, FP, v4hi)
 573   VAR1 (UNOP, lroundv8hf, 2, FP, v8hi)
 574   VAR1 (UNOP, lroundv2sf, 2, FP, v2si)
 575   VAR1 (UNOP, lroundv4sf, 2, FP, v4si)
 576   VAR1 (UNOP, lroundv2df, 2, FP, v2di)
 577   /* Implemented by l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2.  */
 578   BUILTIN_GPI_I16 (UNOP, lroundhf, 2, FP)
 579   VAR1 (UNOP, lroundsf, 2, FP, si)
 580   VAR1 (UNOP, lrounddf, 2, FP, di)
 581
 582   VAR1 (UNOPUS, lrounduv4hf, 2, FP, v4hi)
 583   VAR1 (UNOPUS, lrounduv8hf, 2, FP, v8hi)
 584   VAR1 (UNOPUS, lrounduv2sf, 2, FP, v2si)
 585   VAR1 (UNOPUS, lrounduv4sf, 2, FP, v4si)
 586   VAR1 (UNOPUS, lrounduv2df, 2, FP, v2di)
 587   BUILTIN_GPI_I16 (UNOPUS, lrounduhf, 2, FP)
 588   VAR1 (UNOPUS, lroundusf, 2, FP, si)
 589   VAR1 (UNOPUS, lroundudf, 2, FP, di)
 590
 591   VAR1 (UNOP, lceilv4hf, 2, FP, v4hi)
 592   VAR1 (UNOP, lceilv8hf, 2, FP, v8hi)
 593   VAR1 (UNOP, lceilv2sf, 2, FP, v2si)
 594   VAR1 (UNOP, lceilv4sf, 2, FP, v4si)
 595   VAR1 (UNOP, lceilv2df, 2, FP, v2di)
 596   BUILTIN_GPI_I16 (UNOP, lceilhf, 2, FP)
 597
 598   VAR1 (UNOPUS, lceiluv4hf, 2, FP, v4hi)
 599   VAR1 (UNOPUS, lceiluv8hf, 2, FP, v8hi)
 600   VAR1 (UNOPUS, lceiluv2sf, 2, FP, v2si)
 601   VAR1 (UNOPUS, lceiluv4sf, 2, FP, v4si)
 602   VAR1 (UNOPUS, lceiluv2df, 2, FP, v2di)
 603   BUILTIN_GPI_I16 (UNOPUS, lceiluhf, 2, FP)
 604   VAR1 (UNOPUS, lceilusf, 2, FP, si)
 605   VAR1 (UNOPUS, lceiludf, 2, FP, di)
 606
 607   VAR1 (UNOP, lfloorv4hf, 2, FP, v4hi)
 608   VAR1 (UNOP, lfloorv8hf, 2, FP, v8hi)
 609   VAR1 (UNOP, lfloorv2sf, 2, FP, v2si)
 610   VAR1 (UNOP, lfloorv4sf, 2, FP, v4si)
 611   VAR1 (UNOP, lfloorv2df, 2, FP, v2di)
 612   BUILTIN_GPI_I16 (UNOP, lfloorhf, 2, FP)
 613
 614   VAR1 (UNOPUS, lflooruv4hf, 2, FP, v4hi)
 615   VAR1 (UNOPUS, lflooruv8hf, 2, FP, v8hi)
 616   VAR1 (UNOPUS, lflooruv2sf, 2, FP, v2si)
 617   VAR1 (UNOPUS, lflooruv4sf, 2, FP, v4si)
 618   VAR1 (UNOPUS, lflooruv2df, 2, FP, v2di)
 619   BUILTIN_GPI_I16 (UNOPUS, lflooruhf, 2, FP)
 620   VAR1 (UNOPUS, lfloorusf, 2, FP, si)
 621   VAR1 (UNOPUS, lfloorudf, 2, FP, di)
 622
 623   VAR1 (UNOP, lfrintnv4hf, 2, FP, v4hi)
 624   VAR1 (UNOP, lfrintnv8hf, 2, FP, v8hi)
 625   VAR1 (UNOP, lfrintnv2sf, 2, FP, v2si)
 626   VAR1 (UNOP, lfrintnv4sf, 2, FP, v4si)
 627   VAR1 (UNOP, lfrintnv2df, 2, FP, v2di)
 628   BUILTIN_GPI_I16 (UNOP, lfrintnhf, 2, FP)
 629   VAR1 (UNOP, lfrintnsf, 2, FP, si)
 630   VAR1 (UNOP, lfrintndf, 2, FP, di)
 631
 632   VAR1 (UNOPUS, lfrintnuv4hf, 2, FP, v4hi)
 633   VAR1 (UNOPUS, lfrintnuv8hf, 2, FP, v8hi)
 634   VAR1 (UNOPUS, lfrintnuv2sf, 2, FP, v2si)
 635   VAR1 (UNOPUS, lfrintnuv4sf, 2, FP, v4si)
 636   VAR1 (UNOPUS, lfrintnuv2df, 2, FP, v2di)
 637   BUILTIN_GPI_I16 (UNOPUS, lfrintnuhf, 2, FP)
 638   VAR1 (UNOPUS, lfrintnusf, 2, FP, si)
 639   VAR1 (UNOPUS, lfrintnudf, 2, FP, di)
 640
 641   /* Implemented by <optab><fcvt_target><VDQF:mode>2.  */
 642   VAR1 (UNOP, floatv4hi, 2, FP, v4hf)
 643   VAR1 (UNOP, floatv8hi, 2, FP, v8hf)
 644   VAR1 (UNOP, floatv2si, 2, FP, v2sf)
 645   VAR1 (UNOP, floatv4si, 2, FP, v4sf)
 646   VAR1 (UNOP, floatv2di, 2, FP, v2df)
 647
 648   VAR1 (UNOP, floatunsv4hi, 2, FP, v4hf)
 649   VAR1 (UNOP, floatunsv8hi, 2, FP, v8hf)
 650   VAR1 (UNOP, floatunsv2si, 2, FP, v2sf)
 651   VAR1 (UNOP, floatunsv4si, 2, FP, v4sf)
 652   VAR1 (UNOP, floatunsv2di, 2, FP, v2df)
 653
 654   VAR5 (UNOPU, bswap, 2, DEFAULT, v4hi, v8hi, v2si, v4si, v2di)
 655
 656   BUILTIN_VB (UNOP, rbit, 0, DEFAULT)
 657
 658   /* Implemented by
 659      aarch64_<PERMUTE:perm_insn><mode>.  */
 660   BUILTIN_VALL (BINOP, zip1, 0, QUIET)
 661   BUILTIN_VALL (BINOP, zip2, 0, QUIET)
 662   BUILTIN_VALL (BINOP, uzp1, 0, QUIET)
 663   BUILTIN_VALL (BINOP, uzp2, 0, QUIET)
 664   BUILTIN_VALL (BINOP, trn1, 0, QUIET)
 665   BUILTIN_VALL (BINOP, trn2, 0, QUIET)
 666
 667   BUILTIN_GPF_F16 (UNOP, frecpe, 0, FP)
 668   BUILTIN_GPF_F16 (UNOP, frecpx, 0, FP)
 669
 670   BUILTIN_VDQ_SI (UNOP, urecpe, 0, DEFAULT)
 671
 672   BUILTIN_VHSDF (UNOP, frecpe, 0, FP)
 673   BUILTIN_VHSDF_HSDF (BINOP, frecps, 0, FP)
 674
 675   /* Implemented by a mixture of abs2 patterns.  Note the DImode builtin is
 676      only ever used for the int64x1_t intrinsic, there is no scalar version.  */
 677   BUILTIN_VSDQ_I_DI (UNOP, abs, 0, QUIET)
 678   BUILTIN_VHSDF (UNOP, abs, 2, QUIET)
 679   VAR1 (UNOP, abs, 2, QUIET, hf)
 680
 681   BUILTIN_VQ_HSF (UNOP, vec_unpacks_hi_, 10, FP)
 682   VAR1 (BINOP, float_truncate_hi_, 0, FP, v4sf)
 683   VAR1 (BINOP, float_truncate_hi_, 0, FP, v8hf)
 684
 685   VAR1 (UNOP, float_extend_lo_, 0, FP, v2df)
 686   VAR1 (UNOP, float_extend_lo_,  0, FP, v4sf)
 687   BUILTIN_VDF (UNOP, float_truncate_lo_, 0, FP)
 688
 689   VAR1 (UNOP, float_trunc_rodd_, 0, FP, df)
 690   VAR1 (UNOP, float_trunc_rodd_lo_, 0, FP, v2sf)
 691   VAR1 (BINOP, float_trunc_rodd_hi_, 0, FP, v4sf)
 692
 693   /* Implemented by aarch64_ld1<VALL_F16:mode>.  */
 694   BUILTIN_VALL_F16 (LOAD1, ld1, 0, LOAD)
 695   BUILTIN_VDQ_I (LOAD1_U, ld1, 0, LOAD)
 696   BUILTIN_VALLP_NO_DI (LOAD1_P, ld1, 0, LOAD)
 697
 698   /* Implemented by aarch64_st1<VALL_F16:mode>.  */
 699   BUILTIN_VALL_F16 (STORE1, st1, 0, STORE)
 700   BUILTIN_VDQ_I (STORE1_U, st1, 0, STORE)
 701   BUILTIN_VALLP_NO_DI (STORE1_P, st1, 0, STORE)
 702
 703   /* Implemented by fma<mode>4.  */
 704   BUILTIN_VHSDF (TERNOP, fma, 4, FP)
 705   VAR1 (TERNOP, fma, 4, FP, hf)
 706   /* Implemented by fnma<mode>4.  */
 707   BUILTIN_VHSDF (TERNOP, fnma, 4, FP)
 708   VAR1 (TERNOP, fnma, 4, FP, hf)
 709
 710   BUILTIN_VDQF_DF (TERNOP, float_mla, 0, FP)
 711   BUILTIN_VDQF_DF (TERNOP, float_mls, 0, FP)
 712   BUILTIN_VDQSF (TERNOP, float_mla_n, 0, FP)
 713   BUILTIN_VDQSF (TERNOP, float_mls_n, 0, FP)
 714   BUILTIN_VDQSF (QUADOP_LANE, float_mla_lane, 0, FP)
 715   BUILTIN_VDQSF (QUADOP_LANE, float_mls_lane, 0, FP)
 716   BUILTIN_VDQSF (QUADOP_LANE, float_mla_laneq, 0, FP)
 717   BUILTIN_VDQSF (QUADOP_LANE, float_mls_laneq, 0, FP)
 718
 719   /* Implemented by aarch64_simd_bsl<mode>.  */
 720   BUILTIN_VDQQH (BSL_P, simd_bsl, 0, DEFAULT)
 721   VAR2 (BSL_P, simd_bsl,0, DEFAULT, di, v2di)
 722   BUILTIN_VSDQ_I_DI (BSL_U, simd_bsl, 0, DEFAULT)
 723   BUILTIN_VALLDIF (BSL_S, simd_bsl, 0, QUIET)
 724
 725   /* Implemented by aarch64_crypto_aes<op><mode>.  */
 726   VAR1 (BINOPU, crypto_aese, 0, DEFAULT, v16qi)
 727   VAR1 (BINOPU, crypto_aesd, 0, DEFAULT, v16qi)
 728   VAR1 (UNOPU, crypto_aesmc, 0, DEFAULT, v16qi)
 729   VAR1 (UNOPU, crypto_aesimc, 0, DEFAULT, v16qi)
 730
 731   /* Implemented by aarch64_crypto_sha1<op><mode>.  */
 732   VAR1 (UNOPU, crypto_sha1h, 0, DEFAULT, si)
 733   VAR1 (BINOPU, crypto_sha1su1, 0, DEFAULT, v4si)
 734   VAR1 (TERNOPU, crypto_sha1c, 0, DEFAULT, v4si)
 735   VAR1 (TERNOPU, crypto_sha1m, 0, DEFAULT, v4si)
 736   VAR1 (TERNOPU, crypto_sha1p, 0, DEFAULT, v4si)
 737   VAR1 (TERNOPU, crypto_sha1su0, 0, DEFAULT, v4si)
 738
 739   /* Implemented by aarch64_crypto_sha256<op><mode>.  */
 740   VAR1 (TERNOPU, crypto_sha256h, 0, DEFAULT, v4si)
 741   VAR1 (TERNOPU, crypto_sha256h2, 0, DEFAULT, v4si)
 742   VAR1 (BINOPU, crypto_sha256su0, 0, DEFAULT, v4si)
 743   VAR1 (TERNOPU, crypto_sha256su1, 0, DEFAULT, v4si)
 744
 745   /* Implemented by aarch64_crypto_pmull<mode>.  */
 746   VAR1 (BINOPP, crypto_pmull, 0, DEFAULT, di)
 747   VAR1 (BINOPP, crypto_pmull, 0, DEFAULT, v2di)
 748
 749   /* Implemented by aarch64_qtbl1<mode>.  */
 750   VAR2 (BINOP, qtbl1, 0, DEFAULT, v8qi, v16qi)
 751   VAR2 (BINOPU, qtbl1, 0, DEFAULT, v8qi, v16qi)
 752   VAR2 (BINOP_PPU, qtbl1, 0, DEFAULT, v8qi, v16qi)
 753   VAR2 (BINOP_SSU, qtbl1, 0, DEFAULT, v8qi, v16qi)
 754
 755   /* Implemented by aarch64_qtbl2<mode>.  */
 756   VAR2 (BINOP, qtbl2, 0, DEFAULT, v8qi, v16qi)
 757   VAR2 (BINOPU, qtbl2, 0, DEFAULT, v8qi, v16qi)
 758   VAR2 (BINOP_PPU, qtbl2, 0, DEFAULT, v8qi, v16qi)
 759   VAR2 (BINOP_SSU, qtbl2, 0, DEFAULT, v8qi, v16qi)
 760
 761   /* Implemented by aarch64_qtbl3<mode>.  */
 762   VAR2 (BINOP, qtbl3, 0, DEFAULT, v8qi, v16qi)
 763   VAR2 (BINOPU, qtbl3, 0, DEFAULT, v8qi, v16qi)
 764   VAR2 (BINOP_PPU, qtbl3, 0, DEFAULT, v8qi, v16qi)
 765   VAR2 (BINOP_SSU, qtbl3, 0, DEFAULT, v8qi, v16qi)
 766
 767   /* Implemented by aarch64_qtbl4<mode>.  */
 768   VAR2 (BINOP, qtbl4, 0, DEFAULT, v8qi, v16qi)
 769   VAR2 (BINOPU, qtbl4, 0, DEFAULT, v8qi, v16qi)
 770   VAR2 (BINOP_PPU, qtbl4, 0, DEFAULT, v8qi, v16qi)
 771   VAR2 (BINOP_SSU, qtbl4, 0, DEFAULT, v8qi, v16qi)
 772
 773   /* Implemented by aarch64_qtbx1<mode>.  */
 774   VAR2 (TERNOP, qtbx1, 0, DEFAULT, v8qi, v16qi)
 775   VAR2 (TERNOPU, qtbx1, 0, DEFAULT, v8qi, v16qi)
 776   VAR2 (TERNOP_PPPU, qtbx1, 0, DEFAULT, v8qi, v16qi)
 777   VAR2 (TERNOP_SSSU, qtbx1, 0, DEFAULT, v8qi, v16qi)
 778
 779   /* Implemented by aarch64_qtbx2<mode>.  */
 780   VAR2 (TERNOP, qtbx2, 0, DEFAULT, v8qi, v16qi)
 781   VAR2 (TERNOPU, qtbx2, 0, DEFAULT, v8qi, v16qi)
 782   VAR2 (TERNOP_PPPU, qtbx2, 0, DEFAULT, v8qi, v16qi)
 783   VAR2 (TERNOP_SSSU, qtbx2, 0, DEFAULT, v8qi, v16qi)
 784
 785   /* Implemented by aarch64_qtbx3<mode>.  */
 786   VAR2 (TERNOP, qtbx3, 0, DEFAULT, v8qi, v16qi)
 787   VAR2 (TERNOPU, qtbx3, 0, DEFAULT, v8qi, v16qi)
 788   VAR2 (TERNOP_PPPU, qtbx3, 0, DEFAULT, v8qi, v16qi)
 789   VAR2 (TERNOP_SSSU, qtbx3, 0, DEFAULT, v8qi, v16qi)
 790
 791   /* Implemented by aarch64_qtbx4<mode>.  */
 792   VAR2 (TERNOP, qtbx4, 0, DEFAULT, v8qi, v16qi)
 793   VAR2 (TERNOPU, qtbx4, 0, DEFAULT, v8qi, v16qi)
 794   VAR2 (TERNOP_PPPU, qtbx4, 0, DEFAULT, v8qi, v16qi)
 795   VAR2 (TERNOP_SSSU, qtbx4, 0, DEFAULT, v8qi, v16qi)
 796
 797   /* Builtins for ARMv8.1-A Adv.SIMD instructions.  */
 798
 799   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h<mode>.  */
 800   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlah, 0, DEFAULT)
 801   BUILTIN_VSDQ_HSI (TERNOP, sqrdmlsh, 0, DEFAULT)
 802
 803   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_lane<mode>.  */
 804   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_lane, 0, DEFAULT)
 805   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_lane, 0, DEFAULT)
 806
 807   /* Implemented by aarch64_sqrdml<SQRDMLH_AS:rdma_as>h_laneq<mode>.  */
 808   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlah_laneq, 0, DEFAULT)
 809   BUILTIN_VSDQ_HSI (QUADOP_LANE, sqrdmlsh_laneq, 0, DEFAULT)
 810
 811   /* Implemented by <FCVT_F2FIXED/FIXED2F:fcvt_fixed_insn><*><*>3.  */
 812   BUILTIN_VSDQ_HSDI (SHIFTIMM, scvtf, 3, FP)
 813   BUILTIN_VSDQ_HSDI (FCVTIMM_SUS, ucvtf, 3, FP)
 814   BUILTIN_VHSDF_HSDF (SHIFTIMM, fcvtzs, 3, FP)
 815   BUILTIN_VHSDF_HSDF (SHIFTIMM_USS, fcvtzu, 3, FP)
 816   VAR1 (SHIFTIMM, scvtfsi, 3, FP, hf)
 817   VAR1 (SHIFTIMM, scvtfdi, 3, FP, hf)
 818   VAR1 (FCVTIMM_SUS, ucvtfsi, 3, FP, hf)
 819   VAR1 (FCVTIMM_SUS, ucvtfdi, 3, FP, hf)
 820   BUILTIN_GPI (SHIFTIMM, fcvtzshf, 3, FP)
 821   BUILTIN_GPI (SHIFTIMM_USS, fcvtzuhf, 3, FP)
 822
 823   /* Implemented by aarch64_rsqrte<mode>.  */
 824   BUILTIN_VHSDF_HSDF (UNOP, rsqrte, 0, FP)
 825
 826   /* Implemented by aarch64_rsqrts<mode>.  */
 827   BUILTIN_VHSDF_HSDF (BINOP, rsqrts, 0, FP)
 828
 829   /* Implemented by aarch64_ursqrte<mode>.  */
 830   BUILTIN_VDQ_SI (UNOPU, ursqrte, 0, DEFAULT)
 831
 832   /* Implemented by fabd<mode>3.  */
 833   BUILTIN_VHSDF_HSDF (BINOP, fabd, 3, FP)
 834
 835   /* Implemented by aarch64_faddp<mode>.  */
 836   BUILTIN_VHSDF (BINOP, faddp, 0, FP)
 837
 838   /* Implemented by aarch64_cm<optab><mode>.  */
 839   BUILTIN_VHSDF_HSDF (BINOP_USS, cmeq, 0, FP)
 840   BUILTIN_VHSDF_HSDF (BINOP_USS, cmge, 0, FP)
 841   BUILTIN_VHSDF_HSDF (BINOP_USS, cmgt, 0, FP)
 842   BUILTIN_VHSDF_HSDF (BINOP_USS, cmle, 0, FP)
 843   BUILTIN_VHSDF_HSDF (BINOP_USS, cmlt, 0, FP)
 844
 845   /* Implemented by aarch64_fac<optab><mode>.  */
 846   BUILTIN_VHSDF_HSDF (BINOP_USS, faclt, 0, FP)
 847   BUILTIN_VHSDF_HSDF (BINOP_USS, facle, 0, FP)
 848   BUILTIN_VHSDF_HSDF (BINOP_USS, facgt, 0, FP)
 849   BUILTIN_VHSDF_HSDF (BINOP_USS, facge, 0, FP)
 850
 851   /* Implemented by sqrt<mode>2.  */
 852   VAR1 (UNOP, sqrt, 2, FP, hf)
 853
 854   /* Implemented by <optab><mode>hf2.  */
 855   VAR1 (UNOP, floatdi, 2, FP, hf)
 856   VAR1 (UNOP, floatsi, 2, FP, hf)
 857   VAR1 (UNOP, floathi, 2, FP, hf)
 858   VAR1 (UNOPUS, floatunsdi, 2, FP, hf)
 859   VAR1 (UNOPUS, floatunssi, 2, FP, hf)
 860   VAR1 (UNOPUS, floatunshi, 2, FP, hf)
 861   BUILTIN_GPI_I16 (UNOP, fix_trunchf, 2, FP)
 862   BUILTIN_GPI (UNOP, fix_truncsf, 2, FP)
 863   BUILTIN_GPI (UNOP, fix_truncdf, 2, FP)
 864   BUILTIN_GPI_I16 (UNOPUS, fixuns_trunchf, 2, FP)
 865   BUILTIN_GPI (UNOPUS, fixuns_truncsf, 2, FP)
 866   BUILTIN_GPI (UNOPUS, fixuns_truncdf, 2, FP)
 867
 868   /* Implemented by aarch64_sm3ss1qv4si.  */
 869   VAR1 (TERNOPU, sm3ss1q, 0, DEFAULT, v4si)
 870   /* Implemented by aarch64_sm3tt<sm3tt_op>qv4si.  */
 871   VAR1 (QUADOPUI, sm3tt1aq, 0, DEFAULT, v4si)
 872   VAR1 (QUADOPUI, sm3tt1bq, 0, DEFAULT, v4si)
 873   VAR1 (QUADOPUI, sm3tt2aq, 0, DEFAULT, v4si)
 874   VAR1 (QUADOPUI, sm3tt2bq, 0, DEFAULT, v4si)
 875   /* Implemented by aarch64_sm3partw<sm3part_op>qv4si.  */
 876   VAR1 (TERNOPU, sm3partw1q, 0, DEFAULT, v4si)
 877   VAR1 (TERNOPU, sm3partw2q, 0, DEFAULT, v4si)
 878   /* Implemented by aarch64_sm4eqv4si.  */
 879   VAR1 (BINOPU, sm4eq, 0, DEFAULT, v4si)
 880   /* Implemented by aarch64_sm4ekeyqv4si.  */
 881   VAR1 (BINOPU, sm4ekeyq, 0, DEFAULT, v4si)
 882   /* Implemented by aarch64_crypto_sha512hqv2di.  */
 883   VAR1 (TERNOPU, crypto_sha512hq, 0, DEFAULT, v2di)
 884   /* Implemented by aarch64_sha512h2qv2di.  */
 885   VAR1 (TERNOPU, crypto_sha512h2q, 0, DEFAULT, v2di)
 886   /* Implemented by aarch64_crypto_sha512su0qv2di.  */
 887   VAR1 (BINOPU, crypto_sha512su0q, 0, DEFAULT, v2di)
 888   /* Implemented by aarch64_crypto_sha512su1qv2di.  */
 889   VAR1 (TERNOPU, crypto_sha512su1q, 0, DEFAULT, v2di)
 890   /* Implemented by eor3q<mode>4.  */
 891   BUILTIN_VQ_I (TERNOPU, eor3q, 4, DEFAULT)
 892   BUILTIN_VQ_I (TERNOP, eor3q, 4, DEFAULT)
 893   /* Implemented by aarch64_rax1qv2di.  */
 894   VAR1 (BINOPU, rax1q, 0, DEFAULT, v2di)
 895   /* Implemented by aarch64_xarqv2di.  */
 896   VAR1 (TERNOPUI, xarq, 0, DEFAULT, v2di)
 897   /* Implemented by bcaxq<mode>4.  */
 898   BUILTIN_VQ_I (TERNOPU, bcaxq, 4, DEFAULT)
 899   BUILTIN_VQ_I (TERNOP, bcaxq, 4, DEFAULT)
 900
 901   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_low<mode>.  */
 902   VAR1 (TERNOP, fmlal_low, 0, FP, v2sf)
 903   VAR1 (TERNOP, fmlsl_low, 0, FP, v2sf)
 904   VAR1 (TERNOP, fmlalq_low, 0, FP, v4sf)
 905   VAR1 (TERNOP, fmlslq_low, 0, FP, v4sf)
 906   /* Implemented by aarch64_fml<f16mac1>l<f16quad>_high<mode>.  */
 907   VAR1 (TERNOP, fmlal_high, 0, FP, v2sf)
 908   VAR1 (TERNOP, fmlsl_high, 0, FP, v2sf)
 909   VAR1 (TERNOP, fmlalq_high, 0, FP, v4sf)
 910   VAR1 (TERNOP, fmlslq_high, 0, FP, v4sf)
 911   /* Implemented by aarch64_fml<f16mac1>l_lane_lowv2sf.  */
 912   VAR1 (QUADOP_LANE, fmlal_lane_low, 0, FP, v2sf)
 913   VAR1 (QUADOP_LANE, fmlsl_lane_low, 0, FP, v2sf)
 914   /* Implemented by aarch64_fml<f16mac1>l_laneq_lowv2sf.  */
 915   VAR1 (QUADOP_LANE, fmlal_laneq_low, 0, FP, v2sf)
 916   VAR1 (QUADOP_LANE, fmlsl_laneq_low, 0, FP, v2sf)
 917   /* Implemented by aarch64_fml<f16mac1>lq_lane_lowv4sf.  */
 918   VAR1 (QUADOP_LANE, fmlalq_lane_low, 0, FP, v4sf)
 919   VAR1 (QUADOP_LANE, fmlslq_lane_low, 0, FP, v4sf)
 920   /* Implemented by aarch64_fml<f16mac1>lq_laneq_lowv4sf.  */
 921   VAR1 (QUADOP_LANE, fmlalq_laneq_low, 0, FP, v4sf)
 922   VAR1 (QUADOP_LANE, fmlslq_laneq_low, 0, FP, v4sf)
 923   /* Implemented by aarch64_fml<f16mac1>l_lane_highv2sf.  */
 924   VAR1 (QUADOP_LANE, fmlal_lane_high, 0, FP, v2sf)
 925   VAR1 (QUADOP_LANE, fmlsl_lane_high, 0, FP, v2sf)
 926   /* Implemented by aarch64_fml<f16mac1>l_laneq_highv2sf.  */
 927   VAR1 (QUADOP_LANE, fmlal_laneq_high, 0, FP, v2sf)
 928   VAR1 (QUADOP_LANE, fmlsl_laneq_high, 0, FP, v2sf)
 929   /* Implemented by aarch64_fml<f16mac1>lq_lane_highv4sf.  */
 930   VAR1 (QUADOP_LANE, fmlalq_lane_high, 0, FP, v4sf)
 931   VAR1 (QUADOP_LANE, fmlslq_lane_high, 0, FP, v4sf)
 932   /* Implemented by aarch64_fml<f16mac1>lq_laneq_highv4sf.  */
 933   VAR1 (QUADOP_LANE, fmlalq_laneq_high, 0, FP, v4sf)
 934   VAR1 (QUADOP_LANE, fmlslq_laneq_high, 0, FP, v4sf)
 935
 936   /* Implemented by aarch64_<frintnzs_op><mode>.  */
 937   BUILTIN_VSFDF (UNOP, frint32z, 0, FP)
 938   BUILTIN_VSFDF (UNOP, frint32x, 0, FP)
 939   BUILTIN_VSFDF (UNOP, frint64z, 0, FP)
 940   BUILTIN_VSFDF (UNOP, frint64x, 0, FP)
 941
 942   /* Implemented by aarch64_bfdot{_lane}{q}<mode>.  */
 943   VAR2 (TERNOP, bfdot, 0, QUIET, v2sf, v4sf)
 944   VAR2 (QUADOP_LANE_PAIR, bfdot_lane, 0, QUIET, v2sf, v4sf)
 945   VAR2 (QUADOP_LANE_PAIR, bfdot_laneq, 0, QUIET, v2sf, v4sf)
 946
 947   /* Implemented by aarch64_bfmmlaqv4sf  */
 948   VAR1 (TERNOP, bfmmlaq, 0, QUIET, v4sf)
 949
 950   /* Implemented by aarch64_bfmlal<bt>{_lane{q}}v4sf  */
 951   VAR1 (TERNOP, bfmlalb, 0, FP, v4sf)
 952   VAR1 (TERNOP, bfmlalt, 0, FP, v4sf)
 953   VAR1 (QUADOP_LANE, bfmlalb_lane, 0, FP, v4sf)
 954   VAR1 (QUADOP_LANE, bfmlalt_lane, 0, FP, v4sf)
 955   VAR1 (QUADOP_LANE, bfmlalb_lane_q, 0, FP, v4sf)
 956   VAR1 (QUADOP_LANE, bfmlalt_lane_q, 0, FP, v4sf)
 957
 958   /* Implemented by aarch64_simd_<sur>mmlav16qi.  */
 959   VAR1 (TERNOP, simd_smmla, 0, DEFAULT, v16qi)
 960   VAR1 (TERNOPU, simd_ummla, 0, DEFAULT, v16qi)
 961   VAR1 (TERNOP_SSUS, simd_usmmla, 0, DEFAULT, v16qi)
 962
 963   /* Implemented by aarch64_bfcvtn{q}{2}<mode>  */
 964   VAR1 (UNOP, bfcvtn, 0, FP, v4bf)
 965   VAR1 (UNOP, bfcvtn_q, 0, FP, v8bf)
 966   VAR1 (BINOP, bfcvtn2, 0, FP, v8bf)
 967   VAR1 (UNOP, bfcvt, 0, FP, bf)
 968
 969   /* Implemented by aarch64_{v}bfcvt{_high}<mode>.  */
 970   VAR2 (UNOP, vbfcvt, 0, QUIET, v4bf, v8bf)
 971   VAR1 (UNOP, vbfcvt_high, 0, QUIET, v8bf)
 972   VAR1 (UNOP, bfcvt, 0, QUIET, sf)