include/llvm/IR/IntrinsicsAArch64.td

   1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines all of the AARCH64-specific intrinsics.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 let TargetPrefix = "aarch64" in {
  14
  15 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  16 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  17 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  18 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  19
  20 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  21 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  22 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
  23                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  24 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
  25                                 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  26
  27 def int_aarch64_clrex : Intrinsic<[]>;
  28
  29 def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  30                                 LLVMMatchType<0>], [IntrNoMem]>;
  31 def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  32                                 LLVMMatchType<0>], [IntrNoMem]>;
  33
  34 def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;
  35
  36 //===----------------------------------------------------------------------===//
  37 // HINT
  38
  39 def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
  40
  41 //===----------------------------------------------------------------------===//
  42 // Data Barrier Instructions
  43
  44 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
  45 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
  46 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
  47
  48 // A space-consuming intrinsic primarily for testing block and jump table
  49 // placements. The first argument is the number of bytes this "instruction"
  50 // takes up, the second and return value are essentially chains, used to force
  51 // ordering during ISel.
  52 def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
  53
  54 }
  55
  56 //===----------------------------------------------------------------------===//
  57 // Advanced SIMD (NEON)
  58
  59 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  60   class AdvSIMD_2Scalar_Float_Intrinsic
  61     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  62                 [IntrNoMem]>;
  63
  64   class AdvSIMD_FPToIntRounding_Intrinsic
  65     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
  66
  67   class AdvSIMD_1IntArg_Intrinsic
  68     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  69   class AdvSIMD_1FloatArg_Intrinsic
  70     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  71   class AdvSIMD_1VectorArg_Intrinsic
  72     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  73   class AdvSIMD_1VectorArg_Expand_Intrinsic
  74     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  75   class AdvSIMD_1VectorArg_Long_Intrinsic
  76     : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
  77   class AdvSIMD_1IntArg_Narrow_Intrinsic
  78     : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
  79   class AdvSIMD_1VectorArg_Narrow_Intrinsic
  80     : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
  81   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
  82     : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  83   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
  84     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  85
  86   class AdvSIMD_2IntArg_Intrinsic
  87     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  88                 [IntrNoMem]>;
  89   class AdvSIMD_2FloatArg_Intrinsic
  90     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  91                 [IntrNoMem]>;
  92   class AdvSIMD_2VectorArg_Intrinsic
  93     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  94                 [IntrNoMem]>;
  95   class AdvSIMD_2VectorArg_Compare_Intrinsic
  96     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
  97                 [IntrNoMem]>;
  98   class AdvSIMD_2Arg_FloatCompare_Intrinsic
  99     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
 100                 [IntrNoMem]>;
 101   class AdvSIMD_2VectorArg_Long_Intrinsic
 102     : Intrinsic<[llvm_anyvector_ty],
 103                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
 104                 [IntrNoMem]>;
 105   class AdvSIMD_2VectorArg_Wide_Intrinsic
 106     : Intrinsic<[llvm_anyvector_ty],
 107                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
 108                 [IntrNoMem]>;
 109   class AdvSIMD_2VectorArg_Narrow_Intrinsic
 110     : Intrinsic<[llvm_anyvector_ty],
 111                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
 112                 [IntrNoMem]>;
 113   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
 114     : Intrinsic<[llvm_anyint_ty],
 115                 [LLVMExtendedType<0>, llvm_i32_ty],
 116                 [IntrNoMem]>;
 117   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
 118     : Intrinsic<[llvm_anyvector_ty],
 119                 [llvm_anyvector_ty],
 120                 [IntrNoMem]>;
 121   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
 122     : Intrinsic<[llvm_anyvector_ty],
 123                 [LLVMTruncatedType<0>],
 124                 [IntrNoMem]>;
 125   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
 126     : Intrinsic<[llvm_anyvector_ty],
 127                 [LLVMTruncatedType<0>, llvm_i32_ty],
 128                 [IntrNoMem]>;
 129   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
 130     : Intrinsic<[llvm_anyvector_ty],
 131                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
 132                 [IntrNoMem]>;
 133
 134   class AdvSIMD_3VectorArg_Intrinsic
 135       : Intrinsic<[llvm_anyvector_ty],
 136                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 137                [IntrNoMem]>;
 138   class AdvSIMD_3VectorArg_Scalar_Intrinsic
 139       : Intrinsic<[llvm_anyvector_ty],
 140                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
 141                [IntrNoMem]>;
 142   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
 143       : Intrinsic<[llvm_anyvector_ty],
 144                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
 145                 LLVMMatchType<1>], [IntrNoMem]>;
 146   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
 147     : Intrinsic<[llvm_anyvector_ty],
 148                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
 149                 [IntrNoMem]>;
 150   class AdvSIMD_CvtFxToFP_Intrinsic
 151     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
 152                 [IntrNoMem]>;
 153   class AdvSIMD_CvtFPToFx_Intrinsic
 154     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
 155                 [IntrNoMem]>;
 156
 157   class AdvSIMD_1Arg_Intrinsic
 158     : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 159
 160   class AdvSIMD_Dot_Intrinsic
 161     : Intrinsic<[llvm_anyvector_ty],
 162                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 163                 [IntrNoMem]>;
 164
 165   class AdvSIMD_FP16FML_Intrinsic
 166     : Intrinsic<[llvm_anyvector_ty],
 167                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 168                 [IntrNoMem]>;
 169 }
 170
 171 // Arithmetic ops
 172
 173 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
 174   // Vector Add Across Lanes
 175   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 176   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 177   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 178
 179   // Vector Long Add Across Lanes
 180   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 181   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 182
 183   // Vector Halving Add
 184   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
 185   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
 186
 187   // Vector Rounding Halving Add
 188   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
 189   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
 190
 191   // Vector Saturating Add
 192   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
 193   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
 194   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
 195   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
 196
 197   // Vector Add High-Half
 198   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 199   // header is no longer supported.
 200   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 201
 202   // Vector Rounding Add High-Half
 203   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 204
 205   // Vector Saturating Doubling Multiply High
 206   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
 207
 208   // Vector Saturating Rounding Doubling Multiply High
 209   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
 210
 211   // Vector Polynominal Multiply
 212   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
 213
 214   // Vector Long Multiply
 215   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
 216   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
 217   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 218
 219   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
 220   // it with a v16i8.
 221   def int_aarch64_neon_pmull64 :
 222         Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 223
 224   // Vector Extending Multiply
 225   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
 226     let IntrProperties = [IntrNoMem, Commutative];
 227   }
 228
 229   // Vector Saturating Doubling Long Multiply
 230   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 231   def int_aarch64_neon_sqdmulls_scalar
 232     : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 233
 234   // Vector Halving Subtract
 235   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
 236   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
 237
 238   // Vector Saturating Subtract
 239   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
 240   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
 241
 242   // Vector Subtract High-Half
 243   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 244   // header is no longer supported.
 245   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 246
 247   // Vector Rounding Subtract High-Half
 248   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 249
 250   // Vector Compare Absolute Greater-than-or-equal
 251   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 252
 253   // Vector Compare Absolute Greater-than
 254   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 255
 256   // Vector Absolute Difference
 257   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
 258   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
 259   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
 260
 261   // Scalar Absolute Difference
 262   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
 263
 264   // Vector Max
 265   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
 266   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
 267   def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
 268   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
 269
 270   // Vector Max Across Lanes
 271   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 272   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 273   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 274   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 275
 276   // Vector Min
 277   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
 278   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
 279   def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
 280   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
 281
 282   // Vector Min/Max Number
 283   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
 284   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
 285
 286   // Vector Min Across Lanes
 287   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 288   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 289   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 290   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 291
 292   // Pairwise Add
 293   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
 294   def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;
 295
 296   // Long Pairwise Add
 297   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
 298   // uaddlp, but tblgen's type inference currently can't handle the
 299   // pattern fragments this ends up generating.
 300   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 301   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 302
 303   // Folding Maximum
 304   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
 305   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
 306   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
 307
 308   // Folding Minimum
 309   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
 310   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
 311   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
 312
 313   // Reciprocal Estimate/Step
 314   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
 315   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
 316
 317   // Reciprocal Exponent
 318   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
 319
 320   // Vector Saturating Shift Left
 321   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
 322   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
 323
 324   // Vector Rounding Shift Left
 325   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
 326   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
 327
 328   // Vector Saturating Rounding Shift Left
 329   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
 330   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
 331
 332   // Vector Signed->Unsigned Shift Left by Constant
 333   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
 334
 335   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
 336   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 337
 338   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
 339   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 340
 341   // Vector Narrowing Shift Right by Constant
 342   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 343   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 344
 345   // Vector Rounding Narrowing Shift Right by Constant
 346   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 347
 348   // Vector Rounding Narrowing Saturating Shift Right by Constant
 349   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 350   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 351
 352   // Vector Shift Left
 353   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
 354   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
 355
 356   // Vector Widening Shift Left by Constant
 357   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
 358   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 359   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 360
 361   // Vector Shift Right by Constant and Insert
 362   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 363
 364   // Vector Shift Left by Constant and Insert
 365   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 366
 367   // Vector Saturating Narrow
 368   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
 369   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
 370   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 371   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 372
 373   // Vector Saturating Extract and Unsigned Narrow
 374   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
 375   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 376
 377   // Vector Absolute Value
 378   def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
 379
 380   // Vector Saturating Absolute Value
 381   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
 382
 383   // Vector Saturating Negation
 384   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
 385
 386   // Vector Count Leading Sign Bits
 387   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
 388
 389   // Vector Reciprocal Estimate
 390   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
 391   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
 392
 393   // Vector Square Root Estimate
 394   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
 395   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
 396
 397   // Vector Bitwise Reverse
 398   def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
 399
 400   // Vector Conversions Between Half-Precision and Single-Precision.
 401   def int_aarch64_neon_vcvtfp2hf
 402     : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 403   def int_aarch64_neon_vcvthf2fp
 404     : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
 405
 406   // Vector Conversions Between Floating-point and Fixed-point.
 407   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
 408   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
 409   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 410   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 411
 412   // Vector FP->Int Conversions
 413   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
 414   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
 415   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
 416   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
 417   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
 418   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
 419   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
 420   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
 421   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
 422   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 423
 424   // Vector FP Rounding: only ties to even is unrepresented by a normal
 425   // intrinsic.
 426   def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
 427
 428   // Scalar FP->Int conversions
 429
 430   // Vector FP Inexact Narrowing
 431   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
 432
 433   // Scalar FP Inexact Narrowing
 434   def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
 435                                         [IntrNoMem]>;
 436
 437   // v8.2-A Dot Product
 438   def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
 439   def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
 440
 441   // v8.2-A FP16 Fused Multiply-Add Long
 442   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
 443   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
 444   def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
 445   def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
 446 }
 447
 448 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 449   class AdvSIMD_2Vector2Index_Intrinsic
 450     : Intrinsic<[llvm_anyvector_ty],
 451                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
 452                 [IntrNoMem]>;
 453 }
 454
 455 // Vector element to element moves
 456 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
 457
 458 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 459   class AdvSIMD_1Vec_Load_Intrinsic
 460       : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
 461                   [IntrReadMem, IntrArgMemOnly]>;
 462   class AdvSIMD_1Vec_Store_Lane_Intrinsic
 463     : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
 464                 [IntrArgMemOnly, NoCapture<2>]>;
 465
 466   class AdvSIMD_2Vec_Load_Intrinsic
 467     : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
 468                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 469                 [IntrReadMem, IntrArgMemOnly]>;
 470   class AdvSIMD_2Vec_Load_Lane_Intrinsic
 471     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
 472                 [LLVMMatchType<0>, llvm_anyvector_ty,
 473                  llvm_i64_ty, llvm_anyptr_ty],
 474                 [IntrReadMem, IntrArgMemOnly]>;
 475   class AdvSIMD_2Vec_Store_Intrinsic
 476     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 477                      LLVMAnyPointerType<LLVMMatchType<0>>],
 478                 [IntrArgMemOnly, NoCapture<2>]>;
 479   class AdvSIMD_2Vec_Store_Lane_Intrinsic
 480     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 481                  llvm_i64_ty, llvm_anyptr_ty],
 482                 [IntrArgMemOnly, NoCapture<3>]>;
 483
 484   class AdvSIMD_3Vec_Load_Intrinsic
 485     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
 486                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 487                 [IntrReadMem, IntrArgMemOnly]>;
 488   class AdvSIMD_3Vec_Load_Lane_Intrinsic
 489     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 490                 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
 491                  llvm_i64_ty, llvm_anyptr_ty],
 492                 [IntrReadMem, IntrArgMemOnly]>;
 493   class AdvSIMD_3Vec_Store_Intrinsic
 494     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 495                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
 496                 [IntrArgMemOnly, NoCapture<3>]>;
 497   class AdvSIMD_3Vec_Store_Lane_Intrinsic
 498     : Intrinsic<[], [llvm_anyvector_ty,
 499                  LLVMMatchType<0>, LLVMMatchType<0>,
 500                  llvm_i64_ty, llvm_anyptr_ty],
 501                 [IntrArgMemOnly, NoCapture<4>]>;
 502
 503   class AdvSIMD_4Vec_Load_Intrinsic
 504     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 505                  LLVMMatchType<0>, llvm_anyvector_ty],
 506                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 507                 [IntrReadMem, IntrArgMemOnly]>;
 508   class AdvSIMD_4Vec_Load_Lane_Intrinsic
 509     : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
 510                  LLVMMatchType<0>, LLVMMatchType<0>],
 511                 [LLVMMatchType<0>, LLVMMatchType<0>,
 512                  LLVMMatchType<0>, llvm_anyvector_ty,
 513                  llvm_i64_ty, llvm_anyptr_ty],
 514                 [IntrReadMem, IntrArgMemOnly]>;
 515   class AdvSIMD_4Vec_Store_Intrinsic
 516     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 517                  LLVMMatchType<0>, LLVMMatchType<0>,
 518                  LLVMAnyPointerType<LLVMMatchType<0>>],
 519                 [IntrArgMemOnly, NoCapture<4>]>;
 520   class AdvSIMD_4Vec_Store_Lane_Intrinsic
 521     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 522                  LLVMMatchType<0>, LLVMMatchType<0>,
 523                  llvm_i64_ty, llvm_anyptr_ty],
 524                 [IntrArgMemOnly, NoCapture<5>]>;
 525 }
 526
 527 // Memory ops
 528
 529 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
 530 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
 531 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
 532
 533 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
 534 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
 535 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
 536
 537 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
 538 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
 539 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
 540
 541 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
 542 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
 543 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
 544
 545 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
 546 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
 547 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
 548
 549 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
 550 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
 551 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
 552
 553 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
 554 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
 555 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
 556
 557 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 558   class AdvSIMD_Tbl1_Intrinsic
 559     : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
 560                 [IntrNoMem]>;
 561   class AdvSIMD_Tbl2_Intrinsic
 562     : Intrinsic<[llvm_anyvector_ty],
 563                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
 564   class AdvSIMD_Tbl3_Intrinsic
 565     : Intrinsic<[llvm_anyvector_ty],
 566                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 567                  LLVMMatchType<0>],
 568                 [IntrNoMem]>;
 569   class AdvSIMD_Tbl4_Intrinsic
 570     : Intrinsic<[llvm_anyvector_ty],
 571                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 572                  LLVMMatchType<0>],
 573                 [IntrNoMem]>;
 574
 575   class AdvSIMD_Tbx1_Intrinsic
 576     : Intrinsic<[llvm_anyvector_ty],
 577                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
 578                 [IntrNoMem]>;
 579   class AdvSIMD_Tbx2_Intrinsic
 580     : Intrinsic<[llvm_anyvector_ty],
 581                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 582                  LLVMMatchType<0>],
 583                 [IntrNoMem]>;
 584   class AdvSIMD_Tbx3_Intrinsic
 585     : Intrinsic<[llvm_anyvector_ty],
 586                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 587                  llvm_v16i8_ty, LLVMMatchType<0>],
 588                 [IntrNoMem]>;
 589   class AdvSIMD_Tbx4_Intrinsic
 590     : Intrinsic<[llvm_anyvector_ty],
 591                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 592                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
 593                 [IntrNoMem]>;
 594 }
 595 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
 596 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
 597 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
 598 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
 599
 600 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 601 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 602 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 603 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 604
 605 let TargetPrefix = "aarch64" in {
 606   class FPCR_Get_Intrinsic
 607     : Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>;
 608 }
 609
 610 // FPCR
 611 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
 612
 613 let TargetPrefix = "aarch64" in {
 614   class Crypto_AES_DataKey_Intrinsic
 615     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 616
 617   class Crypto_AES_Data_Intrinsic
 618     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 619
 620   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 621   // (v4i32).
 622   class Crypto_SHA_5Hash4Schedule_Intrinsic
 623     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
 624                 [IntrNoMem]>;
 625
 626   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 627   // (v4i32).
 628   class Crypto_SHA_1Hash_Intrinsic
 629     : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 630
 631   // SHA intrinsic taking 8 words of the schedule
 632   class Crypto_SHA_8Schedule_Intrinsic
 633     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 634
 635   // SHA intrinsic taking 12 words of the schedule
 636   class Crypto_SHA_12Schedule_Intrinsic
 637     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 638                 [IntrNoMem]>;
 639
 640   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
 641   class Crypto_SHA_8Hash4Schedule_Intrinsic
 642     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 643                 [IntrNoMem]>;
 644 }
 645
 646 // AES
 647 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
 648 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
 649 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
 650 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
 651
 652 // SHA1
 653 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 654 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 655 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 656 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
 657
 658 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
 659 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
 660
 661 // SHA256
 662 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
 663 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
 664 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
 665 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
 666
 667 //===----------------------------------------------------------------------===//
 668 // CRC32
 669
 670 let TargetPrefix = "aarch64" in {
 671
 672 def int_aarch64_crc32b  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 673     [IntrNoMem]>;
 674 def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 675     [IntrNoMem]>;
 676 def int_aarch64_crc32h  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 677     [IntrNoMem]>;
 678 def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 679     [IntrNoMem]>;
 680 def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 681     [IntrNoMem]>;
 682 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 683     [IntrNoMem]>;
 684 def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 685     [IntrNoMem]>;
 686 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 687     [IntrNoMem]>;
 688 }
 689
 690 //===----------------------------------------------------------------------===//
 691 // Memory Tagging Extensions (MTE) Intrinsics
 692 let TargetPrefix = "aarch64" in {
 693 def int_aarch64_irg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 694     [IntrNoMem, IntrHasSideEffects]>;
 695 def int_aarch64_addg  : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
 696     [IntrNoMem]>;
 697 def int_aarch64_gmi   : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
 698     [IntrNoMem]>;
 699 def int_aarch64_ldg   : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
 700     [IntrReadMem]>;
 701 def int_aarch64_stg   : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
 702     [IntrWriteMem]>;
 703 def int_aarch64_subp :  Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
 704     [IntrNoMem]>;
 705
 706 // The following are codegen-only intrinsics for stack instrumentation.
 707
 708 // Generate a randomly tagged stack base pointer.
 709 def int_aarch64_irg_sp   : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty],
 710     [IntrNoMem, IntrHasSideEffects]>;
 711
 712 // Transfer pointer tag with offset.
 713 // ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
 714 // * address is the address in ptr0
 715 // * tag is a function of (tag in baseptr, tag_offset).
 716 // Address bits in baseptr and tag bits in ptr0 are ignored.
 717 // When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
 718 //   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
 719 // It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
 720 def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
 721     [IntrNoMem, ImmArg<2>]>;
 722
 723 // Update allocation tags for the memory range to match the tag in the pointer argument.
 724 def int_aarch64_settag  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 725     [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>;
 726
 727 // Update allocation tags for the memory range to match the tag in the pointer argument,
 728 // and set memory contents to zero.
 729 def int_aarch64_settag_zero  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
 730     [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>;
 731
 732 // Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
 733 def int_aarch64_stgp  : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
 734     [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>;
 735 }
 736
 737 // Transactional Memory Extension (TME) Intrinsics
 738 let TargetPrefix = "aarch64" in {
 739 def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
 740                          Intrinsic<[llvm_i64_ty]>;
 741
 742 def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>;
 743
 744 def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
 745                           Intrinsic<[], [llvm_i64_ty], [ImmArg<0>]>;
 746
 747 def int_aarch64_ttest   : GCCBuiltin<"__builtin_arm_ttest">,
 748                           Intrinsic<[llvm_i64_ty], [],
 749                                     [IntrNoMem, IntrHasSideEffects]>;
 750 }
 751
 752 def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
 753 def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
 754 def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
 755 def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
 756 def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
 757 def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
 758 def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
 759 def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
 760 def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
 761 def llvm_nxv2f64_ty : LLVMType<nxv2f64>;
 762
 763 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 764   class AdvSIMD_Merged1VectorArg_Intrinsic
 765     : Intrinsic<[llvm_anyvector_ty],
 766                 [LLVMMatchType<0>,
 767                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 768                  LLVMMatchType<0>],
 769                 [IntrNoMem]>;
 770
 771   class AdvSIMD_SVE_CNT_Intrinsic
 772     : Intrinsic<[LLVMVectorOfBitcastsToInt<0>],
 773                 [LLVMVectorOfBitcastsToInt<0>,
 774                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
 775                  llvm_anyvector_ty],
 776                 [IntrNoMem]>;
 777
 778   class AdvSIMD_SVE_Unpack_Intrinsic
 779    : Intrinsic<[llvm_anyvector_ty],
 780                [LLVMSubdivide2VectorType<0>],
 781                [IntrNoMem]>;
 782
 783   class AdvSIMD_SVE_PUNPKHI_Intrinsic
 784     : Intrinsic<[LLVMHalfElementsVectorType<0>],
 785                 [llvm_anyvector_ty],
 786                 [IntrNoMem]>;
 787
 788   class AdvSIMD_SVE_DOT_Intrinsic
 789     : Intrinsic<[llvm_anyvector_ty],
 790                 [LLVMMatchType<0>,
 791                  LLVMSubdivide4VectorType<0>,
 792                  LLVMSubdivide4VectorType<0>],
 793                 [IntrNoMem]>;
 794
 795   class AdvSIMD_SVE_DOT_Indexed_Intrinsic
 796     : Intrinsic<[llvm_anyvector_ty],
 797                 [LLVMMatchType<0>,
 798                  LLVMSubdivide4VectorType<0>,
 799                  LLVMSubdivide4VectorType<0>,
 800                  llvm_i32_ty],
 801                 [IntrNoMem]>;
 802
 803   // This class of intrinsics are not intended to be useful within LLVM IR but
 804   // are instead here to support some of the more regid parts of the ACLE.
 805   class Builtin_SVCVT<string name, LLVMType OUT, LLVMType IN>
 806   : GCCBuiltin<"__builtin_sve_" # name>,
 807     Intrinsic<[OUT], [OUT, llvm_nxv16i1_ty, IN], [IntrNoMem]>;
 808 }
 809
 810 //===----------------------------------------------------------------------===//
 811 // SVE
 812
 813 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 814
 815 //
 816 // Integer arithmetic
 817 //
 818
 819 def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
 820 def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;
 821
 822 def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
 823 def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
 824
 825 def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
 826 def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
 827
 828 //
 829 // Counting bits
 830 //
 831
 832 def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;
 833
 834 //
 835 // Permutations and selection
 836 //
 837
 838 def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
 839 def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
 840
 841 def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
 842 def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
 843
 844 //
 845 // Floating-point comparisons
 846 //
 847
 848 def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv8f16_ty>;
 849
 850 //
 851 // Predicate operations
 852 //
 853
 854 def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 855 def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;
 856 }