include/llvm/IR/IntrinsicsAArch64.td

   1 //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines all of the AARCH64-specific intrinsics.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 let TargetPrefix = "aarch64" in {
  14
  15 def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  16 def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>;
  17 def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  18 def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>;
  19
  20 def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  21 def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>;
  22 def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
  23                                [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  24 def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
  25                                 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>;
  26
  27 def int_aarch64_clrex : Intrinsic<[]>;
  28
  29 def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  30                                 LLVMMatchType<0>], [IntrNoMem]>;
  31 def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
  32                                 LLVMMatchType<0>], [IntrNoMem]>;
  33
  34 //===----------------------------------------------------------------------===//
  35 // HINT
  36
  37 def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>;
  38
  39 //===----------------------------------------------------------------------===//
  40 // Data Barrier Instructions
  41
  42 def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>;
  43 def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>;
  44 def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>;
  45
  46 // A space-consuming intrinsic primarily for testing block and jump table
  47 // placements. The first argument is the number of bytes this "instruction"
  48 // takes up, the second and return value are essentially chains, used to force
  49 // ordering during ISel.
  50 def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;
  51
  52 }
  53
  54 //===----------------------------------------------------------------------===//
  55 // Advanced SIMD (NEON)
  56
  57 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  58   class AdvSIMD_2Scalar_Float_Intrinsic
  59     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  60                 [IntrNoMem]>;
  61
  62   class AdvSIMD_FPToIntRounding_Intrinsic
  63     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;
  64
  65   class AdvSIMD_1IntArg_Intrinsic
  66     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  67   class AdvSIMD_1FloatArg_Intrinsic
  68     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  69   class AdvSIMD_1VectorArg_Intrinsic
  70     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  71   class AdvSIMD_1VectorArg_Expand_Intrinsic
  72     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  73   class AdvSIMD_1VectorArg_Long_Intrinsic
  74     : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
  75   class AdvSIMD_1IntArg_Narrow_Intrinsic
  76     : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
  77   class AdvSIMD_1VectorArg_Narrow_Intrinsic
  78     : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
  79   class AdvSIMD_1VectorArg_Int_Across_Intrinsic
  80     : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  81   class AdvSIMD_1VectorArg_Float_Across_Intrinsic
  82     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  83
  84   class AdvSIMD_2IntArg_Intrinsic
  85     : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  86                 [IntrNoMem]>;
  87   class AdvSIMD_2FloatArg_Intrinsic
  88     : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  89                 [IntrNoMem]>;
  90   class AdvSIMD_2VectorArg_Intrinsic
  91     : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
  92                 [IntrNoMem]>;
  93   class AdvSIMD_2VectorArg_Compare_Intrinsic
  94     : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
  95                 [IntrNoMem]>;
  96   class AdvSIMD_2Arg_FloatCompare_Intrinsic
  97     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
  98                 [IntrNoMem]>;
  99   class AdvSIMD_2VectorArg_Long_Intrinsic
 100     : Intrinsic<[llvm_anyvector_ty],
 101                 [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
 102                 [IntrNoMem]>;
 103   class AdvSIMD_2VectorArg_Wide_Intrinsic
 104     : Intrinsic<[llvm_anyvector_ty],
 105                 [LLVMMatchType<0>, LLVMTruncatedType<0>],
 106                 [IntrNoMem]>;
 107   class AdvSIMD_2VectorArg_Narrow_Intrinsic
 108     : Intrinsic<[llvm_anyvector_ty],
 109                 [LLVMExtendedType<0>, LLVMExtendedType<0>],
 110                 [IntrNoMem]>;
 111   class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
 112     : Intrinsic<[llvm_anyint_ty],
 113                 [LLVMExtendedType<0>, llvm_i32_ty],
 114                 [IntrNoMem]>;
 115   class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
 116     : Intrinsic<[llvm_anyvector_ty],
 117                 [llvm_anyvector_ty],
 118                 [IntrNoMem]>;
 119   class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
 120     : Intrinsic<[llvm_anyvector_ty],
 121                 [LLVMTruncatedType<0>],
 122                 [IntrNoMem]>;
 123   class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
 124     : Intrinsic<[llvm_anyvector_ty],
 125                 [LLVMTruncatedType<0>, llvm_i32_ty],
 126                 [IntrNoMem]>;
 127   class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
 128     : Intrinsic<[llvm_anyvector_ty],
 129                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
 130                 [IntrNoMem]>;
 131
 132   class AdvSIMD_3VectorArg_Intrinsic
 133       : Intrinsic<[llvm_anyvector_ty],
 134                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
 135                [IntrNoMem]>;
 136   class AdvSIMD_3VectorArg_Scalar_Intrinsic
 137       : Intrinsic<[llvm_anyvector_ty],
 138                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
 139                [IntrNoMem]>;
 140   class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
 141       : Intrinsic<[llvm_anyvector_ty],
 142                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
 143                 LLVMMatchType<1>], [IntrNoMem]>;
 144   class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
 145     : Intrinsic<[llvm_anyvector_ty],
 146                 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
 147                 [IntrNoMem]>;
 148   class AdvSIMD_CvtFxToFP_Intrinsic
 149     : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
 150                 [IntrNoMem]>;
 151   class AdvSIMD_CvtFPToFx_Intrinsic
 152     : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
 153                 [IntrNoMem]>;
 154
 155   class AdvSIMD_1Arg_Intrinsic
 156     : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;
 157
 158   class AdvSIMD_Dot_Intrinsic
 159     : Intrinsic<[llvm_anyvector_ty],
 160                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 161                 [IntrNoMem]>;
 162
 163   class AdvSIMD_FP16FML_Intrinsic
 164     : Intrinsic<[llvm_anyvector_ty],
 165                 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
 166                 [IntrNoMem]>;
 167 }
 168
 169 // Arithmetic ops
 170
 171 let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
 172   // Vector Add Across Lanes
 173   def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 174   def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 175   def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 176
 177   // Vector Long Add Across Lanes
 178   def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 179   def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 180
 181   // Vector Halving Add
 182   def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
 183   def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;
 184
 185   // Vector Rounding Halving Add
 186   def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
 187   def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;
 188
 189   // Vector Saturating Add
 190   def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
 191   def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
 192   def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
 193   def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;
 194
 195   // Vector Add High-Half
 196   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 197   // header is no longer supported.
 198   def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 199
 200   // Vector Rounding Add High-Half
 201   def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 202
 203   // Vector Saturating Doubling Multiply High
 204   def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
 205
 206   // Vector Saturating Rounding Doubling Multiply High
 207   def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
 208
 209   // Vector Polynominal Multiply
 210   def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;
 211
 212   // Vector Long Multiply
 213   def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
 214   def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
 215   def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 216
 217   // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
 218   // it with a v16i8.
 219   def int_aarch64_neon_pmull64 :
 220         Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;
 221
 222   // Vector Extending Multiply
 223   def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
 224     let IntrProperties = [IntrNoMem, Commutative];
 225   }
 226
 227   // Vector Saturating Doubling Long Multiply
 228   def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
 229   def int_aarch64_neon_sqdmulls_scalar
 230     : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;
 231
 232   // Vector Halving Subtract
 233   def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
 234   def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;
 235
 236   // Vector Saturating Subtract
 237   def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
 238   def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;
 239
 240   // Vector Subtract High-Half
 241   // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
 242   // header is no longer supported.
 243   def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 244
 245   // Vector Rounding Subtract High-Half
 246   def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;
 247
 248   // Vector Compare Absolute Greater-than-or-equal
 249   def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 250
 251   // Vector Compare Absolute Greater-than
 252   def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;
 253
 254   // Vector Absolute Difference
 255   def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
 256   def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
 257   def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;
 258
 259   // Scalar Absolute Difference
 260   def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;
 261
 262   // Vector Max
 263   def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
 264   def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
 265   def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
 266   def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;
 267
 268   // Vector Max Across Lanes
 269   def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 270   def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 271   def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 272   def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 273
 274   // Vector Min
 275   def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
 276   def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
 277   def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
 278   def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;
 279
 280   // Vector Min/Max Number
 281   def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
 282   def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;
 283
 284   // Vector Min Across Lanes
 285   def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 286   def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
 287   def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 288   def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
 289
 290   // Pairwise Add
 291   def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
 292
 293   // Long Pairwise Add
 294   // FIXME: In theory, we shouldn't need intrinsics for saddlp or
 295   // uaddlp, but tblgen's type inference currently can't handle the
 296   // pattern fragments this ends up generating.
 297   def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 298   def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
 299
 300   // Folding Maximum
 301   def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
 302   def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
 303   def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;
 304
 305   // Folding Minimum
 306   def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
 307   def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
 308   def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;
 309
 310   // Reciprocal Estimate/Step
 311   def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
 312   def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;
 313
 314   // Reciprocal Exponent
 315   def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;
 316
 317   // Vector Saturating Shift Left
 318   def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
 319   def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;
 320
 321   // Vector Rounding Shift Left
 322   def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
 323   def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;
 324
 325   // Vector Saturating Rounding Shift Left
 326   def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
 327   def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;
 328
 329   // Vector Signed->Unsigned Shift Left by Constant
 330   def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;
 331
 332   // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
 333   def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 334
 335   // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
 336   def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 337
 338   // Vector Narrowing Shift Right by Constant
 339   def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 340   def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 341
 342   // Vector Rounding Narrowing Shift Right by Constant
 343   def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 344
 345   // Vector Rounding Narrowing Saturating Shift Right by Constant
 346   def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 347   def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
 348
 349   // Vector Shift Left
 350   def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
 351   def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;
 352
 353   // Vector Widening Shift Left by Constant
 354   def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
 355   def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 356   def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
 357
 358   // Vector Shift Right by Constant and Insert
 359   def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 360
 361   // Vector Shift Left by Constant and Insert
 362   def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;
 363
 364   // Vector Saturating Narrow
 365   def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
 366   def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
 367   def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 368   def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 369
 370   // Vector Saturating Extract and Unsigned Narrow
 371   def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
 372   def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;
 373
 374   // Vector Absolute Value
 375   def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;
 376
 377   // Vector Saturating Absolute Value
 378   def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;
 379
 380   // Vector Saturating Negation
 381   def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;
 382
 383   // Vector Count Leading Sign Bits
 384   def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;
 385
 386   // Vector Reciprocal Estimate
 387   def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
 388   def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;
 389
 390   // Vector Square Root Estimate
 391   def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
 392   def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;
 393
 394   // Vector Bitwise Reverse
 395   def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic;
 396
 397   // Vector Conversions Between Half-Precision and Single-Precision.
 398   def int_aarch64_neon_vcvtfp2hf
 399     : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
 400   def int_aarch64_neon_vcvthf2fp
 401     : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;
 402
 403   // Vector Conversions Between Floating-point and Fixed-point.
 404   def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
 405   def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
 406   def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 407   def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;
 408
 409   // Vector FP->Int Conversions
 410   def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
 411   def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
 412   def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
 413   def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
 414   def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
 415   def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
 416   def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
 417   def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
 418   def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
 419   def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;
 420
 421   // Vector FP Rounding: only ties to even is unrepresented by a normal
 422   // intrinsic.
 423   def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic;
 424
 425   // Scalar FP->Int conversions
 426
 427   // Vector FP Inexact Narrowing
 428   def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;
 429
 430   // Scalar FP Inexact Narrowing
 431   def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty],
 432                                         [IntrNoMem]>;
 433
 434   // v8.2-A Dot Product
 435   def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
 436   def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;
 437
 438   // v8.2-A FP16 Fused Multiply-Add Long
 439   def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
 440   def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
 441   def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
 442   def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;
 443 }
 444
 445 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 446   class AdvSIMD_2Vector2Index_Intrinsic
 447     : Intrinsic<[llvm_anyvector_ty],
 448                 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
 449                 [IntrNoMem]>;
 450 }
 451
 452 // Vector element to element moves
 453 def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;
 454
 455 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 456   class AdvSIMD_1Vec_Load_Intrinsic
 457       : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
 458                   [IntrReadMem, IntrArgMemOnly]>;
 459   class AdvSIMD_1Vec_Store_Lane_Intrinsic
 460     : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
 461                 [IntrArgMemOnly, NoCapture<2>]>;
 462
 463   class AdvSIMD_2Vec_Load_Intrinsic
 464     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
 465                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 466                 [IntrReadMem, IntrArgMemOnly]>;
 467   class AdvSIMD_2Vec_Load_Lane_Intrinsic
 468     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
 469                 [LLVMMatchType<0>, LLVMMatchType<0>,
 470                  llvm_i64_ty, llvm_anyptr_ty],
 471                 [IntrReadMem, IntrArgMemOnly]>;
 472   class AdvSIMD_2Vec_Store_Intrinsic
 473     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 474                      LLVMAnyPointerType<LLVMMatchType<0>>],
 475                 [IntrArgMemOnly, NoCapture<2>]>;
 476   class AdvSIMD_2Vec_Store_Lane_Intrinsic
 477     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 478                  llvm_i64_ty, llvm_anyptr_ty],
 479                 [IntrArgMemOnly, NoCapture<3>]>;
 480
 481   class AdvSIMD_3Vec_Load_Intrinsic
 482     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
 483                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 484                 [IntrReadMem, IntrArgMemOnly]>;
 485   class AdvSIMD_3Vec_Load_Lane_Intrinsic
 486     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
 487                 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>,
 488                  llvm_i64_ty, llvm_anyptr_ty],
 489                 [IntrReadMem, IntrArgMemOnly]>;
 490   class AdvSIMD_3Vec_Store_Intrinsic
 491     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 492                      LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
 493                 [IntrArgMemOnly, NoCapture<3>]>;
 494   class AdvSIMD_3Vec_Store_Lane_Intrinsic
 495     : Intrinsic<[], [llvm_anyvector_ty,
 496                  LLVMMatchType<0>, LLVMMatchType<0>,
 497                  llvm_i64_ty, llvm_anyptr_ty],
 498                 [IntrArgMemOnly, NoCapture<4>]>;
 499
 500   class AdvSIMD_4Vec_Load_Intrinsic
 501     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
 502                  LLVMMatchType<0>, LLVMMatchType<0>],
 503                 [LLVMAnyPointerType<LLVMMatchType<0>>],
 504                 [IntrReadMem, IntrArgMemOnly]>;
 505   class AdvSIMD_4Vec_Load_Lane_Intrinsic
 506     : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>,
 507                  LLVMMatchType<0>, LLVMMatchType<0>],
 508                 [LLVMMatchType<0>, LLVMMatchType<0>,
 509                  LLVMMatchType<0>, LLVMMatchType<0>,
 510                  llvm_i64_ty, llvm_anyptr_ty],
 511                 [IntrReadMem, IntrArgMemOnly]>;
 512   class AdvSIMD_4Vec_Store_Intrinsic
 513     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 514                  LLVMMatchType<0>, LLVMMatchType<0>,
 515                  LLVMAnyPointerType<LLVMMatchType<0>>],
 516                 [IntrArgMemOnly, NoCapture<4>]>;
 517   class AdvSIMD_4Vec_Store_Lane_Intrinsic
 518     : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
 519                  LLVMMatchType<0>, LLVMMatchType<0>,
 520                  llvm_i64_ty, llvm_anyptr_ty],
 521                 [IntrArgMemOnly, NoCapture<5>]>;
 522 }
 523
 524 // Memory ops
 525
 526 def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
 527 def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
 528 def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;
 529
 530 def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
 531 def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
 532 def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;
 533
 534 def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
 535 def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
 536 def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;
 537
 538 def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
 539 def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
 540 def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;
 541
 542 def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
 543 def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
 544 def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;
 545
 546 def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
 547 def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
 548 def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;
 549
 550 def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
 551 def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
 552 def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;
 553
 554 let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
 555   class AdvSIMD_Tbl1_Intrinsic
 556     : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
 557                 [IntrNoMem]>;
 558   class AdvSIMD_Tbl2_Intrinsic
 559     : Intrinsic<[llvm_anyvector_ty],
 560                 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
 561   class AdvSIMD_Tbl3_Intrinsic
 562     : Intrinsic<[llvm_anyvector_ty],
 563                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 564                  LLVMMatchType<0>],
 565                 [IntrNoMem]>;
 566   class AdvSIMD_Tbl4_Intrinsic
 567     : Intrinsic<[llvm_anyvector_ty],
 568                 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
 569                  LLVMMatchType<0>],
 570                 [IntrNoMem]>;
 571
 572   class AdvSIMD_Tbx1_Intrinsic
 573     : Intrinsic<[llvm_anyvector_ty],
 574                 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
 575                 [IntrNoMem]>;
 576   class AdvSIMD_Tbx2_Intrinsic
 577     : Intrinsic<[llvm_anyvector_ty],
 578                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 579                  LLVMMatchType<0>],
 580                 [IntrNoMem]>;
 581   class AdvSIMD_Tbx3_Intrinsic
 582     : Intrinsic<[llvm_anyvector_ty],
 583                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 584                  llvm_v16i8_ty, LLVMMatchType<0>],
 585                 [IntrNoMem]>;
 586   class AdvSIMD_Tbx4_Intrinsic
 587     : Intrinsic<[llvm_anyvector_ty],
 588                 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
 589                  llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
 590                 [IntrNoMem]>;
 591 }
 592 def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
 593 def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
 594 def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
 595 def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;
 596
 597 def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
 598 def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
 599 def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
 600 def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;
 601
 602 let TargetPrefix = "aarch64" in {
 603   class FPCR_Get_Intrinsic
 604     : Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>;
 605 }
 606
 607 // FPCR
 608 def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
 609
 610 let TargetPrefix = "aarch64" in {
 611   class Crypto_AES_DataKey_Intrinsic
 612     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
 613
 614   class Crypto_AES_Data_Intrinsic
 615     : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;
 616
 617   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 618   // (v4i32).
 619   class Crypto_SHA_5Hash4Schedule_Intrinsic
 620     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
 621                 [IntrNoMem]>;
 622
 623   // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
 624   // (v4i32).
 625   class Crypto_SHA_1Hash_Intrinsic
 626     : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
 627
 628   // SHA intrinsic taking 8 words of the schedule
 629   class Crypto_SHA_8Schedule_Intrinsic
 630     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
 631
 632   // SHA intrinsic taking 12 words of the schedule
 633   class Crypto_SHA_12Schedule_Intrinsic
 634     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 635                 [IntrNoMem]>;
 636
 637   // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
 638   class Crypto_SHA_8Hash4Schedule_Intrinsic
 639     : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
 640                 [IntrNoMem]>;
 641 }
 642
 643 // AES
 644 def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
 645 def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
 646 def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
 647 def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;
 648
 649 // SHA1
 650 def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 651 def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 652 def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
 653 def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;
 654
 655 def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
 656 def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;
 657
 658 // SHA256
 659 def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
 660 def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
 661 def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
 662 def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;
 663
 664 //===----------------------------------------------------------------------===//
 665 // CRC32
 666
 667 let TargetPrefix = "aarch64" in {
 668
 669 def int_aarch64_crc32b  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 670     [IntrNoMem]>;
 671 def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 672     [IntrNoMem]>;
 673 def int_aarch64_crc32h  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 674     [IntrNoMem]>;
 675 def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 676     [IntrNoMem]>;
 677 def int_aarch64_crc32w  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 678     [IntrNoMem]>;
 679 def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
 680     [IntrNoMem]>;
 681 def int_aarch64_crc32x  : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 682     [IntrNoMem]>;
 683 def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
 684     [IntrNoMem]>;
 685 }