clang/lib/Headers/immintrin.h

   1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
   2  *
   3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4  * See https://llvm.org/LICENSE.txt for license information.
   5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6  *
   7  *===-----------------------------------------------------------------------===
   8  */
   9
  10 #ifndef __IMMINTRIN_H
  11 #define __IMMINTRIN_H
  12
  13 #if !defined(__i386__) && !defined(__x86_64__)
  14 #error "This header is only meant to be used on x86 and x64 architecture"
  15 #endif
  16
  17 #include <x86gprintrin.h>
  18
  19 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  20     defined(__MMX__)
  21 #include <mmintrin.h>
  22 #endif
  23
  24 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  25     defined(__SSE__)
  26 #include <xmmintrin.h>
  27 #endif
  28
  29 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  30     defined(__SSE2__)
  31 #include <emmintrin.h>
  32 #endif
  33
  34 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  35     defined(__SSE3__)
  36 #include <pmmintrin.h>
  37 #endif
  38
  39 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  40     defined(__SSSE3__)
  41 #include <tmmintrin.h>
  42 #endif
  43
  44 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  45     (defined(__SSE4_2__) || defined(__SSE4_1__))
  46 #include <smmintrin.h>
  47 #endif
  48
  49 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  50     (defined(__AES__) || defined(__PCLMUL__))
  51 #include <wmmintrin.h>
  52 #endif
  53
  54 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  55     defined(__CLFLUSHOPT__)
  56 #include <clflushoptintrin.h>
  57 #endif
  58
  59 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  60     defined(__CLWB__)
  61 #include <clwbintrin.h>
  62 #endif
  63
  64 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  65     defined(__AVX__)
  66 #include <avxintrin.h>
  67 #endif
  68
  69 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  70     defined(__AVX2__)
  71 #include <avx2intrin.h>
  72 #endif
  73
  74 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  75     defined(__F16C__)
  76 #include <f16cintrin.h>
  77 #endif
  78
  79 /* No feature check desired due to internal checks */
  80 #include <bmiintrin.h>
  81
  82 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  83     defined(__BMI2__)
  84 #include <bmi2intrin.h>
  85 #endif
  86
  87 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  88     defined(__LZCNT__)
  89 #include <lzcntintrin.h>
  90 #endif
  91
  92 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  93     defined(__POPCNT__)
  94 #include <popcntintrin.h>
  95 #endif
  96
  97 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  98     defined(__FMA__)
  99 #include <fmaintrin.h>
 100 #endif
 101
 102 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 103     defined(__AVX512F__)
 104 #include <avx512fintrin.h>
 105 #endif
 106
 107 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 108     defined(__AVX512VL__)
 109 #include <avx512vlintrin.h>
 110 #endif
 111
 112 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 113     defined(__AVX512BW__)
 114 #include <avx512bwintrin.h>
 115 #endif
 116
 117 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 118     defined(__AVX512BITALG__)
 119 #include <avx512bitalgintrin.h>
 120 #endif
 121
 122 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 123     defined(__AVX512CD__)
 124 #include <avx512cdintrin.h>
 125 #endif
 126
 127 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 128     defined(__AVX512VPOPCNTDQ__)
 129 #include <avx512vpopcntdqintrin.h>
 130 #endif
 131
 132 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 133     (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
 134 #include <avx512vpopcntdqvlintrin.h>
 135 #endif
 136
 137 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 138     defined(__AVX512VNNI__)
 139 #include <avx512vnniintrin.h>
 140 #endif
 141
 142 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 143     (defined(__AVX512VL__) && defined(__AVX512VNNI__))
 144 #include <avx512vlvnniintrin.h>
 145 #endif
 146
 147 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 148     defined(__AVXVNNI__)
 149 #include <avxvnniintrin.h>
 150 #endif
 151
 152 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 153     defined(__AVX512DQ__)
 154 #include <avx512dqintrin.h>
 155 #endif
 156
 157 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 158     (defined(__AVX512VL__) && defined(__AVX512BITALG__))
 159 #include <avx512vlbitalgintrin.h>
 160 #endif
 161
 162 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 163     (defined(__AVX512VL__) && defined(__AVX512BW__))
 164 #include <avx512vlbwintrin.h>
 165 #endif
 166
 167 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 168     (defined(__AVX512VL__) && defined(__AVX512CD__))
 169 #include <avx512vlcdintrin.h>
 170 #endif
 171
 172 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 173     (defined(__AVX512VL__) && defined(__AVX512DQ__))
 174 #include <avx512vldqintrin.h>
 175 #endif
 176
 177 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 178     defined(__AVX512ER__)
 179 #include <avx512erintrin.h>
 180 #endif
 181
 182 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 183     defined(__AVX512IFMA__)
 184 #include <avx512ifmaintrin.h>
 185 #endif
 186
 187 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 188     (defined(__AVX512IFMA__) && defined(__AVX512VL__))
 189 #include <avx512ifmavlintrin.h>
 190 #endif
 191
 192 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 193     defined(__AVXIFMA__)
 194 #include <avxifmaintrin.h>
 195 #endif
 196
 197 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 198     defined(__AVX512VBMI__)
 199 #include <avx512vbmiintrin.h>
 200 #endif
 201
 202 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 203     (defined(__AVX512VBMI__) && defined(__AVX512VL__))
 204 #include <avx512vbmivlintrin.h>
 205 #endif
 206
 207 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 208     defined(__AVX512VBMI2__)
 209 #include <avx512vbmi2intrin.h>
 210 #endif
 211
 212 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 213     (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
 214 #include <avx512vlvbmi2intrin.h>
 215 #endif
 216
 217 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 218     defined(__AVX512PF__)
 219 #include <avx512pfintrin.h>
 220 #endif
 221
 222 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 223     defined(__AVX512FP16__)
 224 #include <avx512fp16intrin.h>
 225 #endif
 226
 227 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 228     (defined(__AVX512VL__) && defined(__AVX512FP16__))
 229 #include <avx512vlfp16intrin.h>
 230 #endif
 231
 232 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 233     defined(__AVX512BF16__)
 234 #include <avx512bf16intrin.h>
 235 #endif
 236
 237 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 238     (defined(__AVX512VL__) && defined(__AVX512BF16__))
 239 #include <avx512vlbf16intrin.h>
 240 #endif
 241
 242 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 243     defined(__PKU__)
 244 #include <pkuintrin.h>
 245 #endif
 246
 247 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 248     defined(__VPCLMULQDQ__)
 249 #include <vpclmulqdqintrin.h>
 250 #endif
 251
 252 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 253     defined(__VAES__)
 254 #include <vaesintrin.h>
 255 #endif
 256
 257 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 258     defined(__GFNI__)
 259 #include <gfniintrin.h>
 260 #endif
 261
 262 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 263     defined(__AVXVNNIINT8__)
 264 #include <avxvnniint8intrin.h>
 265 #endif
 266
 267 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 268     defined(__AVXNECONVERT__)
 269 #include <avxneconvertintrin.h>
 270 #endif
 271
 272 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 273     defined(__RDPID__)
 274 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
 275 ///
 276 /// \headerfile <immintrin.h>
 277 ///
 278 /// This intrinsic corresponds to the <c> RDPID </c> instruction.
 279 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))
 280 _rdpid_u32(void) {
 281   return __builtin_ia32_rdpid();
 282 }
 283 #endif // __RDPID__
 284
 285 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 286     defined(__RDRND__)
 287 /// Returns a 16-bit hardware-generated random value.
 288 ///
 289 /// \headerfile <immintrin.h>
 290 ///
 291 /// This intrinsic corresponds to the <c> RDRAND </c> instruction.
 292 ///
 293 /// \param __p
 294 ///    A pointer to a 16-bit memory location to place the random value.
 295 /// \returns 1 if the value was successfully generated, 0 otherwise.
 296 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 297 _rdrand16_step(unsigned short *__p)
 298 {
 299   return (int)__builtin_ia32_rdrand16_step(__p);
 300 }
 301
 302 /// Returns a 32-bit hardware-generated random value.
 303 ///
 304 /// \headerfile <immintrin.h>
 305 ///
 306 /// This intrinsic corresponds to the <c> RDRAND </c> instruction.
 307 ///
 308 /// \param __p
 309 ///    A pointer to a 32-bit memory location to place the random value.
 310 /// \returns 1 if the value was successfully generated, 0 otherwise.
 311 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 312 _rdrand32_step(unsigned int *__p)
 313 {
 314   return (int)__builtin_ia32_rdrand32_step(__p);
 315 }
 316
 317 /// Returns a 64-bit hardware-generated random value.
 318 ///
 319 /// \headerfile <immintrin.h>
 320 ///
 321 /// This intrinsic corresponds to the <c> RDRAND </c> instruction.
 322 ///
 323 /// \param __p
 324 ///    A pointer to a 64-bit memory location to place the random value.
 325 /// \returns 1 if the value was successfully generated, 0 otherwise.
 326 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 327 _rdrand64_step(unsigned long long *__p)
 328 {
 329 #ifdef __x86_64__
 330   return (int)__builtin_ia32_rdrand64_step(__p);
 331 #else
 332   // We need to emulate the functionality of 64-bit rdrand with 2 32-bit
 333   // rdrand instructions.
 334   unsigned int __lo, __hi;
 335   unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);
 336   unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);
 337   if (__res_lo && __res_hi) {
 338     *__p = ((unsigned long long)__hi << 32) | (unsigned long long)__lo;
 339     return 1;
 340   } else {
 341     *__p = 0;
 342     return 0;
 343   }
 344 #endif
 345 }
 346 #endif /* __RDRND__ */
 347
 348 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 349     defined(__FSGSBASE__)
 350 #ifdef __x86_64__
 351 /// Reads the FS base register.
 352 ///
 353 /// \headerfile <immintrin.h>
 354 ///
 355 /// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.
 356 ///
 357 /// \returns The lower 32 bits of the FS base register.
 358 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 359 _readfsbase_u32(void)
 360 {
 361   return __builtin_ia32_rdfsbase32();
 362 }
 363
 364 /// Reads the FS base register.
 365 ///
 366 /// \headerfile <immintrin.h>
 367 ///
 368 /// This intrinsic corresponds to the <c> RDFSBASE </c> instruction.
 369 ///
 370 /// \returns The contents of the FS base register.
 371 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 372 _readfsbase_u64(void)
 373 {
 374   return __builtin_ia32_rdfsbase64();
 375 }
 376
 377 /// Reads the GS base register.
 378 ///
 379 /// \headerfile <immintrin.h>
 380 ///
 381 /// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.
 382 ///
 383 /// \returns The lower 32 bits of the GS base register.
 384 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 385 _readgsbase_u32(void)
 386 {
 387   return __builtin_ia32_rdgsbase32();
 388 }
 389
 390 /// Reads the GS base register.
 391 ///
 392 /// \headerfile <immintrin.h>
 393 ///
 394 /// This intrinsic corresponds to the <c> RDGSBASE </c> instruction.
 395 ///
 396 /// \returns The contents of the GS base register.
 397 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 398 _readgsbase_u64(void)
 399 {
 400   return __builtin_ia32_rdgsbase64();
 401 }
 402
 403 /// Modifies the FS base register.
 404 ///
 405 /// \headerfile <immintrin.h>
 406 ///
 407 /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
 408 ///
 409 /// \param __V
 410 ///    Value to use for the lower 32 bits of the FS base register.
 411 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 412 _writefsbase_u32(unsigned int __V)
 413 {
 414   __builtin_ia32_wrfsbase32(__V);
 415 }
 416
 417 /// Modifies the FS base register.
 418 ///
 419 /// \headerfile <immintrin.h>
 420 ///
 421 /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
 422 ///
 423 /// \param __V
 424 ///    Value to use for the FS base register.
 425 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 426 _writefsbase_u64(unsigned long long __V)
 427 {
 428   __builtin_ia32_wrfsbase64(__V);
 429 }
 430
 431 /// Modifies the GS base register.
 432 ///
 433 /// \headerfile <immintrin.h>
 434 ///
 435 /// This intrinsic corresponds to the <c> WRGSBASE </c> instruction.
 436 ///
 437 /// \param __V
 438 ///    Value to use for the lower 32 bits of the GS base register.
 439 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 440 _writegsbase_u32(unsigned int __V)
 441 {
 442   __builtin_ia32_wrgsbase32(__V);
 443 }
 444
 445 /// Modifies the GS base register.
 446 ///
 447 /// \headerfile <immintrin.h>
 448 ///
 449 /// This intrinsic corresponds to the <c> WRFSBASE </c> instruction.
 450 ///
 451 /// \param __V
 452 ///    Value to use for GS base register.
 453 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 454 _writegsbase_u64(unsigned long long __V)
 455 {
 456   __builtin_ia32_wrgsbase64(__V);
 457 }
 458
 459 #endif
 460 #endif /* __FSGSBASE__ */
 461
 462 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 463     defined(__MOVBE__)
 464
 465 /* The structs used below are to force the load/store to be unaligned. This
 466  * is accomplished with the __packed__ attribute. The __may_alias__ prevents
 467  * tbaa metadata from being generated based on the struct and the type of the
 468  * field inside of it.
 469  */
 470
 471 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 472 _loadbe_i16(void const * __P) {
 473   struct __loadu_i16 {
 474     unsigned short __v;
 475   } __attribute__((__packed__, __may_alias__));
 476   return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v);
 477 }
 478
 479 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 480 _storebe_i16(void * __P, short __D) {
 481   struct __storeu_i16 {
 482     unsigned short __v;
 483   } __attribute__((__packed__, __may_alias__));
 484   ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D);
 485 }
 486
 487 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 488 _loadbe_i32(void const * __P) {
 489   struct __loadu_i32 {
 490     unsigned int __v;
 491   } __attribute__((__packed__, __may_alias__));
 492   return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v);
 493 }
 494
 495 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 496 _storebe_i32(void * __P, int __D) {
 497   struct __storeu_i32 {
 498     unsigned int __v;
 499   } __attribute__((__packed__, __may_alias__));
 500   ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D);
 501 }
 502
 503 #ifdef __x86_64__
 504 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 505 _loadbe_i64(void const * __P) {
 506   struct __loadu_i64 {
 507     unsigned long long __v;
 508   } __attribute__((__packed__, __may_alias__));
 509   return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v);
 510 }
 511
 512 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 513 _storebe_i64(void * __P, long long __D) {
 514   struct __storeu_i64 {
 515     unsigned long long __v;
 516   } __attribute__((__packed__, __may_alias__));
 517   ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D);
 518 }
 519 #endif
 520 #endif /* __MOVBE */
 521
 522 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 523     defined(__RTM__)
 524 #include <rtmintrin.h>
 525 #include <xtestintrin.h>
 526 #endif
 527
 528 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 529     defined(__SHA__)
 530 #include <shaintrin.h>
 531 #endif
 532
 533 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 534     defined(__FXSR__)
 535 #include <fxsrintrin.h>
 536 #endif
 537
 538 /* No feature check desired due to internal MSC_VER checks */
 539 #include <xsaveintrin.h>
 540
 541 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 542     defined(__XSAVEOPT__)
 543 #include <xsaveoptintrin.h>
 544 #endif
 545
 546 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 547     defined(__XSAVEC__)
 548 #include <xsavecintrin.h>
 549 #endif
 550
 551 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 552     defined(__XSAVES__)
 553 #include <xsavesintrin.h>
 554 #endif
 555
 556 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 557     defined(__SHSTK__)
 558 #include <cetintrin.h>
 559 #endif
 560
 561 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
 562  * whereas others are also available at all times. */
 563 #include <adxintrin.h>
 564
 565 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 566     defined(__RDSEED__)
 567 #include <rdseedintrin.h>
 568 #endif
 569
 570 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 571     defined(__WBNOINVD__)
 572 #include <wbnoinvdintrin.h>
 573 #endif
 574
 575 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 576     defined(__CLDEMOTE__)
 577 #include <cldemoteintrin.h>
 578 #endif
 579
 580 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 581     defined(__WAITPKG__)
 582 #include <waitpkgintrin.h>
 583 #endif
 584
 585 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 586     defined(__MOVDIRI__) || defined(__MOVDIR64B__)
 587 #include <movdirintrin.h>
 588 #endif
 589
 590 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 591     defined(__PCONFIG__)
 592 #include <pconfigintrin.h>
 593 #endif
 594
 595 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 596     defined(__SGX__)
 597 #include <sgxintrin.h>
 598 #endif
 599
 600 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 601     defined(__PTWRITE__)
 602 #include <ptwriteintrin.h>
 603 #endif
 604
 605 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 606     defined(__INVPCID__)
 607 #include <invpcidintrin.h>
 608 #endif
 609 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 610     defined(__AMX_FP16__)
 611 #include <amxfp16intrin.h>
 612 #endif
 613
 614 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 615     defined(__KL__) || defined(__WIDEKL__)
 616 #include <keylockerintrin.h>
 617 #endif
 618
 619 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 620     defined(__AMX_TILE__) || defined(__AMX_INT8__) || defined(__AMX_BF16__)
 621 #include <amxintrin.h>
 622 #endif
 623
 624 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 625     defined(__AMX_COMPLEX__)
 626 #include <amxcomplexintrin.h>
 627 #endif
 628
 629 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 630     defined(__AVX512VP2INTERSECT__)
 631 #include <avx512vp2intersectintrin.h>
 632 #endif
 633
 634 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 635     (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
 636 #include <avx512vlvp2intersectintrin.h>
 637 #endif
 638
 639 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 640     defined(__ENQCMD__)
 641 #include <enqcmdintrin.h>
 642 #endif
 643
 644 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 645     defined(__SERIALIZE__)
 646 #include <serializeintrin.h>
 647 #endif
 648
 649 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 650     defined(__TSXLDTRK__)
 651 #include <tsxldtrkintrin.h>
 652 #endif
 653
 654 #if defined(_MSC_VER) && __has_extension(gnu_asm)
 655 /* Define the default attributes for these intrinsics */
 656 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
 657 #ifdef __cplusplus
 658 extern "C" {
 659 #endif
 660 /*----------------------------------------------------------------------------*\
 661 |* Interlocked Exchange HLE
 662 \*----------------------------------------------------------------------------*/
 663 #if defined(__i386__) || defined(__x86_64__)
 664 static __inline__ long __DEFAULT_FN_ATTRS
 665 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {
 666   __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"
 667                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 668   return _Value;
 669 }
 670 static __inline__ long __DEFAULT_FN_ATTRS
 671 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {
 672   __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"
 673                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 674   return _Value;
 675 }
 676 #endif
 677 #if defined(__x86_64__)
 678 static __inline__ __int64 __DEFAULT_FN_ATTRS
 679 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {
 680   __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"
 681                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 682   return _Value;
 683 }
 684 static __inline__ __int64 __DEFAULT_FN_ATTRS
 685 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {
 686   __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"
 687                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 688   return _Value;
 689 }
 690 #endif
 691 /*----------------------------------------------------------------------------*\
 692 |* Interlocked Compare Exchange HLE
 693 \*----------------------------------------------------------------------------*/
 694 #if defined(__i386__) || defined(__x86_64__)
 695 static __inline__ long __DEFAULT_FN_ATTRS
 696 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,
 697                               long _Exchange, long _Comparand) {
 698   __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"
 699                        : "+a" (_Comparand), "+m" (*_Destination)
 700                        : "r" (_Exchange) : "memory");
 701   return _Comparand;
 702 }
 703 static __inline__ long __DEFAULT_FN_ATTRS
 704 _InterlockedCompareExchange_HLERelease(long volatile *_Destination,
 705                               long _Exchange, long _Comparand) {
 706   __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"
 707                        : "+a" (_Comparand), "+m" (*_Destination)
 708                        : "r" (_Exchange) : "memory");
 709   return _Comparand;
 710 }
 711 #endif
 712 #if defined(__x86_64__)
 713 static __inline__ __int64 __DEFAULT_FN_ATTRS
 714 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,
 715                               __int64 _Exchange, __int64 _Comparand) {
 716   __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"
 717                        : "+a" (_Comparand), "+m" (*_Destination)
 718                        : "r" (_Exchange) : "memory");
 719   return _Comparand;
 720 }
 721 static __inline__ __int64 __DEFAULT_FN_ATTRS
 722 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
 723                               __int64 _Exchange, __int64 _Comparand) {
 724   __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"
 725                        : "+a" (_Comparand), "+m" (*_Destination)
 726                        : "r" (_Exchange) : "memory");
 727   return _Comparand;
 728 }
 729 #endif
 730 #ifdef __cplusplus
 731 }
 732 #endif
 733
 734 #undef __DEFAULT_FN_ATTRS
 735
 736 #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */
 737
 738 #endif /* __IMMINTRIN_H */