clang/lib/Headers/immintrin.h

   1 /*===---- immintrin.h - Intel intrinsics -----------------------------------===
   2  *
   3  * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4  * See https://llvm.org/LICENSE.txt for license information.
   5  * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6  *
   7  *===-----------------------------------------------------------------------===
   8  */
   9
  10 #ifndef __IMMINTRIN_H
  11 #define __IMMINTRIN_H
  12
  13 #if !defined(__i386__) && !defined(__x86_64__)
  14 #error "This header is only meant to be used on x86 and x64 architecture"
  15 #endif
  16
  17 #include <x86gprintrin.h>
  18
  19 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  20     defined(__MMX__)
  21 #include <mmintrin.h>
  22 #endif
  23
  24 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  25     defined(__SSE__)
  26 #include <xmmintrin.h>
  27 #endif
  28
  29 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  30     defined(__SSE2__)
  31 #include <emmintrin.h>
  32 #endif
  33
  34 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  35     defined(__SSE3__)
  36 #include <pmmintrin.h>
  37 #endif
  38
  39 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  40     defined(__SSSE3__)
  41 #include <tmmintrin.h>
  42 #endif
  43
  44 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  45     (defined(__SSE4_2__) || defined(__SSE4_1__))
  46 #include <smmintrin.h>
  47 #endif
  48
  49 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  50     (defined(__AES__) || defined(__PCLMUL__))
  51 #include <wmmintrin.h>
  52 #endif
  53
  54 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  55     defined(__CLFLUSHOPT__)
  56 #include <clflushoptintrin.h>
  57 #endif
  58
  59 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  60     defined(__CLWB__)
  61 #include <clwbintrin.h>
  62 #endif
  63
  64 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  65     defined(__AVX__)
  66 #include <avxintrin.h>
  67 #endif
  68
  69 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  70     defined(__AVX2__)
  71 #include <avx2intrin.h>
  72 #endif
  73
  74 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  75     defined(__F16C__)
  76 #include <f16cintrin.h>
  77 #endif
  78
  79 /* No feature check desired due to internal checks */
  80 #include <bmiintrin.h>
  81
  82 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  83     defined(__BMI2__)
  84 #include <bmi2intrin.h>
  85 #endif
  86
  87 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  88     defined(__LZCNT__)
  89 #include <lzcntintrin.h>
  90 #endif
  91
  92 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  93     defined(__POPCNT__)
  94 #include <popcntintrin.h>
  95 #endif
  96
  97 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
  98     defined(__FMA__)
  99 #include <fmaintrin.h>
 100 #endif
 101
 102 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 103     defined(__AVX512F__)
 104 #include <avx512fintrin.h>
 105 #endif
 106
 107 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 108     defined(__AVX512VL__)
 109 #include <avx512vlintrin.h>
 110 #endif
 111
 112 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 113     defined(__AVX512BW__)
 114 #include <avx512bwintrin.h>
 115 #endif
 116
 117 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 118     defined(__AVX512BITALG__)
 119 #include <avx512bitalgintrin.h>
 120 #endif
 121
 122 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 123     defined(__AVX512CD__)
 124 #include <avx512cdintrin.h>
 125 #endif
 126
 127 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 128     defined(__AVX512VPOPCNTDQ__)
 129 #include <avx512vpopcntdqintrin.h>
 130 #endif
 131
 132 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 133     (defined(__AVX512VL__) && defined(__AVX512VPOPCNTDQ__))
 134 #include <avx512vpopcntdqvlintrin.h>
 135 #endif
 136
 137 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 138     defined(__AVX512VNNI__)
 139 #include <avx512vnniintrin.h>
 140 #endif
 141
 142 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 143     (defined(__AVX512VL__) && defined(__AVX512VNNI__))
 144 #include <avx512vlvnniintrin.h>
 145 #endif
 146
 147 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 148     defined(__AVXVNNI__)
 149 #include <avxvnniintrin.h>
 150 #endif
 151
 152 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 153     defined(__AVX512DQ__)
 154 #include <avx512dqintrin.h>
 155 #endif
 156
 157 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 158     (defined(__AVX512VL__) && defined(__AVX512BITALG__))
 159 #include <avx512vlbitalgintrin.h>
 160 #endif
 161
 162 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 163     (defined(__AVX512VL__) && defined(__AVX512BW__))
 164 #include <avx512vlbwintrin.h>
 165 #endif
 166
 167 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 168     (defined(__AVX512VL__) && defined(__AVX512CD__))
 169 #include <avx512vlcdintrin.h>
 170 #endif
 171
 172 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 173     (defined(__AVX512VL__) && defined(__AVX512DQ__))
 174 #include <avx512vldqintrin.h>
 175 #endif
 176
 177 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 178     defined(__AVX512ER__)
 179 #include <avx512erintrin.h>
 180 #endif
 181
 182 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 183     defined(__AVX512IFMA__)
 184 #include <avx512ifmaintrin.h>
 185 #endif
 186
 187 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 188     (defined(__AVX512IFMA__) && defined(__AVX512VL__))
 189 #include <avx512ifmavlintrin.h>
 190 #endif
 191
 192 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 193     defined(__AVX512VBMI__)
 194 #include <avx512vbmiintrin.h>
 195 #endif
 196
 197 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 198     (defined(__AVX512VBMI__) && defined(__AVX512VL__))
 199 #include <avx512vbmivlintrin.h>
 200 #endif
 201
 202 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 203     defined(__AVX512VBMI2__)
 204 #include <avx512vbmi2intrin.h>
 205 #endif
 206
 207 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 208     (defined(__AVX512VBMI2__) && defined(__AVX512VL__))
 209 #include <avx512vlvbmi2intrin.h>
 210 #endif
 211
 212 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 213     defined(__AVX512PF__)
 214 #include <avx512pfintrin.h>
 215 #endif
 216
 217 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 218     defined(__AVX512FP16__)
 219 #include <avx512fp16intrin.h>
 220 #endif
 221
 222 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 223     (defined(__AVX512VL__) && defined(__AVX512FP16__))
 224 #include <avx512vlfp16intrin.h>
 225 #endif
 226
 227 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 228     defined(__AVX512BF16__)
 229 #include <avx512bf16intrin.h>
 230 #endif
 231
 232 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 233     (defined(__AVX512VL__) && defined(__AVX512BF16__))
 234 #include <avx512vlbf16intrin.h>
 235 #endif
 236
 237 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 238     defined(__PKU__)
 239 #include <pkuintrin.h>
 240 #endif
 241
 242 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 243     defined(__VPCLMULQDQ__)
 244 #include <vpclmulqdqintrin.h>
 245 #endif
 246
 247 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 248     defined(__VAES__)
 249 #include <vaesintrin.h>
 250 #endif
 251
 252 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 253     defined(__GFNI__)
 254 #include <gfniintrin.h>
 255 #endif
 256
 257 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 258     defined(__RDPID__)
 259 /// Returns the value of the IA32_TSC_AUX MSR (0xc0000103).
 260 ///
 261 /// \headerfile <immintrin.h>
 262 ///
 263 /// This intrinsic corresponds to the <c> RDPID </c> instruction.
 264 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("rdpid")))
 265 _rdpid_u32(void) {
 266   return __builtin_ia32_rdpid();
 267 }
 268 #endif // __RDPID__
 269
 270 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 271     defined(__RDRND__)
 272 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 273 _rdrand16_step(unsigned short *__p)
 274 {
 275   return (int)__builtin_ia32_rdrand16_step(__p);
 276 }
 277
 278 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 279 _rdrand32_step(unsigned int *__p)
 280 {
 281   return (int)__builtin_ia32_rdrand32_step(__p);
 282 }
 283
 284 #ifdef __x86_64__
 285 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 286 _rdrand64_step(unsigned long long *__p)
 287 {
 288   return (int)__builtin_ia32_rdrand64_step(__p);
 289 }
 290 #else
 291 // We need to emulate the functionality of 64-bit rdrand with 2 32-bit
 292 // rdrand instructions.
 293 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("rdrnd")))
 294 _rdrand64_step(unsigned long long *__p)
 295 {
 296   unsigned int __lo, __hi;
 297   unsigned int __res_lo = __builtin_ia32_rdrand32_step(&__lo);
 298   unsigned int __res_hi = __builtin_ia32_rdrand32_step(&__hi);
 299   if (__res_lo && __res_hi) {
 300     *__p = ((unsigned long long)__hi << 32) | (unsigned long long)__lo;
 301     return 1;
 302   } else {
 303     *__p = 0;
 304     return 0;
 305   }
 306 }
 307 #endif
 308 #endif /* __RDRND__ */
 309
 310 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 311     defined(__FSGSBASE__)
 312 #ifdef __x86_64__
 313 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 314 _readfsbase_u32(void)
 315 {
 316   return __builtin_ia32_rdfsbase32();
 317 }
 318
 319 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 320 _readfsbase_u64(void)
 321 {
 322   return __builtin_ia32_rdfsbase64();
 323 }
 324
 325 static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 326 _readgsbase_u32(void)
 327 {
 328   return __builtin_ia32_rdgsbase32();
 329 }
 330
 331 static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 332 _readgsbase_u64(void)
 333 {
 334   return __builtin_ia32_rdgsbase64();
 335 }
 336
 337 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 338 _writefsbase_u32(unsigned int __V)
 339 {
 340   __builtin_ia32_wrfsbase32(__V);
 341 }
 342
 343 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 344 _writefsbase_u64(unsigned long long __V)
 345 {
 346   __builtin_ia32_wrfsbase64(__V);
 347 }
 348
 349 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 350 _writegsbase_u32(unsigned int __V)
 351 {
 352   __builtin_ia32_wrgsbase32(__V);
 353 }
 354
 355 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("fsgsbase")))
 356 _writegsbase_u64(unsigned long long __V)
 357 {
 358   __builtin_ia32_wrgsbase64(__V);
 359 }
 360
 361 #endif
 362 #endif /* __FSGSBASE__ */
 363
 364 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 365     defined(__MOVBE__)
 366
 367 /* The structs used below are to force the load/store to be unaligned. This
 368  * is accomplished with the __packed__ attribute. The __may_alias__ prevents
 369  * tbaa metadata from being generated based on the struct and the type of the
 370  * field inside of it.
 371  */
 372
 373 static __inline__ short __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 374 _loadbe_i16(void const * __P) {
 375   struct __loadu_i16 {
 376     unsigned short __v;
 377   } __attribute__((__packed__, __may_alias__));
 378   return (short)__builtin_bswap16(((const struct __loadu_i16*)__P)->__v);
 379 }
 380
 381 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 382 _storebe_i16(void * __P, short __D) {
 383   struct __storeu_i16 {
 384     unsigned short __v;
 385   } __attribute__((__packed__, __may_alias__));
 386   ((struct __storeu_i16*)__P)->__v = __builtin_bswap16((unsigned short)__D);
 387 }
 388
 389 static __inline__ int __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 390 _loadbe_i32(void const * __P) {
 391   struct __loadu_i32 {
 392     unsigned int __v;
 393   } __attribute__((__packed__, __may_alias__));
 394   return (int)__builtin_bswap32(((const struct __loadu_i32*)__P)->__v);
 395 }
 396
 397 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 398 _storebe_i32(void * __P, int __D) {
 399   struct __storeu_i32 {
 400     unsigned int __v;
 401   } __attribute__((__packed__, __may_alias__));
 402   ((struct __storeu_i32*)__P)->__v = __builtin_bswap32((unsigned int)__D);
 403 }
 404
 405 #ifdef __x86_64__
 406 static __inline__ long long __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 407 _loadbe_i64(void const * __P) {
 408   struct __loadu_i64 {
 409     unsigned long long __v;
 410   } __attribute__((__packed__, __may_alias__));
 411   return (long long)__builtin_bswap64(((const struct __loadu_i64*)__P)->__v);
 412 }
 413
 414 static __inline__ void __attribute__((__always_inline__, __nodebug__, __target__("movbe")))
 415 _storebe_i64(void * __P, long long __D) {
 416   struct __storeu_i64 {
 417     unsigned long long __v;
 418   } __attribute__((__packed__, __may_alias__));
 419   ((struct __storeu_i64*)__P)->__v = __builtin_bswap64((unsigned long long)__D);
 420 }
 421 #endif
 422 #endif /* __MOVBE */
 423
 424 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 425     defined(__RTM__)
 426 #include <rtmintrin.h>
 427 #include <xtestintrin.h>
 428 #endif
 429
 430 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 431     defined(__SHA__)
 432 #include <shaintrin.h>
 433 #endif
 434
 435 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 436     defined(__FXSR__)
 437 #include <fxsrintrin.h>
 438 #endif
 439
 440 /* No feature check desired due to internal MSC_VER checks */
 441 #include <xsaveintrin.h>
 442
 443 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 444     defined(__XSAVEOPT__)
 445 #include <xsaveoptintrin.h>
 446 #endif
 447
 448 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 449     defined(__XSAVEC__)
 450 #include <xsavecintrin.h>
 451 #endif
 452
 453 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 454     defined(__XSAVES__)
 455 #include <xsavesintrin.h>
 456 #endif
 457
 458 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 459     defined(__SHSTK__)
 460 #include <cetintrin.h>
 461 #endif
 462
 463 /* Some intrinsics inside adxintrin.h are available only on processors with ADX,
 464  * whereas others are also available at all times. */
 465 #include <adxintrin.h>
 466
 467 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 468     defined(__RDSEED__)
 469 #include <rdseedintrin.h>
 470 #endif
 471
 472 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 473     defined(__WBNOINVD__)
 474 #include <wbnoinvdintrin.h>
 475 #endif
 476
 477 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 478     defined(__CLDEMOTE__)
 479 #include <cldemoteintrin.h>
 480 #endif
 481
 482 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 483     defined(__WAITPKG__)
 484 #include <waitpkgintrin.h>
 485 #endif
 486
 487 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 488     defined(__MOVDIRI__) || defined(__MOVDIR64B__)
 489 #include <movdirintrin.h>
 490 #endif
 491
 492 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 493     defined(__PCONFIG__)
 494 #include <pconfigintrin.h>
 495 #endif
 496
 497 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 498     defined(__SGX__)
 499 #include <sgxintrin.h>
 500 #endif
 501
 502 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 503     defined(__PTWRITE__)
 504 #include <ptwriteintrin.h>
 505 #endif
 506
 507 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 508     defined(__INVPCID__)
 509 #include <invpcidintrin.h>
 510 #endif
 511
 512 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 513     defined(__KL__) || defined(__WIDEKL__)
 514 #include <keylockerintrin.h>
 515 #endif
 516
 517 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 518     defined(__AMXTILE__) || defined(__AMXINT8__) || defined(__AMXBF16__)
 519 #include <amxintrin.h>
 520 #endif
 521
 522 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 523     defined(__AVX512VP2INTERSECT__)
 524 #include <avx512vp2intersectintrin.h>
 525 #endif
 526
 527 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 528     (defined(__AVX512VL__) && defined(__AVX512VP2INTERSECT__))
 529 #include <avx512vlvp2intersectintrin.h>
 530 #endif
 531
 532 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 533     defined(__ENQCMD__)
 534 #include <enqcmdintrin.h>
 535 #endif
 536
 537 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 538     defined(__SERIALIZE__)
 539 #include <serializeintrin.h>
 540 #endif
 541
 542 #if !(defined(_MSC_VER) || defined(__SCE__)) || __has_feature(modules) ||      \
 543     defined(__TSXLDTRK__)
 544 #include <tsxldtrkintrin.h>
 545 #endif
 546
 547 #if defined(_MSC_VER) && __has_extension(gnu_asm)
 548 /* Define the default attributes for these intrinsics */
 549 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
 550 #ifdef __cplusplus
 551 extern "C" {
 552 #endif
 553 /*----------------------------------------------------------------------------*\
 554 |* Interlocked Exchange HLE
 555 \*----------------------------------------------------------------------------*/
 556 #if defined(__i386__) || defined(__x86_64__)
 557 static __inline__ long __DEFAULT_FN_ATTRS
 558 _InterlockedExchange_HLEAcquire(long volatile *_Target, long _Value) {
 559   __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"
 560                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 561   return _Value;
 562 }
 563 static __inline__ long __DEFAULT_FN_ATTRS
 564 _InterlockedExchange_HLERelease(long volatile *_Target, long _Value) {
 565   __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"
 566                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 567   return _Value;
 568 }
 569 #endif
 570 #if defined(__x86_64__)
 571 static __inline__ __int64 __DEFAULT_FN_ATTRS
 572 _InterlockedExchange64_HLEAcquire(__int64 volatile *_Target, __int64 _Value) {
 573   __asm__ __volatile__(".byte 0xf2 ; lock ; xchg {%0, %1|%1, %0}"
 574                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 575   return _Value;
 576 }
 577 static __inline__ __int64 __DEFAULT_FN_ATTRS
 578 _InterlockedExchange64_HLERelease(__int64 volatile *_Target, __int64 _Value) {
 579   __asm__ __volatile__(".byte 0xf3 ; lock ; xchg {%0, %1|%1, %0}"
 580                        : "+r" (_Value), "+m" (*_Target) :: "memory");
 581   return _Value;
 582 }
 583 #endif
 584 /*----------------------------------------------------------------------------*\
 585 |* Interlocked Compare Exchange HLE
 586 \*----------------------------------------------------------------------------*/
 587 #if defined(__i386__) || defined(__x86_64__)
 588 static __inline__ long __DEFAULT_FN_ATTRS
 589 _InterlockedCompareExchange_HLEAcquire(long volatile *_Destination,
 590                               long _Exchange, long _Comparand) {
 591   __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"
 592                        : "+a" (_Comparand), "+m" (*_Destination)
 593                        : "r" (_Exchange) : "memory");
 594   return _Comparand;
 595 }
 596 static __inline__ long __DEFAULT_FN_ATTRS
 597 _InterlockedCompareExchange_HLERelease(long volatile *_Destination,
 598                               long _Exchange, long _Comparand) {
 599   __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"
 600                        : "+a" (_Comparand), "+m" (*_Destination)
 601                        : "r" (_Exchange) : "memory");
 602   return _Comparand;
 603 }
 604 #endif
 605 #if defined(__x86_64__)
 606 static __inline__ __int64 __DEFAULT_FN_ATTRS
 607 _InterlockedCompareExchange64_HLEAcquire(__int64 volatile *_Destination,
 608                               __int64 _Exchange, __int64 _Comparand) {
 609   __asm__ __volatile__(".byte 0xf2 ; lock ; cmpxchg {%2, %1|%1, %2}"
 610                        : "+a" (_Comparand), "+m" (*_Destination)
 611                        : "r" (_Exchange) : "memory");
 612   return _Comparand;
 613 }
 614 static __inline__ __int64 __DEFAULT_FN_ATTRS
 615 _InterlockedCompareExchange64_HLERelease(__int64 volatile *_Destination,
 616                               __int64 _Exchange, __int64 _Comparand) {
 617   __asm__ __volatile__(".byte 0xf3 ; lock ; cmpxchg {%2, %1|%1, %2}"
 618                        : "+a" (_Comparand), "+m" (*_Destination)
 619                        : "r" (_Exchange) : "memory");
 620   return _Comparand;
 621 }
 622 #endif
 623 #ifdef __cplusplus
 624 }
 625 #endif
 626
 627 #undef __DEFAULT_FN_ATTRS
 628
 629 #endif /* defined(_MSC_VER) && __has_extension(gnu_asm) */
 630
 631 #endif /* __IMMINTRIN_H */