src/vendorcode/amd/include/gcc-intrin.h

   1 /*
   2  * Copyright (c) 2011, Advanced Micro Devices, Inc. All rights reserved.
   3  * Copyright (c) 2014, Edward O'Callaghan <eocallaghan@alterapraxis.com>
   4  *
   5  * Redistribution and use in source and binary forms, with or without
   6  * modification, are permitted provided that the following conditions are met:
   7  *     * Redistributions of source code must retain the above copyright
   8  *       notice, this list of conditions and the following disclaimer.
   9  *     * Redistributions in binary form must reproduce the above copyright
  10  *       notice, this list of conditions and the following disclaimer in the
  11  *       documentation and/or other materials provided with the distribution.
  12  *     * Neither the name of Advanced Micro Devices, Inc. nor the names of
  13  *       its contributors may be used to endorse or promote products derived
  14  *       from this software without specific prior written permission.
  15  *
  16  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
  17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19  * DISCLAIMED. IN NO EVENT SHALL ADVANCED MICRO DEVICES, INC. BE LIABLE FOR ANY
  20  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26  *
  27  */
  28
  29 #if defined (__GNUC__)
  30 #include <stdint.h>
  31 /* I/O intrin functions.  */
  32 static __inline__ __attribute__((always_inline)) uint8_t __inbyte(uint16_t Port)
  33 {
  34   uint8_t value;
  35
  36   __asm__ __volatile__ (
  37     "in  %1, %0"
  38     : "=a" (value)
  39     : "Nd" (Port)
  40     );
  41
  42   return value;
  43 }
  44
  45 static __inline__ __attribute__((always_inline)) uint16_t __inword(uint16_t Port)
  46 {
  47   uint16_t value;
  48
  49   __asm__ __volatile__ (
  50     "in  %1, %0"
  51     : "=a" (value)
  52     : "Nd" (Port)
  53     );
  54
  55   return value;
  56 }
  57
  58 static __inline__ __attribute__((always_inline)) uint32_t __indword(uint16_t Port)
  59 {
  60   uint32_t value;
  61
  62   __asm__ __volatile__ (
  63     "in  %1, %0"
  64     : "=a" (value)
  65     : "Nd" (Port)
  66     );
  67   return value;
  68
  69 }
  70
  71 static __inline__ __attribute__((always_inline)) void __outbyte(uint16_t Port,uint8_t Data)
  72 {
  73   __asm__ __volatile__ (
  74     "out  %0, %1"
  75     :
  76     : "a" (Data), "Nd" (Port)
  77     );
  78 }
  79
  80 static __inline__ __attribute__((always_inline)) void __outword(uint16_t Port,uint16_t Data)
  81 {
  82   __asm__ __volatile__ (
  83     "out  %0, %1"
  84     :
  85     : "a" (Data), "Nd" (Port)
  86     );
  87 }
  88
  89 static __inline__ __attribute__((always_inline)) void __outdword(uint16_t Port,uint32_t Data)
  90 {
  91   __asm__ __volatile__ (
  92     "out  %0, %1"
  93     :
  94     : "a" (Data), "Nd" (Port)
  95     );
  96 }
  97
  98 static __inline__ __attribute__((always_inline)) void __inbytestring(uint16_t Port,uint8_t *Buffer,unsigned long Count)
  99 {
 100   __asm__ __volatile__ (
 101     "rep ; insb"
 102     : "+D" (Buffer), "+c" (Count)
 103     : "d"(Port)
 104     );
 105 }
 106
 107 static __inline__ __attribute__((always_inline)) void __inwordstring(uint16_t Port,uint16_t *Buffer,unsigned long Count)
 108 {
 109   __asm__ __volatile__ (
 110     "rep ; insw"
 111     : "+D" (Buffer), "+c" (Count)
 112     : "d"(Port)
 113     );
 114 }
 115
 116 static __inline__ __attribute__((always_inline)) void __indwordstring(uint16_t Port,unsigned long *Buffer,unsigned long Count)
 117 {
 118   __asm__ __volatile__ (
 119     "rep ; insl"
 120     : "+D" (Buffer), "+c" (Count)
 121     : "d"(Port)
 122     );
 123 }
 124
 125 static __inline__ __attribute__((always_inline)) void __outbytestring(uint16_t Port,uint8_t *Buffer,unsigned long Count)
 126 {
 127   __asm__ __volatile__ (
 128     "rep ; outsb"
 129     : "+S" (Buffer), "+c" (Count)
 130     : "d"(Port)
 131     );
 132 }
 133
 134 static __inline__ __attribute__((always_inline)) void __outwordstring(uint16_t Port,uint16_t *Buffer,unsigned long Count)
 135 {
 136   __asm__ __volatile__ (
 137     "rep ; outsw"
 138     : "+S" (Buffer), "+c" (Count)
 139     : "d"(Port)
 140   );
 141 }
 142
 143 static __inline__ __attribute__((always_inline)) void __outdwordstring(uint16_t Port,unsigned long *Buffer,unsigned long Count)
 144 {
 145   __asm__ __volatile__ (
 146    "rep ; outsl"
 147    : "+S" (Buffer), "+c" (Count)
 148    : "d"(Port)
 149    );
 150 }
 151
 152 static __inline__ __attribute__((always_inline)) unsigned long __readdr0(void)
 153 {
 154   unsigned long value;
 155   __asm__ __volatile__ (
 156     "mov %%dr0, %[value]"
 157     : [value] "=r" (value)
 158     );
 159   return value;
 160 }
 161
 162 static __inline__ __attribute__((always_inline)) unsigned long __readdr1(void)
 163 {
 164   unsigned long value;
 165   __asm__ __volatile__ (
 166     "mov %%dr1, %[value]"
 167     : [value] "=r" (value)
 168     );
 169   return value;
 170 }
 171
 172 static __inline__ __attribute__((always_inline)) unsigned long __readdr2(void)
 173 {
 174   unsigned long value;
 175   __asm__ __volatile__ (
 176     "mov %%dr2, %[value]"
 177     : [value] "=r" (value)
 178     );
 179   return value;
 180 }
 181
 182 static __inline__ __attribute__((always_inline)) unsigned long __readdr3(void)
 183 {
 184   unsigned long value;
 185   __asm__ __volatile__ (
 186     "mov %%dr3, %[value]"
 187     : [value] "=r" (value)
 188     );
 189   return value;
 190 }
 191
 192 static __inline__ __attribute__((always_inline)) unsigned long __readdr7(void)
 193 {
 194   unsigned long value;
 195   __asm__ __volatile__ (
 196     "mov %%dr7, %[value]"
 197     : [value] "=r" (value)
 198     );
 199   return value;
 200 }
 201
 202 static __inline__ __attribute__((always_inline)) unsigned long __readdr(unsigned long reg)
 203 {
 204   switch (reg){
 205     case 0:
 206       return __readdr0 ();
 207       break;
 208
 209     case 1:
 210       return __readdr1 ();
 211       break;
 212
 213     case 2:
 214       return __readdr2 ();
 215       break;
 216
 217     case 3:
 218       return __readdr3 ();
 219       break;
 220
 221     case 7:
 222       return __readdr7 ();
 223       break;
 224
 225     default:
 226       return -1;
 227   }
 228 }
 229
 230 static __inline__ __attribute__((always_inline)) void __writedr0(unsigned long Data)
 231 {
 232   __asm__ __volatile__ (
 233     "mov %0, %%dr0"
 234     :
 235     : "r" (Data)
 236     );
 237 }
 238
 239 static __inline__ __attribute__((always_inline)) void __writedr1(unsigned long Data)
 240 {
 241   __asm__ __volatile__ (
 242     "mov %0, %%dr1"
 243     :
 244     : "r" (Data)
 245     );
 246 }
 247
 248 static __inline__ __attribute__((always_inline)) void __writedr2(unsigned long Data)
 249 {
 250   __asm__ __volatile__ (
 251     "mov %0, %%dr2"
 252     :
 253     : "r" (Data)
 254     );
 255 }
 256
 257 static __inline__ __attribute__((always_inline)) void __writedr3(unsigned long Data)
 258 {
 259   __asm__ __volatile__ (
 260     "mov %0, %%dr3"
 261     :
 262     : "r" (Data)
 263     );
 264 }
 265
 266 static __inline__ __attribute__((always_inline)) void __writedr7(unsigned long Data)
 267 {
 268   __asm__ __volatile__ (
 269     "mov %0, %%dr7"
 270     :
 271     : "r" (Data)
 272     );
 273 }
 274
 275 static __inline__ __attribute__((always_inline)) void __writedr(unsigned long reg, unsigned long Data)
 276 {
 277   switch (reg){
 278     case 0:
 279       __writedr0 (Data);
 280       break;
 281
 282     case 1:
 283       __writedr1 (Data);
 284       break;
 285
 286     case 2:
 287       __writedr2 (Data);
 288       break;
 289
 290     case 3:
 291       __writedr3 (Data);
 292       break;
 293
 294     case 7:
 295       __writedr7 (Data);
 296       break;
 297
 298     default:
 299       ;
 300   }
 301 }
 302
 303 static __inline__ __attribute__((always_inline)) unsigned long __readcr0(void)
 304 {
 305   unsigned long value;
 306   __asm__ __volatile__ (
 307     "mov %%cr0, %[value]"
 308     : [value] "=r" (value));
 309   return value;
 310 }
 311
 312 static __inline__ __attribute__((always_inline)) unsigned long __readcr2(void)
 313 {
 314   unsigned long value;
 315   __asm__ __volatile__ (
 316     "mov %%cr2, %[value]"
 317     : [value] "=r" (value));
 318   return value;
 319 }
 320
 321 static __inline__ __attribute__((always_inline)) unsigned long __readcr3(void)
 322 {
 323   unsigned long value;
 324   __asm__ __volatile__ (
 325     "mov %%cr3, %[value]"
 326     : [value] "=r" (value));
 327   return value;
 328 }
 329
 330 static __inline__ __attribute__((always_inline)) unsigned long __readcr4(void)
 331 {
 332   unsigned long value;
 333   __asm__ __volatile__ (
 334     "mov %%cr4, %[value]"
 335     : [value] "=r" (value));
 336   return value;
 337 }
 338
 339 static __inline__ __attribute__((always_inline)) unsigned long __readcr8(void)
 340 {
 341   unsigned long value;
 342   __asm__ __volatile__ (
 343     "mov %%cr8, %[value]"
 344     : [value] "=r" (value));
 345   return value;
 346 }
 347
 348 static __inline__ __attribute__((always_inline)) unsigned long __readcr(unsigned long reg)
 349 {
 350   switch (reg){
 351     case 0:
 352       return __readcr0 ();
 353       break;
 354
 355     case 2:
 356       return __readcr2 ();
 357       break;
 358
 359     case 3:
 360       return __readcr3 ();
 361       break;
 362
 363     case 4:
 364       return __readcr4 ();
 365       break;
 366
 367     case 8:
 368       return __readcr8 ();
 369       break;
 370
 371     default:
 372       return -1;
 373   }
 374 }
 375
 376 static __inline__ __attribute__((always_inline)) void __writecr0(unsigned long Data)
 377 {
 378   __asm__ __volatile__ (
 379     "mov %0, %%cr0"
 380     :
 381     : "r" (Data)
 382     : "memory"
 383     );
 384 }
 385
 386 static __inline__ __attribute__((always_inline)) void __writecr2(unsigned long Data)
 387 {
 388   __asm__ __volatile__ (
 389     "mov %0, %%cr2"
 390     :
 391     : "r" (Data)
 392     );
 393 }
 394
 395 static __inline__ __attribute__((always_inline)) void __writecr3(unsigned long Data)
 396 {
 397   __asm__ __volatile__ (
 398     "mov %0, %%cr3"
 399     :
 400     : "r" (Data)
 401     );
 402 }
 403
 404 static __inline__ __attribute__((always_inline)) void __writecr4(unsigned long Data)
 405 {
 406   __asm__ __volatile__ (
 407     "mov %0, %%cr4"
 408     :
 409     : "r" (Data)
 410     );
 411 }
 412
 413 static __inline__ __attribute__((always_inline)) void __writecr8(unsigned long Data)
 414 {
 415   __asm__ __volatile__ (
 416     "mov %0, %%cr8"
 417     :
 418     : "r" (Data)
 419     );
 420 }
 421
 422 static __inline__ __attribute__((always_inline)) void __writecr(unsigned long reg, unsigned long Data)
 423 {
 424   switch (reg){
 425     case 0:
 426       __writecr0 (Data);
 427       break;
 428
 429     case 2:
 430       __writecr2 (Data);
 431       break;
 432
 433     case 3:
 434       __writecr3 (Data);
 435       break;
 436
 437     case 4:
 438       __writecr4 (Data);
 439       break;
 440
 441     case 8:
 442       __writecr8 (Data);
 443       break;
 444
 445     default:
 446       ;
 447   }
 448 }
 449
 450 static __inline__ __attribute__((always_inline)) UINT64 __readmsr(UINT32 msr)
 451 {
 452   UINT64 retval;
 453   __asm__ __volatile__(
 454        "rdmsr"
 455        : "=A" (retval)
 456        : "c" (msr)
 457        );
 458    return retval;
 459 }
 460
 461 static __inline__ __attribute__((always_inline)) void __writemsr (UINT32 msr, UINT64 Value)
 462 {
 463   __asm__ __volatile__ (
 464      "wrmsr"
 465      :
 466      : "c" (msr), "A" (Value)
 467      );
 468 }
 469
 470 #if !defined(__clang__)
 471 static __inline__ __attribute__((always_inline)) UINT64 __rdtsc(void)
 472 {
 473   UINT64 retval;
 474   __asm__ __volatile__ (
 475      "rdtsc"
 476      : "=A" (retval));
 477   return retval;
 478 }
 479 #endif
 480
 481 static __inline__ __attribute__((always_inline)) void __cpuid(int CPUInfo[], const int InfoType)
 482 {
 483    __asm__ __volatile__(
 484      "cpuid"
 485      :"=a" (CPUInfo[0]), "=b" (CPUInfo[1]), "=c" (CPUInfo[2]), "=d" (CPUInfo[3])
 486      : "a" (InfoType)
 487      );
 488 }
 489
 490
 491 static __inline__ __attribute__((always_inline)) void _disable(void)
 492 {
 493   __asm__ __volatile__ ("cli");
 494 }
 495
 496
 497 static __inline__ __attribute__((always_inline)) void _enable(void)
 498 {
 499   __asm__ __volatile__ ("sti");
 500 }
 501
 502
 503 static __inline__ __attribute__((always_inline)) void __halt(void)
 504 {
 505   __asm__ __volatile__ ("hlt");
 506 }
 507
 508
 509 static __inline__ __attribute__((always_inline)) void __debugbreak(void)
 510 {
 511   __asm__ __volatile__ ("int3");
 512 }
 513
 514 static __inline__ __attribute__((always_inline)) void __invd(void)
 515 {
 516   __asm__ __volatile__ ("invd");
 517 }
 518
 519 static __inline__ __attribute__((always_inline)) void __wbinvd(void)
 520 {
 521   __asm__ __volatile__ ("wbinvd");
 522 }
 523
 524 static __inline__ __attribute__((always_inline)) void __lidt(void *Source)
 525 {
 526   __asm__ __volatile__("lidt %0" : : "m"(*(short*)Source));
 527 }
 528
 529 static __inline__ __attribute__((always_inline)) void
 530 __writefsbyte(const unsigned long Offset, const uint8_t Data)
 531 {
 532   __asm__ ("movb %[Data], %%fs:%a[Offset]"
 533           :
 534           : [Offset] "ir" (Offset), [Data] "iq" (Data));
 535 }
 536
 537 static __inline__ __attribute__((always_inline)) void
 538 __writefsword(const unsigned long Offset, const uint16_t Data)
 539 {
 540   __asm__ ("movw %[Data], %%fs:%a[Offset]"
 541           :
 542           : [Offset] "ir" (Offset), [Data] "ir" (Data));
 543 }
 544
 545 static __inline__ __attribute__((always_inline)) void
 546 __writefsdword(const unsigned long Offset, const uint32_t Data)
 547 {
 548   __asm__ ("movl %[Data], %%fs:%a[Offset]"
 549            :
 550            : [Offset] "ir" (Offset), [Data] "ir" (Data));
 551 }
 552
 553 static __inline__ __attribute__((always_inline)) uint8_t
 554 __readfsbyte(const unsigned long Offset)
 555 {
 556   unsigned char value;
 557   __asm__ ("movb %%fs:%a[Offset], %[value]"
 558           : [value] "=q" (value)
 559           : [Offset] "ir" (Offset));
 560   return value;
 561 }
 562
 563 static __inline__ __attribute__((always_inline)) uint16_t
 564 __readfsword(const unsigned long Offset)
 565 {
 566   unsigned short value;
 567   __asm__ ("movw %%fs:%a[Offset], %[value]"
 568            : [value] "=q" (value)
 569            : [Offset] "ir" (Offset));
 570   return value;
 571 }
 572
 573 static __inline__ __attribute__((always_inline)) uint32_t
 574 __readfsdword(unsigned long Offset)
 575 {
 576   unsigned long value;
 577   __asm__ ("mov %%fs:%a[Offset], %[value]"
 578            : [value] "=r" (value)
 579            : [Offset] "ir" (Offset));
 580   return value;
 581 }
 582
 583 #ifdef __SSE3__
 584 typedef long long __v2di __attribute__((__vector_size__ (16)));
 585 typedef long long __m128i __attribute__((__vector_size__ (16), __may_alias__));
 586
 587 static __inline__ __attribute__((always_inline)) void _mm_stream_si128_fs2 (void *__A, __m128i __B)
 588 {
 589   __asm__(".byte 0x64"); // fs prefix
 590 #if defined(__clang__)
 591   __builtin_nontemporal_store((__v2di)__B, (__v2di *)__A);
 592 #else
 593   __builtin_ia32_movntdq ((__v2di *)__A, (__v2di)__B);
 594 #endif
 595 }
 596
 597 static __inline__ __attribute__((always_inline)) void _mm_stream_si128_fs (void *__A, void *__B)
 598 {
 599   __m128i data;
 600   data = (__m128i) __builtin_ia32_lddqu ((char const *)__B);
 601   _mm_stream_si128_fs2 (__A, data);
 602 }
 603
 604 static __inline__ __attribute__((always_inline)) void _mm_clflush_fs (void *__A)
 605 {
 606   __asm__(".byte 0x64"); // fs prefix
 607   __builtin_ia32_clflush (__A);
 608 }
 609
 610 #if !defined(__clang__)
 611 static __inline __attribute__(( __always_inline__)) void _mm_mfence (void)
 612 {
 613   __builtin_ia32_mfence ();
 614 }
 615 #else
 616 void _mm_mfence(void);
 617 #endif
 618
 619 #if !defined(__clang__)
 620 static __inline __attribute__(( __always_inline__)) void _mm_sfence (void)
 621 {
 622   __builtin_ia32_sfence ();
 623 }
 624 #else
 625 void _mm_sfence(void);
 626 #endif
 627 #endif /* __SSE3__ */
 628
 629 #endif /* defined (__GNUC__) */